3 * Backend for uploading files from chunks.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Implements uploading from chunks
28 * @author Michael Dale
30 class UploadFromChunks
extends UploadFromFile
{
31 protected $mOffset, $mChunkIndex, $mFileKey, $mVirtualTempPath;
34 * Setup local pointers to stash, repo and user ( similar to UploadFromStash )
37 * @param $stash UploadStash
38 * @param $repo FileRepo
40 public function __construct( $user = false, $stash = false, $repo = false ) {
41 // user object. sometimes this won't exist, as when running from cron.
47 $this->repo
= RepoGroup
::singleton()->getLocalRepo();
51 $this->stash
= $stash;
54 wfDebug( __METHOD__
. " creating new UploadFromChunks instance for " . $user->getId() . "\n" );
56 wfDebug( __METHOD__
. " creating new UploadFromChunks instance with no user\n" );
58 $this->stash
= new UploadStash( $this->repo
, $this->user
);
64 * Calls the parent stashFile and updates the uploadsession table to handle "chunks"
66 * @return UploadStashFile stashed file
68 public function stashFile() {
69 // Stash file is the called on creating a new chunk session:
70 $this->mChunkIndex
= 0;
72 // Create a local stash target
73 $this->mLocalFile
= parent
::stashFile();
74 // Update the initial file offset ( based on file size )
75 $this->mOffset
= $this->mLocalFile
->getSize();
76 $this->mFileKey
= $this->mLocalFile
->getFileKey();
78 // Output a copy of this first to chunk 0 location:
79 $this->outputChunk( $this->mLocalFile
->getPath() );
81 // Update db table to reflect initial "chunk" state
82 $this->updateChunkStatus();
83 return $this->mLocalFile
;
87 * Continue chunk uploading
89 public function continueChunks( $name, $key, $webRequestUpload ) {
90 $this->mFileKey
= $key;
91 $this->mUpload
= $webRequestUpload;
92 // Get the chunk status form the db:
93 $this->getChunkStatus();
95 $metadata = $this->stash
->getMetadata( $key );
96 $this->initializePathInfo( $name,
97 $this->getRealPath( $metadata['us_path'] ),
104 * Append the final chunk and ready file for parent::performUpload()
105 * @return FileRepoStatus
107 public function concatenateChunks() {
108 wfDebug( __METHOD__
. " concatenate {$this->mChunkIndex} chunks:" .
109 $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
111 // Concatenate all the chunks to mVirtualTempPath
113 // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
114 for( $i = 0; $i <= $this->getChunkIndex(); $i++
){
115 $fileList[] = $this->getVirtualChunkLocation( $i );
118 // Get the file extension from the last chunk
119 $ext = FileBackend
::extensionFromPath( $this->mVirtualTempPath
);
120 // Get a 0-byte temp file to perform the concatenation at
121 $tmpFile = TempFSFile
::factory( 'chunkedupload_', $ext );
123 ?
$tmpFile->bind( $this )->getPath() // keep alive with $this
124 : false; // fail in concatenate()
125 // Concatenate the chunks at the temp file
126 $tStart = microtime( true );
127 $status = $this->repo
->concatenate( $fileList, $tmpPath, FileRepo
::DELETE_SOURCE
);
128 $tAmount = microtime( true ) - $tStart;
129 if( !$status->isOk() ){
132 wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds.\n" );
133 // Update the mTempPath and mLocalFile
134 // ( for FileUpload or normal Stash to take over )
135 $this->mTempPath
= $tmpPath; // file system path
136 $tStart = microtime( true );
137 $this->mLocalFile
= parent
::stashFile();
138 $tAmount = microtime( true ) - $tStart;
139 $this->mLocalFile
->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
140 wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds.\n" );
146 * Perform the upload, then remove the temp copy afterward
147 * @param $comment string
148 * @param $pageText string
153 public function performUpload( $comment, $pageText, $watch, $user ) {
154 $rv = parent
::performUpload( $comment, $pageText, $watch, $user );
159 * Returns the virtual chunk location:
163 function getVirtualChunkLocation( $index ){
164 return $this->repo
->getVirtualUrl( 'temp' ) .
166 $this->repo
->getHashPath(
167 $this->getChunkFileKey( $index )
169 $this->getChunkFileKey( $index );
173 * Add a chunk to the temporary directory
175 * @param $chunkPath string path to temporary chunk file
176 * @param $chunkSize int size of the current chunk
177 * @param $offset int offset of current chunk ( mutch match database chunk offset )
180 public function addChunk( $chunkPath, $chunkSize, $offset ) {
181 // Get the offset before we add the chunk to the file system
182 $preAppendOffset = $this->getOffset();
184 if ( $preAppendOffset +
$chunkSize > $this->getMaxUploadSize()) {
185 $status = Status
::newFatal( 'file-too-large' );
187 // Make sure the client is uploading the correct chunk with a matching offset.
188 if ( $preAppendOffset == $offset ) {
189 // Update local chunk index for the current chunk
190 $this->mChunkIndex++
;
191 $status = $this->outputChunk( $chunkPath );
192 if( $status->isGood() ){
193 // Update local offset:
194 $this->mOffset
= $preAppendOffset +
$chunkSize;
195 // Update chunk table status db
196 $this->updateChunkStatus();
199 $status = Status
::newFatal( 'invalid-chunk-offset' );
206 * Update the chunk db table with the current status:
208 private function updateChunkStatus(){
209 wfDebug( __METHOD__
. " update chunk status for {$this->mFileKey} offset:" .
210 $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
212 $dbw = $this->repo
->getMasterDb();
213 // Use a quick transaction since we will upload the full temp file into shared
214 // storage, which takes time for large files. We don't want to hold locks then.
219 'us_status' => 'chunks',
220 'us_chunk_inx' => $this->getChunkIndex(),
221 'us_size' => $this->getOffset()
223 array( 'us_key' => $this->mFileKey
),
230 * Get the chunk db state and populate update relevant local values
232 private function getChunkStatus(){
233 // get Master db to avoid race conditions.
234 // Otherwise, if chunk upload time < replag there will be spurious errors
235 $dbw = $this->repo
->getMasterDb();
236 $row = $dbw->selectRow(
243 array( 'us_key' => $this->mFileKey
),
248 $this->mChunkIndex
= $row->us_chunk_inx
;
249 $this->mOffset
= $row->us_size
;
250 $this->mVirtualTempPath
= $row->us_path
;
255 * Get the current Chunk index
256 * @return Integer index of the current chunk
258 private function getChunkIndex(){
259 if( $this->mChunkIndex
!== null ){
260 return $this->mChunkIndex
;
266 * Gets the current offset in fromt the stashedupload table
267 * @return Integer current byte offset of the chunk file set
269 private function getOffset(){
270 if ( $this->mOffset
!== null ){
271 return $this->mOffset
;
277 * Output the chunk to disk
279 * @param $chunkPath string
280 * @throws UploadChunkFileException
281 * @return FileRepoStatus
283 private function outputChunk( $chunkPath ){
284 // Key is fileKey + chunk index
285 $fileKey = $this->getChunkFileKey();
287 // Store the chunk per its indexed fileKey:
288 $hashPath = $this->repo
->getHashPath( $fileKey );
289 $storeStatus = $this->repo
->quickImport( $chunkPath,
290 $this->repo
->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" );
292 // Check for error in stashing the chunk:
293 if ( ! $storeStatus->isOK() ) {
294 $error = $storeStatus->getErrorsArray();
295 $error = reset( $error );
296 if ( ! count( $error ) ) {
297 $error = $storeStatus->getWarningsArray();
298 $error = reset( $error );
299 if ( ! count( $error ) ) {
300 $error = array( 'unknown', 'no error recorded' );
303 throw new UploadChunkFileException( "error storing file in '$chunkPath': " . implode( '; ', $error ) );
308 private function getChunkFileKey( $index = null ){
309 if( $index === null ){
310 $index = $this->getChunkIndex();
312 return $this->mFileKey
. '.' . $index ;
316 class UploadChunkZeroLengthFileException
extends MWException
{};
317 class UploadChunkFileException
extends MWException
{};