Added async upload-from-stash support.
[lhc/web/wiklou.git] / includes / api / ApiUpload.php
index 3a9b5c5..2e18e8a 100644 (file)
@@ -51,6 +51,8 @@ class ApiUpload extends ApiBase {
                // Parameter handling
                $this->mParams = $this->extractRequestParams();
                $request = $this->getMain()->getRequest();
+               // Check if async mode is actually supported
+               $this->mParams['async'] = ( $this->mParams['async'] && !wfIsWindows() );
                // Add the uploaded file to the params array
                $this->mParams['file'] = $request->getFileName( 'file' );
                $this->mParams['chunk'] = $request->getFileName( 'chunk' );
@@ -62,17 +64,15 @@ class ApiUpload extends ApiBase {
 
                // Select an upload module
                if ( !$this->selectUploadModule() ) {
-                       // This is not a true upload, but a status request or similar
-                       return;
-               }
-               if ( !isset( $this->mUpload ) ) {
+                       return; // not a true upload, but a status request or similar
+               } elseif ( !isset( $this->mUpload ) ) {
                        $this->dieUsage( 'No upload module set', 'nomodule' );
                }
 
                // First check permission to upload
                $this->checkPermissions( $user );
 
-               // Fetch the file
+               // Fetch the file (usually a no-op)
                $status = $this->mUpload->fetchFile();
                if ( !$status->isGood() ) {
                        $errors = $status->getErrorsArray();
@@ -86,22 +86,27 @@ class ApiUpload extends ApiBase {
                        if( $this->mParams['filesize'] > $maxSize ) {
                                $this->dieUsage( 'The file you submitted was too large', 'file-too-large' );
                        }
+                       if ( !$this->mUpload->getTitle() ) {
+                               $this->dieUsage( 'Invalid file title supplied', 'internal-error' );
+                       }
+               } elseif ( $this->mParams['async'] ) {
+                       // defer verification to background process
                } else {
                        $this->verifyUpload();
                }
+
                // Check if the user has the rights to modify or overwrite the requested title
                // (This check is irrelevant if stashing is already requested, since the errors
                //  can always be fixed by changing the title)
-               if ( ! $this->mParams['stash'] ) {
+               if ( !$this->mParams['stash'] ) {
                        $permErrors = $this->mUpload->verifyTitlePermissions( $user );
                        if ( $permErrors !== true ) {
                                $this->dieRecoverableError( $permErrors[0], 'filename' );
                        }
                }
-               // Get the result based on the current upload context: 
-               $result = $this->getContextResult();
 
+               // Get the result based on the current upload context:
+               $result = $this->getContextResult();
                if ( $result['result'] === 'Success' ) {
                        $result['imageinfo'] = $this->mUpload->getImageInfo( $this->getResult() );
                }
@@ -111,6 +116,7 @@ class ApiUpload extends ApiBase {
                // Cleanup any temporary mess
                $this->mUpload->cleanupTempFile();
        }
+
        /**
         * Get an uplaod result based on upload context
         * @return array
@@ -131,6 +137,7 @@ class ApiUpload extends ApiBase {
                // performUpload will return a formatted properly for the API with status
                return $this->performUpload( $warnings );
        }
+
        /**
         * Get Stash Result, throws an expetion if the file could not be stashed.
         * @param $warnings array Array of Api upload warnings
@@ -152,6 +159,7 @@ class ApiUpload extends ApiBase {
                }
                return $result;
        }
+
        /**
         * Get Warnings Result
         * @param $warnings array Array of Api upload warnings
@@ -171,12 +179,15 @@ class ApiUpload extends ApiBase {
                }
                return $result;
        }
+
        /**
         * Get the result of a chunk upload.
         * @param $warnings array Array of Api upload warnings
         * @return array
         */
-       private function getChunkResult( $warnings ){
+       private function getChunkResult( $warnings ) {
+               global $IP;
+
                $result = array();
 
                $result['result'] = 'Continue';
@@ -189,32 +200,63 @@ class ApiUpload extends ApiBase {
                if ($this->mParams['offset'] == 0) {
                        $result['filekey'] = $this->performStash();
                } else {
-                       $status = $this->mUpload->addChunk($chunkPath, $chunkSize,
-                                                                               $this->mParams['offset']);
+                       $status = $this->mUpload->addChunk(
+                               $chunkPath, $chunkSize, $this->mParams['offset'] );
                        if ( !$status->isGood() ) {
                                $this->dieUsage( $status->getWikiText(), 'stashfailed' );
                                return array();
                        }
 
-                       // Check we added the last chunk: 
+                       // Check we added the last chunk:
                        if( $this->mParams['offset'] + $chunkSize == $this->mParams['filesize'] ) {
-                               $status = $this->mUpload->concatenateChunks();
-
-                               if ( !$status->isGood() ) {
-                                       $this->dieUsage( $status->getWikiText(), 'stashfailed' );
-                                       return array();
-                               }
-
-                               // We have a new filekey for the fully concatenated file.
-                               $result['filekey'] =  $this->mUpload->getLocalFile()->getFileKey();
+                               if ( $this->mParams['async'] && !wfIsWindows() ) {
+                                       $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
+                                       if ( $progress && $progress['result'] === 'Poll' ) {
+                                               $this->dieUsage( "Chunk assembly already in progress.", 'stashfailed' );
+                                       }
+                                       UploadBase::setSessionStatus(
+                                               $this->mParams['filekey'],
+                                               array( 'result' => 'Poll',
+                                                       'stage' => 'queued', 'status' => Status::newGood() )
+                                       );
+                                       $retVal = 1;
+                                       $cmd = wfShellWikiCmd(
+                                               "$IP/includes/upload/AssembleUploadChunks.php",
+                                               array(
+                                                       '--wiki', wfWikiID(),
+                                                       '--filename', $this->mParams['filename'],
+                                                       '--filekey', $this->mParams['filekey'],
+                                                       '--userid', $this->getUser()->getId(),
+                                                       '--sessionid', session_id(),
+                                                       '--quiet'
+                                               )
+                                       ) . " < " . wfGetNull() . " > " . wfGetNull() . " 2>&1 &";
+                                       // Start a process in the background. Enforce the time limits via PHP
+                                       // since ulimit4.sh seems to often not work for this particular usage.
+                                       wfShellExec( $cmd, $retVal, array(), array( 'time' => 0, 'memory' => 0 ) );
+                                       if ( $retVal == 0 ) {
+                                               $result['result'] = 'Poll';
+                                       } else {
+                                               UploadBase::setSessionStatus( $this->mParams['filekey'], false );
+                                               $this->dieUsage(
+                                                       "Failed to start AssembleUploadChunks.php", 'stashfailed' );
+                                       }
+                               } else {
+                                       $status = $this->mUpload->concatenateChunks();
+                                       if ( !$status->isGood() ) {
+                                               $this->dieUsage( $status->getWikiText(), 'stashfailed' );
+                                               return array();
+                                       }
 
-                               // Remove chunk from stash. (Checks against user ownership of chunks.)
-                               $this->mUpload->stash->removeFile( $this->mParams['filekey'] );
+                                       // We have a new filekey for the fully concatenated file.
+                                       $result['filekey'] =  $this->mUpload->getLocalFile()->getFileKey();
 
-                               $result['result'] = 'Success';
+                                       // Remove chunk from stash. (Checks against user ownership of chunks.)
+                                       $this->mUpload->stash->removeFile( $this->mParams['filekey'] );
 
+                                       $result['result'] = 'Success';
+                               }
                        } else {
-
                                // Continue passing through the filekey for adding further chunks.
                                $result['filekey'] = $this->mParams['filekey'];
                        }
@@ -222,14 +264,14 @@ class ApiUpload extends ApiBase {
                $result['offset'] = $this->mParams['offset'] + $chunkSize;
                return $result;
        }
-       
+
        /**
         * Stash the file and return the file key
         * Also re-raises exceptions with slightly more informative message strings (useful for API)
         * @throws MWException
         * @return String file key
         */
-       function performStash() {
+       private function performStash() {
                try {
                        $stashFile = $this->mUpload->stashFile();
 
@@ -254,7 +296,7 @@ class ApiUpload extends ApiBase {
         * @param $data array Optional extra data to pass to the user
         * @throws UsageException
         */
-       function dieRecoverableError( $error, $parameter, $data = array() ) {
+       private function dieRecoverableError( $error, $parameter, $data = array() ) {
                try {
                        $data['filekey'] = $this->performStash();
                        $data['sessionkey'] = $data['filekey'];
@@ -278,11 +320,27 @@ class ApiUpload extends ApiBase {
                $request = $this->getMain()->getRequest();
 
                // chunk or one and only one of the following parameters is needed
-               if( !$this->mParams['chunk'] ) {
+               if ( !$this->mParams['chunk'] ) {
                        $this->requireOnlyOneParameter( $this->mParams,
                                'filekey', 'file', 'url', 'statuskey' );
                }
 
+               // Status report for "upload to stash"/"upload from stash"
+               if ( $this->mParams['filekey'] && $this->mParams['checkstatus'] ) {
+                       $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
+                       if ( !$progress ) {
+                               $this->dieUsage( 'No result in status data', 'missingresult' );
+                       } elseif ( !$progress['status']->isGood() ) {
+                               $this->dieUsage( $progress['status']->getWikiText(), 'stashfailed' );
+                       }
+                       if ( isset( $progress['status']->value['verification'] ) ) {
+                               $this->checkVerification( $progress['status']->value['verification'] );
+                       }
+                       unset( $progress['status'] ); // remove Status object
+                       $this->getResult()->addValue( null, $this->getModuleName(), $progress );
+                       return false;
+               }
+
                if ( $this->mParams['statuskey'] ) {
                        $this->checkAsyncDownloadEnabled();
 
@@ -297,7 +355,6 @@ class ApiUpload extends ApiBase {
                        }
                        $this->getResult()->addValue( null, $this->getModuleName(), $sessionData );
                        return false;
-
                }
 
                // The following modules all require the filename parameter to be set
@@ -329,8 +386,11 @@ class ApiUpload extends ApiBase {
                        }
 
                        $this->mUpload = new UploadFromStash( $this->getUser() );
-
-                       $this->mUpload->initialize( $this->mParams['filekey'], $this->mParams['filename'] );
+                       // This will not download the temp file in initialize() in async mode.
+                       // We still have enough information to call checkWarnings() and such.
+                       $this->mUpload->initialize(
+                               $this->mParams['filekey'], $this->mParams['filename'], !$this->mParams['async']
+                       );
                } elseif ( isset( $this->mParams['file'] ) ) {
                        $this->mUpload = new UploadFromFile();
                        $this->mUpload->initialize(
@@ -392,12 +452,19 @@ class ApiUpload extends ApiBase {
         * Performs file verification, dies on error.
         */
        protected function verifyUpload( ) {
-               global $wgFileExtensions;
-
                $verification = $this->mUpload->verifyUpload( );
                if ( $verification['status'] === UploadBase::OK ) {
                        return;
+               } else {
+                       return $this->checkVerification( $verification );
                }
+       }
+
+       /**
+        * Performs file verification, dies on error.
+        */
+       protected function checkVerification( array $verification ) {
+               global $wgFileExtensions;
 
                // TODO: Move them to ApiBase's message map
                switch( $verification['status'] ) {
@@ -507,6 +574,8 @@ class ApiUpload extends ApiBase {
         * @return array
         */
        protected function performUpload( $warnings ) {
+               global $IP;
+
                // Use comment as initial page text by default
                if ( is_null( $this->mParams['text'] ) ) {
                        $this->mParams['text'] = $this->mParams['comment'];
@@ -521,29 +590,63 @@ class ApiUpload extends ApiBase {
                }
 
                // No errors, no warnings: do the upload
-               $status = $this->mUpload->performUpload( $this->mParams['comment'],
-                       $this->mParams['text'], $watch, $this->getUser() );
-
-               if ( !$status->isGood() ) {
-                       $error = $status->getErrorsArray();
-
-                       if ( count( $error ) == 1 && $error[0][0] == 'async' ) {
-                               // The upload can not be performed right now, because the user
-                               // requested so
-                               return array(
-                                       'result' => 'Queued',
-                                       'statuskey' => $error[0][1],
-                               );
+               if ( $this->mParams['async'] ) {
+                       $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
+                       if ( $progress && $progress['result'] === 'Poll' ) {
+                               $this->dieUsage( "Upload from stash already in progress.", 'publishfailed' );
+                       }
+                       UploadBase::setSessionStatus(
+                               $this->mParams['filekey'],
+                               array( 'result' => 'Poll', 'stage' => 'queued', 'status' => Status::newGood() )
+                       );
+                       $retVal = 1;
+                       $cmd = wfShellWikiCmd(
+                               "$IP/includes/upload/PublishStashedFile.php",
+                               array(
+                                       '--wiki', wfWikiID(),
+                                       '--filename', $this->mParams['filename'],
+                                       '--filekey', $this->mParams['filekey'],
+                                       '--userid', $this->getUser()->getId(),
+                                       '--comment', $this->mParams['comment'],
+                                       '--text', $this->mParams['text'],
+                                       '--watch', $watch,
+                                       '--sessionid', session_id(),
+                                       '--quiet'
+                               )
+                       ) . " < " . wfGetNull() . " > " . wfGetNull() . " 2>&1 &";
+                       // Start a process in the background. Enforce the time limits via PHP
+                       // since ulimit4.sh seems to often not work for this particular usage.
+                       wfShellExec( $cmd, $retVal, array(), array( 'time' => 0, 'memory' => 0 ) );
+                       if ( $retVal == 0 ) {
+                               $result['result'] = 'Poll';
                        } else {
-                               $this->getResult()->setIndexedTagName( $error, 'error' );
+                               UploadBase::setSessionStatus( $this->mParams['filekey'], false );
+                               $this->dieUsage(
+                                       "Failed to start PublishStashedFile.php", 'publishfailed' );
+                       }
+               } else {
+                       $status = $this->mUpload->performUpload( $this->mParams['comment'],
+                               $this->mParams['text'], $watch, $this->getUser() );
 
-                               $this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error );
+                       if ( !$status->isGood() ) {
+                               $error = $status->getErrorsArray();
+
+                               if ( count( $error ) == 1 && $error[0][0] == 'async' ) {
+                                       // The upload can not be performed right now, because the user
+                                       // requested so
+                                       return array(
+                                               'result' => 'Queued',
+                                               'statuskey' => $error[0][1],
+                                       );
+                               } else {
+                                       $this->getResult()->setIndexedTagName( $error, 'error' );
+
+                                       $this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error );
+                               }
                        }
+                       $result['result'] = 'Success';
                }
 
-               $file = $this->mUpload->getLocalFile();
-
-               $result['result'] = 'Success';
                $result['filename'] = $file->getName();
                if ( $warnings && count( $warnings ) > 0 ) {
                        $result['warnings'] = $warnings;
@@ -609,9 +712,11 @@ class ApiUpload extends ApiBase {
                        'offset' => null,
                        'chunk' => null,
 
+                       'async' => false,
                        'asyncdownload' => false,
                        'leavemessage' => false,
                        'statuskey' => null,
+                       'checkstatus' => false,
                );
 
                return $params;
@@ -636,9 +741,11 @@ class ApiUpload extends ApiBase {
                        'offset' => 'Offset of chunk in bytes',
                        'filesize' => 'Filesize of entire upload',
 
+                       'async' => 'Make potentially large file operations asynchronous when possible',
                        'asyncdownload' => 'Make fetching a URL asynchronous',
                        'leavemessage' => 'If asyncdownload is used, leave a message on the user talk page if finished',
-                       'statuskey' => 'Fetch the upload status for this file key',
+                       'statuskey' => 'Fetch the upload status for this file key (upload by URL)',
+                       'checkstatus' => 'Only fetch the upload status for the given file key',
                );
 
                return $params;
@@ -707,6 +814,7 @@ class ApiUpload extends ApiBase {
                                array( 'code' => 'filename-tooshort', 'info' => 'The filename is too short' ),
                                array( 'code' => 'overwrite', 'info' => 'Overwriting an existing file is not allowed' ),
                                array( 'code' => 'stashfailed', 'info' => 'Stashing temporary file failed' ),
+                               array( 'code' => 'publishfailed', 'info' => 'Publishing of stashed file failed' ),
                                array( 'code' => 'internal-error', 'info' => 'An internal error occurred' ),
                                array( 'code' => 'asynccopyuploaddisabled', 'info' => 'Asynchronous copy uploads disabled' ),
                                array( 'fileexists-forbidden' ),