Added async upload-from-stash support.
authorAaron Schulz <aschulz@wikimedia.org>
Tue, 4 Dec 2012 02:18:48 +0000 (18:18 -0800)
committerGerrit Code Review <gerrit@wikimedia.org>
Wed, 9 Jan 2013 02:13:37 +0000 (02:13 +0000)
* This works similarly to the async concatenation code.

Change-Id: Iae38b93d65182158561b92168af51a1e9f50374c

includes/api/ApiUpload.php
includes/upload/PublishStashedFile.php [new file with mode: 0644]
includes/upload/UploadBase.php
includes/upload/UploadFromStash.php

index cfa1683..2e18e8a 100644 (file)
@@ -51,6 +51,8 @@ class ApiUpload extends ApiBase {
                // Parameter handling
                $this->mParams = $this->extractRequestParams();
                $request = $this->getMain()->getRequest();
+               // Check if async mode is actually supported
+               $this->mParams['async'] = ( $this->mParams['async'] && !wfIsWindows() );
                // Add the uploaded file to the params array
                $this->mParams['file'] = $request->getFileName( 'file' );
                $this->mParams['chunk'] = $request->getFileName( 'chunk' );
@@ -62,17 +64,15 @@ class ApiUpload extends ApiBase {
 
                // Select an upload module
                if ( !$this->selectUploadModule() ) {
-                       // This is not a true upload, but a status request or similar
-                       return;
-               }
-               if ( !isset( $this->mUpload ) ) {
+                       return; // not a true upload, but a status request or similar
+               } elseif ( !isset( $this->mUpload ) ) {
                        $this->dieUsage( 'No upload module set', 'nomodule' );
                }
 
                // First check permission to upload
                $this->checkPermissions( $user );
 
-               // Fetch the file
+               // Fetch the file (usually a no-op)
                $status = $this->mUpload->fetchFile();
                if ( !$status->isGood() ) {
                        $errors = $status->getErrorsArray();
@@ -89,6 +89,8 @@ class ApiUpload extends ApiBase {
                        if ( !$this->mUpload->getTitle() ) {
                                $this->dieUsage( 'Invalid file title supplied', 'internal-error' );
                        }
+               } elseif ( $this->mParams['async'] ) {
+                       // defer verification to background process
                } else {
                        $this->verifyUpload();
                }
@@ -96,15 +98,15 @@ class ApiUpload extends ApiBase {
                // Check if the user has the rights to modify or overwrite the requested title
                // (This check is irrelevant if stashing is already requested, since the errors
                //  can always be fixed by changing the title)
-               if ( ! $this->mParams['stash'] ) {
+               if ( !$this->mParams['stash'] ) {
                        $permErrors = $this->mUpload->verifyTitlePermissions( $user );
                        if ( $permErrors !== true ) {
                                $this->dieRecoverableError( $permErrors[0], 'filename' );
                        }
                }
+
                // Get the result based on the current upload context:
                $result = $this->getContextResult();
-
                if ( $result['result'] === 'Success' ) {
                        $result['imageinfo'] = $this->mUpload->getImageInfo( $this->getResult() );
                }
@@ -135,6 +137,7 @@ class ApiUpload extends ApiBase {
                // performUpload will return a formatted properly for the API with status
                return $this->performUpload( $warnings );
        }
+
        /**
         * Get Stash Result, throws an expetion if the file could not be stashed.
         * @param $warnings array Array of Api upload warnings
@@ -156,6 +159,7 @@ class ApiUpload extends ApiBase {
                }
                return $result;
        }
+
        /**
         * Get Warnings Result
         * @param $warnings array Array of Api upload warnings
@@ -175,6 +179,7 @@ class ApiUpload extends ApiBase {
                }
                return $result;
        }
+
        /**
         * Get the result of a chunk upload.
         * @param $warnings array Array of Api upload warnings
@@ -206,7 +211,7 @@ class ApiUpload extends ApiBase {
                        if( $this->mParams['offset'] + $chunkSize == $this->mParams['filesize'] ) {
                                if ( $this->mParams['async'] && !wfIsWindows() ) {
                                        $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
-                                       if ( $progress && $progress['result'] !== 'Failed' ) {
+                                       if ( $progress && $progress['result'] === 'Poll' ) {
                                                $this->dieUsage( "Chunk assembly already in progress.", 'stashfailed' );
                                        }
                                        UploadBase::setSessionStatus(
@@ -266,7 +271,7 @@ class ApiUpload extends ApiBase {
         * @throws MWException
         * @return String file key
         */
-       function performStash() {
+       private function performStash() {
                try {
                        $stashFile = $this->mUpload->stashFile();
 
@@ -291,7 +296,7 @@ class ApiUpload extends ApiBase {
         * @param $data array Optional extra data to pass to the user
         * @throws UsageException
         */
-       function dieRecoverableError( $error, $parameter, $data = array() ) {
+       private function dieRecoverableError( $error, $parameter, $data = array() ) {
                try {
                        $data['filekey'] = $this->performStash();
                        $data['sessionkey'] = $data['filekey'];
@@ -320,6 +325,7 @@ class ApiUpload extends ApiBase {
                                'filekey', 'file', 'url', 'statuskey' );
                }
 
+               // Status report for "upload to stash"/"upload from stash"
                if ( $this->mParams['filekey'] && $this->mParams['checkstatus'] ) {
                        $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
                        if ( !$progress ) {
@@ -327,6 +333,9 @@ class ApiUpload extends ApiBase {
                        } elseif ( !$progress['status']->isGood() ) {
                                $this->dieUsage( $progress['status']->getWikiText(), 'stashfailed' );
                        }
+                       if ( isset( $progress['status']->value['verification'] ) ) {
+                               $this->checkVerification( $progress['status']->value['verification'] );
+                       }
                        unset( $progress['status'] ); // remove Status object
                        $this->getResult()->addValue( null, $this->getModuleName(), $progress );
                        return false;
@@ -377,8 +386,11 @@ class ApiUpload extends ApiBase {
                        }
 
                        $this->mUpload = new UploadFromStash( $this->getUser() );
-
-                       $this->mUpload->initialize( $this->mParams['filekey'], $this->mParams['filename'] );
+                       // This will not download the temp file in initialize() in async mode.
+                       // We still have enough information to call checkWarnings() and such.
+                       $this->mUpload->initialize(
+                               $this->mParams['filekey'], $this->mParams['filename'], !$this->mParams['async']
+                       );
                } elseif ( isset( $this->mParams['file'] ) ) {
                        $this->mUpload = new UploadFromFile();
                        $this->mUpload->initialize(
@@ -440,12 +452,19 @@ class ApiUpload extends ApiBase {
         * Performs file verification, dies on error.
         */
        protected function verifyUpload( ) {
-               global $wgFileExtensions;
-
                $verification = $this->mUpload->verifyUpload( );
                if ( $verification['status'] === UploadBase::OK ) {
                        return;
+               } else {
+                       return $this->checkVerification( $verification );
                }
+       }
+
+       /**
+        * Performs file verification, dies on error.
+        */
+       protected function checkVerification( array $verification ) {
+               global $wgFileExtensions;
 
                // TODO: Move them to ApiBase's message map
                switch( $verification['status'] ) {
@@ -555,6 +574,8 @@ class ApiUpload extends ApiBase {
         * @return array
         */
        protected function performUpload( $warnings ) {
+               global $IP;
+
                // Use comment as initial page text by default
                if ( is_null( $this->mParams['text'] ) ) {
                        $this->mParams['text'] = $this->mParams['comment'];
@@ -569,29 +590,63 @@ class ApiUpload extends ApiBase {
                }
 
                // No errors, no warnings: do the upload
-               $status = $this->mUpload->performUpload( $this->mParams['comment'],
-                       $this->mParams['text'], $watch, $this->getUser() );
-
-               if ( !$status->isGood() ) {
-                       $error = $status->getErrorsArray();
-
-                       if ( count( $error ) == 1 && $error[0][0] == 'async' ) {
-                               // The upload can not be performed right now, because the user
-                               // requested so
-                               return array(
-                                       'result' => 'Queued',
-                                       'statuskey' => $error[0][1],
-                               );
+               if ( $this->mParams['async'] ) {
+                       $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] );
+                       if ( $progress && $progress['result'] === 'Poll' ) {
+                               $this->dieUsage( "Upload from stash already in progress.", 'publishfailed' );
+                       }
+                       UploadBase::setSessionStatus(
+                               $this->mParams['filekey'],
+                               array( 'result' => 'Poll', 'stage' => 'queued', 'status' => Status::newGood() )
+                       );
+                       $retVal = 1;
+                       $cmd = wfShellWikiCmd(
+                               "$IP/includes/upload/PublishStashedFile.php",
+                               array(
+                                       '--wiki', wfWikiID(),
+                                       '--filename', $this->mParams['filename'],
+                                       '--filekey', $this->mParams['filekey'],
+                                       '--userid', $this->getUser()->getId(),
+                                       '--comment', $this->mParams['comment'],
+                                       '--text', $this->mParams['text'],
+                                       '--watch', $watch,
+                                       '--sessionid', session_id(),
+                                       '--quiet'
+                               )
+                       ) . " < " . wfGetNull() . " > " . wfGetNull() . " 2>&1 &";
+                       // Start a process in the background. Enforce the time limits via PHP
+                       // since ulimit4.sh seems to often not work for this particular usage.
+                       wfShellExec( $cmd, $retVal, array(), array( 'time' => 0, 'memory' => 0 ) );
+                       if ( $retVal == 0 ) {
+                               $result['result'] = 'Poll';
                        } else {
-                               $this->getResult()->setIndexedTagName( $error, 'error' );
+                               UploadBase::setSessionStatus( $this->mParams['filekey'], false );
+                               $this->dieUsage(
+                                       "Failed to start PublishStashedFile.php", 'publishfailed' );
+                       }
+               } else {
+                       $status = $this->mUpload->performUpload( $this->mParams['comment'],
+                               $this->mParams['text'], $watch, $this->getUser() );
+
+                       if ( !$status->isGood() ) {
+                               $error = $status->getErrorsArray();
+
+                               if ( count( $error ) == 1 && $error[0][0] == 'async' ) {
+                                       // The upload can not be performed right now, because the user
+                                       // requested so
+                                       return array(
+                                               'result' => 'Queued',
+                                               'statuskey' => $error[0][1],
+                                       );
+                               } else {
+                                       $this->getResult()->setIndexedTagName( $error, 'error' );
 
-                               $this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error );
+                                       $this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error );
+                               }
                        }
+                       $result['result'] = 'Success';
                }
 
-               $file = $this->mUpload->getLocalFile();
-
-               $result['result'] = 'Success';
                $result['filename'] = $file->getName();
                if ( $warnings && count( $warnings ) > 0 ) {
                        $result['warnings'] = $warnings;
@@ -759,6 +814,7 @@ class ApiUpload extends ApiBase {
                                array( 'code' => 'filename-tooshort', 'info' => 'The filename is too short' ),
                                array( 'code' => 'overwrite', 'info' => 'Overwriting an existing file is not allowed' ),
                                array( 'code' => 'stashfailed', 'info' => 'Stashing temporary file failed' ),
+                               array( 'code' => 'publishfailed', 'info' => 'Publishing of stashed file failed' ),
                                array( 'code' => 'internal-error', 'info' => 'An internal error occurred' ),
                                array( 'code' => 'asynccopyuploaddisabled', 'info' => 'Asynchronous copy uploads disabled' ),
                                array( 'fileexists-forbidden' ),
diff --git a/includes/upload/PublishStashedFile.php b/includes/upload/PublishStashedFile.php
new file mode 100644 (file)
index 0000000..fec3c73
--- /dev/null
@@ -0,0 +1,125 @@
+<?php
+/**
+ * Upload a file from the upload stash into the local file repo.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+require_once( __DIR__ . '/../../maintenance/Maintenance.php' );
+set_time_limit( 3600 ); // 1 hour
+
+/**
+ * Upload a file from the upload stash into the local file repo.
+ *
+ * @ingroup Maintenance
+ */
+class PublishStashedFile extends Maintenance {
+       public function __construct() {
+               parent::__construct();
+               $this->mDescription = "Upload stashed file into the local file repo";
+               $this->addOption( 'filename', "Desired file name", true, true );
+               $this->addOption( 'filekey', "Upload stash file key", true, true );
+               $this->addOption( 'userid', "Upload owner user ID", true, true );
+               $this->addOption( 'comment', "Upload comment", true, true );
+               $this->addOption( 'text', "Upload description", true, true );
+               $this->addOption( 'watch', "Whether the uploader should watch the page", true, true );
+               $this->addOption( 'sessionid', "Upload owner session ID", true, true );
+       }
+
+       public function execute() {
+               wfSetupSession( $this->getOption( 'sessionid' ) );
+               try {
+                       $user = User::newFromId( $this->getOption( 'userid' ) );
+                       if ( !$user ) {
+                               throw new MWException( "No user with ID " . $this->getOption( 'userid' ) . "." );
+                       }
+
+                       UploadBase::setSessionStatus(
+                               $this->getOption( 'filekey' ),
+                               array( 'result' => 'Poll', 'stage' => 'publish', 'status' => Status::newGood() )
+                       );
+
+                       $upload = new UploadFromStash( $user );
+                       // @TODO: initialize() causes a GET, ideally we could frontload the antivirus
+                       // checks and anything else to the stash stage (which includes concatenation and
+                       // the local file is thus already there). That way, instead of GET+PUT, there could
+                       // just be a COPY operation from the stash to the public zone.
+                       $upload->initialize( $this->getOption( 'filekey' ), $this->getOption( 'filename' ) );
+
+                       // Check if the local file checks out (this is generally a no-op)
+                       $verification = $upload->verifyUpload();
+                       if ( $verification['status'] !== UploadBase::OK ) {
+                               $status = Status::newFatal( 'verification-error' );
+                               $status->value = array( 'verification' => $verification );
+                               UploadBase::setSessionStatus(
+                                       $this->getOption( 'filekey' ),
+                                       array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status )
+                               );
+                               $this->error( "Could not verify upload.\n", 1 ); // die
+                       }
+
+                       // Upload the stashed file to a permanent location
+                       $status = $upload->performUpload(
+                               $this->getOption( 'comment' ),
+                               $this->getOption( 'text' ),
+                               $this->getOption( 'watch' ),
+                               $user
+                       );
+                       if ( !$status->isGood() ) {
+                               UploadBase::setSessionStatus(
+                                       $this->getOption( 'filekey' ),
+                                       array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status )
+                               );
+                               $this->error( $status->getWikiText() . "\n", 1 ); // die
+                       }
+
+                       // Build the image info array while we have the local reference handy
+                       $apiMain = new ApiMain(); // dummy object (XXX)
+                       $imageInfo = $upload->getImageInfo( $apiMain->getResult() );
+
+                       // Cleanup any temporary local file
+                       $upload->cleanupTempFile();
+
+                       // Cache the info so the user doesn't have to wait forever to get the final info
+                       UploadBase::setSessionStatus(
+                               $this->getOption( 'filekey' ),
+                               array(
+                                       'result'    => 'Success',
+                                       'stage'     => 'publish',
+                                       'filename'  => $upload->getLocalFile()->getName(),
+                                       'imageinfo' => $imageInfo,
+                                       'status'    => Status::newGood()
+                               )
+                       );
+               } catch ( MWException $e ) {
+                       UploadBase::setSessionStatus(
+                               $this->getOption( 'filekey' ),
+                               array(
+                                       'result' => 'Failure',
+                                       'stage'  => 'publish',
+                                       'status' => Status::newFatal( 'api-error-stashfailed' )
+                               )
+                       );
+                       throw $e;
+               }
+               session_write_close();
+       }
+}
+
+$maintClass = "PublishStashedFile";
+require_once( RUN_MAINTENANCE_IF_MAIN );
index dc32a29..48ea584 100644 (file)
@@ -235,6 +235,14 @@ abstract class UploadBase {
                return $this->mFileSize;
        }
 
+       /**
+        * Get the base 36 SHA1 of the file
+        * @return string
+        */
+       protected function getTempFileSha1Base36() {
+               return FSFile::getSha1Base36FromPath( $this->mTempPath );
+       }
+
        /**
         * @param $srcPath String: the source path
         * @return string the real path if it was a virtual URL
@@ -546,7 +554,9 @@ abstract class UploadBase {
        }
 
        /**
-        * Check for non fatal problems with the file
+        * Check for non fatal problems with the file.
+        *
+        * This should not assume that mTempPath is set.
         *
         * @return Array of warnings
         */
@@ -594,7 +604,7 @@ abstract class UploadBase {
                }
 
                // Check dupes against existing files
-               $hash = FSFile::getSha1Base36FromPath( $this->mTempPath );
+               $hash = $this->getTempFileSha1Base36();
                $dupes = RepoGroup::singleton()->findBySha1( $hash );
                $title = $this->getTitle();
                // Remove all matches against self
index c857f25..71ee96b 100644 (file)
@@ -89,7 +89,7 @@ class UploadFromStash extends UploadBase {
         * @param $key string
         * @param $name string
         */
-       public function initialize( $key, $name = 'upload_file' ) {
+       public function initialize( $key, $name = 'upload_file', $initTempFile = true ) {
                /**
                 * Confirming a temporarily stashed upload.
                 * We don't want path names to be forged, so we keep
@@ -98,7 +98,7 @@ class UploadFromStash extends UploadBase {
                 */
                $metadata = $this->stash->getMetadata( $key );
                $this->initializePathInfo( $name,
-                       $this->getRealPath( $metadata['us_path'] ),
+                       $initTempFile ? $this->getRealPath( $metadata['us_path'] ) : false,
                        $metadata['us_size'],
                        false
                );
@@ -129,6 +129,14 @@ class UploadFromStash extends UploadBase {
                return $this->mSourceType;
        }
 
+       /**
+        * Get the base 36 SHA1 of the file
+        * @return string
+        */
+       protected function getTempFileSha1Base36() {
+               return $this->mFileProps['sha1'];
+       }
+
        /**
         * File has been previously verified so no need to do so again.
         *