<?php
/**
- * @file
- * @ingroup upload
+ * Backend for uploading files from chunks.
*
- * @author Michael Dale
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
*
- * first destination checks are made (if ignorewarnings is not checked) errors / warning is returned.
+ * @file
+ * @ingroup Upload
+ */
+
+/**
+ * Implements uploading from chunks
*
- * we return the uploadUrl
- * we then accept chunk uploads from the client.
- * return chunk id on each POSTED chunk
- * once the client posts done=1 concatenated the files together.
- * more info at: http://firefogg.org/dev/chunk_post.html
+ * @ingroup Upload
+ * @author Michael Dale
*/
-class UploadFromChunks extends UploadBase {
+class UploadFromChunks extends UploadFromFile {
+ protected $mOffset, $mChunkIndex, $mFileKey, $mVirtualTempPath;
- var $chunk_mode; // init, chunk, done
- var $mSessionKey = false;
- var $status = array();
+ /**
+ * Setup local pointers to stash, repo and user ( similar to UploadFromStash )
+ *
+ * @param $user User
+ * @param $stash UploadStash
+ * @param $repo FileRepo
+ */
+ public function __construct( $user = false, $stash = false, $repo = false ) {
+ // user object. sometimes this won't exist, as when running from cron.
+ $this->user = $user;
- const INIT = 1;
- const CHUNK = 2;
- const DONE = 3;
- public function initializeFromRequest( &$request ){
- //should merge initializeFromParams (but just needs to be working atm)
- }
- public function initializeFromParams( $param, &$request ) {
- $this->initFromSessionKey( $param['chunksessionkey'], $request );
- // set the chunk mode:
- if( !$this->mSessionKey && !$param['done'] ){
- // session key not set init the chunk upload system:
- $this->chunk_mode = UploadFromChunks::INIT;
- $this->mDesiredDestName = $param['filename'];
- } else if( $this->mSessionKey && !$param['done'] ){
- // this is a chunk piece
- $this->chunk_mode = UploadFromChunks::CHUNK;
- } else if( $this->mSessionKey && $param['done'] ){
- // this is the last chunk
- $this->chunk_mode = UploadFromChunks::DONE;
+ if( $repo ) {
+ $this->repo = $repo;
+ } else {
+ $this->repo = RepoGroup::singleton()->getLocalRepo();
}
- if( $this->chunk_mode == UploadFromChunks::CHUNK ||
- $this->chunk_mode == UploadFromChunks::DONE ){
- // set chunk related vars:
- $this->mTempPath = $request->getFileTempName( 'chunk' );
- $this->mFileSize = $request->getFileSize( 'chunk' );
+
+ if( $stash ) {
+ $this->stash = $stash;
+ } else {
+ if( $user ) {
+ wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() . "\n" );
+ } else {
+ wfDebug( __METHOD__ . " creating new UploadFromChunks instance with no user\n" );
+ }
+ $this->stash = new UploadStash( $this->repo, $this->user );
}
- return $this->status;
+ return true;
}
- static function isValidRequest( $request ) {
- $sessionData = $request->getSessionData( 'wsUploadData' );
- if( !self::isValidSessionKey(
- $request->getInt( 'wpSessionKey' ),
- $sessionData ) )
- return false;
- // check for the file:
- return (bool)$request->getFileTempName( 'file' );
+ /**
+ * Calls the parent stashFile and updates the uploadsession table to handle "chunks"
+ *
+ * @return UploadStashFile stashed file
+ */
+ public function stashFile() {
+ // Stash file is the called on creating a new chunk session:
+ $this->mChunkIndex = 0;
+ $this->mOffset = 0;
+ // Create a local stash target
+ $this->mLocalFile = parent::stashFile();
+ // Update the initial file offset ( based on file size )
+ $this->mOffset = $this->mLocalFile->getSize();
+ $this->mFileKey = $this->mLocalFile->getFileKey();
+
+ // Output a copy of this first to chunk 0 location:
+ $status = $this->outputChunk( $this->mLocalFile->getPath() );
+
+ // Update db table to reflect initial "chunk" state
+ $this->updateChunkStatus();
+ return $this->mLocalFile;
}
- /* check warnings depending on chunk_mode */
- function checkWarnings(){
- $warning = array();
- return $warning;
+ /**
+ * Continue chunk uploading
+ */
+ public function continueChunks( $name, $key, $webRequestUpload ) {
+ $this->mFileKey = $key;
+ $this->mUpload = $webRequestUpload;
+ // Get the chunk status form the db:
+ $this->getChunkStatus();
+
+ $metadata = $this->stash->getMetadata( $key );
+ $this->initializePathInfo( $name,
+ $this->getRealPath( $metadata['us_path'] ),
+ $metadata['us_size'],
+ false
+ );
}
- function isEmptyFile(){
- // does not apply to chunk init
- if( $this->chunk_mode == UploadFromChunks::INIT ){
- return false;
- } else {
- return parent::isEmptyFile();
+ /**
+ * Append the final chunk and ready file for parent::performUpload()
+ * @return FileRepoStatus
+ */
+ public function concatenateChunks() {
+ wfDebug( __METHOD__ . " concatenate {$this->mChunkIndex} chunks:" .
+ $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
+
+ // Concatenate all the chunks to mVirtualTempPath
+ $fileList = Array();
+ // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
+ for( $i = 0; $i <= $this->getChunkIndex(); $i++ ){
+ $fileList[] = $this->getVirtualChunkLocation( $i );
}
+
+ // Get the file extension from the last chunk
+ $ext = FileBackend::extensionFromPath( $this->mVirtualTempPath );
+ // Get a 0-byte temp file to perform the concatenation at
+ $tmpFile = TempFSFile::factory( 'chunkedupload_', $ext );
+ $tmpPath = $tmpFile
+ ? $tmpFile->getPath()
+ : false; // fail in concatenate()
+ // Concatenate the chunks at the temp file
+ $status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE );
+ if( !$status->isOk() ){
+ return $status;
+ }
+ // Update the mTempPath and mLocalFile
+ // ( for FileUpload or normal Stash to take over )
+ $this->mTempPath = $tmpPath; // file system path
+ $this->mLocalFile = parent::stashFile();
+
+ return $status;
}
- /**
- * Verify whether the upload is sane.
- * Returns self::OK or else an array with error information
+ /**
+ * Perform the upload, then remove the temp copy afterward
+ * @param $comment string
+ * @param $pageText string
+ * @param $watch bool
+ * @param $user User
+ * @return Status
*/
- function verifyUpload() {
- // no checks on chunk upload mode:
- if( $this->chunk_mode == UploadFromChunks::INIT )
- return array( 'status' => self::OK );
-
- // verify on init and last chunk request
- if( $this->chunk_mode == UploadFromChunks::CHUNK ||
- $this->chunk_mode == UploadFromChunks::DONE )
- return parent::verifyUpload();
+ public function performUpload( $comment, $pageText, $watch, $user ) {
+ $rv = parent::performUpload( $comment, $pageText, $watch, $user );
+ return $rv;
}
- // only run verifyFile on completed uploaded chunks
- function verifyFile(){
- if( $this->chunk_mode == UploadFromChunks::DONE ){
- // first append last chunk (so we can do a real verifyFile check... (check file type etc)
- $status = $this->doChunkAppend();
- if( $status->isOK() ){
- $this->mTempPath = $this->getRealPath( $this->mTempAppendPath );
- // verify the completed merged chunks as if it was the file that got uploaded:
- return parent::verifyFile( $this->mTempPath );
+ /**
+ * Returns the virtual chunk location:
+ * @param $index
+ * @return string
+ */
+ function getVirtualChunkLocation( $index ){
+ return $this->repo->getVirtualUrl( 'temp' ) .
+ '/' .
+ $this->repo->getHashPath(
+ $this->getChunkFileKey( $index )
+ ) .
+ $this->getChunkFileKey( $index );
+ }
+
+ /**
+ * Add a chunk to the temporary directory
+ *
+ * @param $chunkPath string path to temporary chunk file
+ * @param $chunkSize int size of the current chunk
+ * @param $offset int offset of current chunk ( mutch match database chunk offset )
+ * @return Status
+ */
+ public function addChunk( $chunkPath, $chunkSize, $offset ) {
+ // Get the offset before we add the chunk to the file system
+ $preAppendOffset = $this->getOffset();
+
+ if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize()) {
+ $status = Status::newFatal( 'file-too-large' );
+ } else {
+ // Make sure the client is uploading the correct chunk with a matching offset.
+ if ( $preAppendOffset == $offset ) {
+ // Update local chunk index for the current chunk
+ $this->mChunkIndex++;
+ $status = $this->outputChunk( $chunkPath );
+ if( $status->isGood() ){
+ // Update local offset:
+ $this->mOffset = $preAppendOffset + $chunkSize;
+ // Update chunk table status db
+ $this->updateChunkStatus();
+ }
} else {
- // conflict of status returns (have to return the error ary) ... why we don't consistantly use a status object is beyond me..
- return $status->getErrorsArray();
+ $status = Status::newFatal( 'invalid-chunk-offset' );
}
- } else {
- return true;
}
+ return $status;
}
- // pretty ugly inter-mixing of mParam and local vars
- function setupChunkSession( $summary, $comment, $watch ) {
- $this->mSessionKey = $this->getSessionKey();
- $_SESSION['wsUploadData'][$this->mSessionKey] = array(
- 'mComment' => $comment,
- 'mSummary' => $summary,
- 'mWatch' => $watch,
- 'mIgnorewarnings' => true, //ignore warning on chunk uploads (for now)
- 'mFilteredName' => $this->mFilteredName,
- 'mTempAppendPath' => null, // the repo append path (not temporary local node mTempPath)
- 'mDesiredDestName' => $this->mDesiredDestName,
- 'version' => self::SESSION_VERSION,
+ /**
+ * Update the chunk db table with the current status:
+ */
+ private function updateChunkStatus(){
+ wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" .
+ $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
+
+ $dbw = $this->repo->getMasterDb();
+ $dbw->update(
+ 'uploadstash',
+ array(
+ 'us_status' => 'chunks',
+ 'us_chunk_inx' => $this->getChunkIndex(),
+ 'us_size' => $this->getOffset()
+ ),
+ array( 'us_key' => $this->mFileKey ),
+ __METHOD__
);
- return $this->mSessionKey;
}
- function initFromSessionKey( $sessionKey, $request ){
- if( !$sessionKey || empty( $sessionKey ) ){
- return false;
- }
- $this->mSessionKey = $sessionKey;
- // load the sessionData array:
- $sessionData = $request->getSessionData( 'wsUploadData' );
-
- if( isset( $sessionData[$this->mSessionKey]['version'] ) &&
- $sessionData[$this->mSessionKey]['version'] == self::SESSION_VERSION ) {
- // update the local object from the session
- $this->mComment = $sessionData[$this->mSessionKey]['mComment'];
- $this->mSummary = $sessionData[$this->mSessionKey]['mSummary'];
- $this->mWatch = $sessionData[$this->mSessionKey]['mWatch'];
- $this->mIgnorewarnings = $sessionData[$this->mSessionKey]['mIgnorewarnings'];
- $this->mFilteredName = $sessionData[$this->mSessionKey]['mFilteredName'];
- $this->mTempAppendPath = $sessionData[$this->mSessionKey]['mTempAppendPath'];
- $this->mDesiredDestName = $sessionData[$this->mSessionKey]['mDesiredDestName'];
- } else {
- $this->status = array( 'error' => 'missing session data' );
- return false;
+ /**
+ * Get the chunk db state and populate update relevant local values
+ */
+ private function getChunkStatus(){
+ // get Master db to avoid race conditions.
+ // Otherwise, if chunk upload time < replag there will be spurious errors
+ $dbw = $this->repo->getMasterDb();
+ $row = $dbw->selectRow(
+ 'uploadstash',
+ array(
+ 'us_chunk_inx',
+ 'us_size',
+ 'us_path',
+ ),
+ array( 'us_key' => $this->mFileKey ),
+ __METHOD__
+ );
+ // Handle result:
+ if ( $row ) {
+ $this->mChunkIndex = $row->us_chunk_inx;
+ $this->mOffset = $row->us_size;
+ $this->mVirtualTempPath = $row->us_path;
}
}
- // Lets us return an api result (as flow for chunk uploads is kind of different than others.
- function performUpload( $summary = '', $comment = '', $watch = '', $user ){
- global $wgUser;
-
- if( $this->chunk_mode == UploadFromChunks::INIT ){
- // firefogg expects a specific result per:
- // http://www.firefogg.org/dev/chunk_post.html
-
- // it's okay to return the token here because
- // a) the user must have requested the token to get here and
- // b) should only happen over POST
- // c) (we need the token to validate chunks are coming from a non-xss request)
- $token = urlencode( $wgUser->editToken() );
- ob_clean();
- echo FormatJson::encode( array(
- 'uploadUrl' => wfExpandUrl( wfScript( 'api' ) ) . "?action=upload&".
- "token={$token}&format=json&enablechunks=true&chunksessionkey=".
- $this->setupChunkSession( $summary, $comment, $watch ) ) );
- exit( 0 );
- } else if( $this->chunk_mode == UploadFromChunks::CHUNK ){
- $status = $this->doChunkAppend();
- if( $status->isOK() ){
- // return success:
- // firefogg expects a specific result per:
- // http://www.firefogg.org/dev/chunk_post.html
- ob_clean();
- echo FormatJson::encode( array(
- 'result' => 1,
- 'filesize' => filesize( $this->getRealPath( $this->mTempAppendPath ) )
- )
- );
- exit( 0 );
- /*return array(
- 'result' => 1
- );*/
- } else {
- return $status;
- }
- } else if( $this->chunk_mode == UploadFromChunks::DONE ){
- // update the values from the local (session init) if not paseed again)
- if( $summary == '' )
- $summary = $this->mSummary;
-
- if( $comment == '' )
- $comment = $this->mComment;
-
- if( $watch == '' )
- $watch = $this->mWatch;
- $status = parent::performUpload( $summary, $comment, $watch, $user );
- if( !$status->isGood() ) {
- return $status;
- }
- $file = $this->getLocalFile();
- // firefogg expects a specific result per:
- // http://www.firefogg.org/dev/chunk_post.html
- ob_clean();
- echo FormatJson::encode( array(
- 'result' => 1,
- 'done' => 1,
- 'resultUrl' => $file->getDescriptionUrl()
- )
- );
- exit( 0 );
+ /**
+ * Get the current Chunk index
+ * @return Integer index of the current chunk
+ */
+ private function getChunkIndex(){
+ if( $this->mChunkIndex !== null ){
+ return $this->mChunkIndex;
+ }
+ return 0;
+ }
+ /**
+ * Gets the current offset in fromt the stashedupload table
+ * @return Integer current byte offset of the chunk file set
+ */
+ private function getOffset(){
+ if ( $this->mOffset !== null ){
+ return $this->mOffset;
}
+ return 0;
}
- // append the given chunk to the temporary uploaded file. (if no temporary uploaded file exists created it.
- function doChunkAppend(){
- global $wgMaxUploadSize;
- // if we don't have a mTempAppendPath to generate a file from the chunk packaged var:
- if( !$this->mTempAppendPath ){
- // get temp name:
- // make a chunk store path. (append tmp file to chunk)
- $status = $this->saveTempUploadedFile( $this->mDestName, $this->mTempPath );
-
- if( $status->isOK() ) {
- $this->mTempAppendPath = $status->value;
- $_SESSION['wsUploadData'][$this->mSessionKey]['mTempAppendPath'] = $this->mTempAppendPath;
- }
- return $status;
- } else {
- if( is_file( $this->getRealPath( $this->mTempAppendPath ) ) ){
- $status = $this->appendToUploadFile( $this->mTempAppendPath, $this->mTempPath );
- } else {
- $status = Status::newFatal( 'filenotfound', $this->mTempAppendPath );
+ /**
+ * Output the chunk to disk
+ *
+ * @param $chunkPath string
+ * @throws UploadChunkFileException
+ * @return FileRepoStatus
+ */
+ private function outputChunk( $chunkPath ){
+ // Key is fileKey + chunk index
+ $fileKey = $this->getChunkFileKey();
+
+ // Store the chunk per its indexed fileKey:
+ $hashPath = $this->repo->getHashPath( $fileKey );
+ $storeStatus = $this->repo->store( $chunkPath, 'temp', "$hashPath$fileKey" );
+
+ // Check for error in stashing the chunk:
+ if ( ! $storeStatus->isOK() ) {
+ $error = $storeStatus->getErrorsArray();
+ $error = reset( $error );
+ if ( ! count( $error ) ) {
+ $error = $storeStatus->getWarningsArray();
+ $error = reset( $error );
+ if ( ! count( $error ) ) {
+ $error = array( 'unknown', 'no error recorded' );
+ }
}
- //check to make sure we have not expanded beyond $wgMaxUploadSize
- if( filesize( $this->getRealPath( $this->mTempAppendPath ) ) > $wgMaxUploadSize )
- $status = Status::newFatal( 'largefileserver' );
-
- return $status;
+ throw new UploadChunkFileException( "error storing file in '$chunkPath': " . implode( '; ', $error ) );
}
+ return $storeStatus;
}
+ private function getChunkFileKey( $index = null ){
+ if( $index === null ){
+ $index = $this->getChunkIndex();
+ }
+ return $this->mFileKey . '.' . $index ;
+ }
}
+
+class UploadChunkZeroLengthFileException extends MWException {};
+class UploadChunkFileException extends MWException {};
dépôts / lhc / web / wiklou.git / blobdiff