X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2Ffilerepo%2Fbackend%2FFileBackendStore.php;h=f02724f2faf7114b13eeb03b2059d0a1059cfebb;hb=b47a2148c68b7bc58b12ec77fa3fe159f834e4e7;hp=fab62e5bd2df29c043fb2e954b6bd500145cae92;hpb=769a1cd2616fbcaf93f7c237ce3a3af7fc942858;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/filerepo/backend/FileBackendStore.php b/includes/filerepo/backend/FileBackendStore.php index fab62e5bd2..f02724f2fa 100644 --- a/includes/filerepo/backend/FileBackendStore.php +++ b/includes/filerepo/backend/FileBackendStore.php @@ -1,5 +1,22 @@ name ); if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { - $status = Status::newFatal( 'backend-fail-store', $params['dst'] ); + $status = Status::newFatal( 'backend-fail-maxsize', + $params['dst'], $this->maxFileSizeInternal() ); } else { $status = $this->doStoreInternal( $params ); $this->clearCache( array( $params['dst'] ) ); @@ -138,6 +162,9 @@ abstract class FileBackendStore extends FileBackend { * src : source storage path * dst : destination storage path * overwrite : overwrite any file that exists at the destination + * async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. * * @param $params Array * @return Status @@ -165,6 +192,9 @@ abstract class FileBackendStore extends FileBackend { * $params include: * src : source storage path * ignoreMissingSource : do nothing if the source file does not exist + * async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. * * @param $params Array * @return Status @@ -193,6 +223,9 @@ abstract class FileBackendStore extends FileBackend { * src : source storage path * dst : destination storage path * overwrite : overwrite any file that exists at the destination + * async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. * * @param $params Array * @return Status @@ -214,6 +247,7 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ protected function doMoveInternal( array $params ) { + unset( $params['async'] ); // two steps, won't work here :) // Copy source to dest $status = $this->copyInternal( $params ); if ( $status->isOK() ) { @@ -224,6 +258,17 @@ abstract class FileBackendStore extends FileBackend { return $status; } + /** + * No-op file operation that does nothing. + * Do not call this function from places outside FileBackend and FileOp. + * + * @param $params Array + * @return Status + */ + final public function nullInternal( array $params ) { + return Status::newGood(); + } + /** * @see FileBackend::concatenate() * @return Status @@ -519,6 +564,8 @@ abstract class FileBackendStore extends FileBackend { $this->trimCache(); // limit memory $this->cache[$path]['stat'] = $stat; $this->setFileCache( $path, $stat ); // update persistent cache + } else { + wfDebug( __METHOD__ . ": File $path does not exist.\n" ); } wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); @@ -785,22 +832,6 @@ abstract class FileBackendStore extends FileBackend { */ abstract public function getFileListInternal( $container, $dir, array $params ); - /** - * Get the list of supported operations and their corresponding FileOp classes. - * - * @return Array - */ - protected function supportedOperations() { - return array( - 'store' => 'StoreFileOp', - 'copy' => 'CopyFileOp', - 'move' => 'MoveFileOp', - 'delete' => 'DeleteFileOp', - 'create' => 'CreateFileOp', - 'null' => 'NullFileOp' - ); - } - /** * Return a list of FileOp objects from a list of operations. * Do not call this function from places outside FileBackend. @@ -813,7 +844,14 @@ abstract class FileBackendStore extends FileBackend { * @throws MWException */ final public function getOperationsInternal( array $ops ) { - $supportedOps = $this->supportedOperations(); + $supportedOps = array( + 'store' => 'StoreFileOp', + 'copy' => 'CopyFileOp', + 'move' => 'MoveFileOp', + 'delete' => 'DeleteFileOp', + 'create' => 'CreateFileOp', + 'null' => 'NullFileOp' + ); $performOps = array(); // array of FileOp objects // Build up ordered array of FileOps... @@ -856,6 +894,18 @@ abstract class FileBackendStore extends FileBackend { return $paths; } + /** + * @see FileBackend::getScopedLocksForOps() + * @return Array + */ + public function getScopedLocksForOps( array $ops, Status $status ) { + $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); + return array( + $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ), + $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status ) + ); + } + /** * @see FileBackend::doOperationsInternal() * @return Status @@ -890,7 +940,7 @@ abstract class FileBackendStore extends FileBackend { $this->primeContainerCache( $performOps ); // Actually attempt the operation batch... - $subStatus = FileOp::attemptBatch( $performOps, $opts, $this->fileJournal ); + $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal ); // Merge errors into status fields $status->merge( $subStatus ); @@ -901,6 +951,105 @@ abstract class FileBackendStore extends FileBackend { return $status; } + /** + * @see FileBackend::doQuickOperationsInternal() + * @return Status + * @throws MWException + */ + final protected function doQuickOperationsInternal( array $ops ) { + wfProfileIn( __METHOD__ ); + wfProfileIn( __METHOD__ . '-' . $this->name ); + $status = Status::newGood(); + + $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' ); + $async = ( $this->parallelize === 'implicit' ); + $maxConcurrency = $this->concurrency; // throttle + + $statuses = array(); // array of (index => Status) + $fileOpHandles = array(); // list of (index => handle) arrays + $curFileOpHandles = array(); // current handle batch + // Perform the sync-only ops and build up op handles for the async ops... + foreach ( $ops as $index => $params ) { + if ( !in_array( $params['op'], $supportedOps ) ) { + wfProfileOut( __METHOD__ . '-' . $this->name ); + wfProfileOut( __METHOD__ ); + throw new MWException( "Operation '{$params['op']}' is not supported." ); + } + $method = $params['op'] . 'Internal'; // e.g. "storeInternal" + $subStatus = $this->$method( array( 'async' => $async ) + $params ); + if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async + if ( count( $curFileOpHandles ) >= $maxConcurrency ) { + $fileOpHandles[] = $curFileOpHandles; // push this batch + $curFileOpHandles = array(); + } + $curFileOpHandles[$index] = $subStatus->value; // keep index + } else { // error or completed + $statuses[$index] = $subStatus; // keep index + } + } + if ( count( $curFileOpHandles ) ) { + $fileOpHandles[] = $curFileOpHandles; // last batch + } + // Do all the async ops that can be done concurrently... + foreach ( $fileOpHandles as $fileHandleBatch ) { + $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch ); + } + // Marshall and merge all the responses... + foreach ( $statuses as $index => $subStatus ) { + $status->merge( $subStatus ); + if ( $subStatus->isOK() ) { + $status->success[$index] = true; + ++$status->successCount; + } else { + $status->success[$index] = false; + ++$status->failCount; + } + } + + wfProfileOut( __METHOD__ . '-' . $this->name ); + wfProfileOut( __METHOD__ ); + return $status; + } + + /** + * Execute a list of FileBackendStoreOpHandle handles in parallel. + * The resulting Status object fields will correspond + * to the order in which the handles where given. + * + * @param $handles Array List of FileBackendStoreOpHandle objects + * @return Array Map of Status objects + * @throws MWException + */ + final public function executeOpHandlesInternal( array $fileOpHandles ) { + wfProfileIn( __METHOD__ ); + wfProfileIn( __METHOD__ . '-' . $this->name ); + foreach ( $fileOpHandles as $fileOpHandle ) { + if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { + throw new MWException( "Given a non-FileBackendStoreOpHandle object." ); + } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) { + throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." ); + } + } + $res = $this->doExecuteOpHandlesInternal( $fileOpHandles ); + foreach ( $fileOpHandles as $fileOpHandle ) { + $fileOpHandle->closeResources(); + } + wfProfileOut( __METHOD__ . '-' . $this->name ); + wfProfileOut( __METHOD__ ); + return $res; + } + + /** + * @see FileBackendStore::executeOpHandlesInternal() + * @return Array List of corresponding Status objects + */ + protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { + foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty + throw new MWException( "This backend supports no asynchronous operations." ); + } + return array(); + } + /** * @see FileBackend::clearCache() */ @@ -1068,7 +1217,7 @@ abstract class FileBackendStore extends FileBackend { * Any empty suffix means the container is not sharded. * * @param $container string Container name - * @param $relStoragePath string Storage path relative to the container + * @param $relPath string Storage path relative to the container * @return string|null Returns null if shard could not be determined */ final protected function getContainerShard( $container, $relPath ) { @@ -1195,7 +1344,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get the cache key for a container * - * @param $container Resolved container name + * @param $container string Resolved container name * @return string */ private function containerCacheKey( $container ) { @@ -1205,9 +1354,8 @@ abstract class FileBackendStore extends FileBackend { /** * Set the cached info for a container * - * @param $container Resolved container name + * @param $container string Resolved container name * @param $val mixed Information to cache - * @return void */ final protected function setContainerCache( $container, $val ) { $this->memCache->set( $this->containerCacheKey( $container ), $val, 14*86400 ); @@ -1216,16 +1364,12 @@ abstract class FileBackendStore extends FileBackend { /** * Delete the cached info for a container * - * @param $container Resolved container name - * @return void + * @param $container string Resolved container name */ final protected function deleteContainerCache( $container ) { - for ( $attempts=1; $attempts <= 3; $attempts++ ) { - if ( $this->memCache->delete( $this->containerCacheKey( $container ) ) ) { - return; // done! - } + if ( !$this->memCache->delete( $this->containerCacheKey( $container ) ) ) { + trigger_error( "Unable to delete stat cache for container $container." ); } - trigger_error( "Unable to delete stat cache for container $container." ); } /** @@ -1238,6 +1382,7 @@ abstract class FileBackendStore extends FileBackend { final protected function primeContainerCache( array $items ) { wfProfileIn( __METHOD__ ); wfProfileIn( __METHOD__ . '-' . $this->name ); + $paths = array(); // list of storage paths $contNames = array(); // (cache key => resolved container name) // Get all the paths/containers from the items... @@ -1261,13 +1406,14 @@ abstract class FileBackendStore extends FileBackend { $contInfo = array(); // (resolved container name => cache value) // Get all cache entries for these container cache keys... - $values = $this->memCache->getBatch( array_keys( $contNames ) ); + $values = $this->memCache->getMulti( array_keys( $contNames ) ); foreach ( $values as $cacheKey => $val ) { $contInfo[$contNames[$cacheKey]] = $val; } // Populate the container process cache for the backend... $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); + wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); } @@ -1285,7 +1431,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get the cache key for a file path * - * @param $path Storage path + * @param $path string Storage path * @return string */ private function fileCacheKey( $path ) { @@ -1295,9 +1441,8 @@ abstract class FileBackendStore extends FileBackend { /** * Set the cached stat info for a file path * - * @param $path Storage path + * @param $path string Storage path * @param $val mixed Information to cache - * @return void */ final protected function setFileCache( $path, $val ) { $this->memCache->set( $this->fileCacheKey( $path ), $val, 7*86400 ); @@ -1306,16 +1451,12 @@ abstract class FileBackendStore extends FileBackend { /** * Delete the cached stat info for a file path * - * @param $path Storage path - * @return void + * @param $path string Storage path */ final protected function deleteFileCache( $path ) { - for ( $attempts=1; $attempts <= 3; $attempts++ ) { - if ( $this->memCache->delete( $this->fileCacheKey( $path ) ) ) { - return; // done! - } + if ( !$this->memCache->delete( $this->fileCacheKey( $path ) ) ) { + trigger_error( "Unable to delete stat cache for file $path." ); } - trigger_error( "Unable to delete stat cache for file $path." ); } /** @@ -1328,6 +1469,7 @@ abstract class FileBackendStore extends FileBackend { final protected function primeFileCache( array $items ) { wfProfileIn( __METHOD__ ); wfProfileIn( __METHOD__ . '-' . $this->name ); + $paths = array(); // list of storage paths $pathNames = array(); // (cache key => storage path) // Get all the paths/containers from the items... @@ -1347,18 +1489,47 @@ abstract class FileBackendStore extends FileBackend { } } // Get all cache entries for these container cache keys... - $values = $this->memCache->getBatch( array_keys( $pathNames ) ); + $values = $this->memCache->getMulti( array_keys( $pathNames ) ); foreach ( $values as $cacheKey => $val ) { if ( is_array( $val ) ) { $this->trimCache(); // limit memory $this->cache[$pathNames[$cacheKey]]['stat'] = $val; } } + wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); } } +/** + * FileBackendStore helper class for performing asynchronous file operations. + * + * For example, calling FileBackendStore::createInternal() with the "async" + * param flag may result in a Status that contains this object as a value. + * This class is largely backend-specific and is mostly just "magic" to be + * passed to FileBackendStore::executeOpHandlesInternal(). + */ +abstract class FileBackendStoreOpHandle { + /** @var Array */ + public $params = array(); // params to caller functions + /** @var FileBackendStore */ + public $backend; + /** @var Array */ + public $resourcesToClose = array(); + + public $call; // string; name that identifies the function called + + /** + * Close all open file handles + * + * @return void + */ + public function closeResources() { + array_map( 'fclose', $this->resourcesToClose ); + } +} + /** * FileBackendStore helper function to handle listings that span container shards. * Do not use this class from places outside of FileBackendStore. @@ -1522,6 +1693,12 @@ abstract class FileBackendStoreShardListIterator implements Iterator { * Iterator for listing directories */ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator { + /** + * @param string $container + * @param string $dir + * @param array $params + * @return Array|null|Traversable + */ protected function listFromShard( $container, $dir, array $params ) { return $this->backend->getDirectoryListInternal( $container, $dir, $params ); } @@ -1531,6 +1708,12 @@ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator * Iterator for listing regular files */ class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator { + /** + * @param string $container + * @param string $dir + * @param array $params + * @return Array|null|Traversable + */ protected function listFromShard( $container, $dir, array $params ) { return $this->backend->getFileListInternal( $container, $dir, $params ); }