includes/filerepo/backend/FileBackend.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * Base class for all file backend classes (including multi-write backends).
  10  * This class defines the methods as abstract that subclasses must implement.
  11  * Outside callers can assume that all backends will have these functions.
  12  *
  13  * All "storage paths" are of the format "mwstore://backend/container/path".
  14  * The paths use UNIX file system (FS) notation, though any particular backend may
  15  * not actually be using a local filesystem. Therefore, the paths are only virtual.
  16  *
  17  * Backend contents are stored under wiki-specific container names by default.
  18  * For legacy reasons, this has no effect for the FS backend class, and per-wiki
  19  * segregation must be done by setting the container paths appropriately.
  20  *
  21  * FS-based backends are somewhat more restrictive due to the existence of real
  22  * directory files; a regular file cannot have the same name as a directory. Other
  23  * backends with virtual directories may not have this limitation. Callers should
  24  * store files in such a way that no files and directories are under the same path.
  25  *
  26  * Methods should avoid throwing exceptions at all costs.
  27  * As a corollary, external dependencies should be kept to a minimum.
  28  *
  29  * @ingroup FileBackend
  30  * @since 1.19
  31  */
  32 abstract class FileBackend {
  33         protected $name; // string; unique backend name
  34         protected $wikiId; // string; unique wiki name
  35         protected $readOnly; // string; read-only explanation message
  36         /** @var LockManager */
  37         protected $lockManager;
  38
  39         /**
  40          * Create a new backend instance from configuration.
  41          * This should only be called from within FileBackendGroup.
  42          *
  43          * $config includes:
  44          *     'name'        : The unique name of this backend.
  45          *                     This should consist of alphanumberic, '-', and '_' characters.
  46          *     'wikiId'      : Prefix to container names that is unique to this wiki.
  47          *                     This should consist of alphanumberic, '-', and '_' characters.
  48          *     'lockManager' : Registered name of a file lock manager to use.
  49          *     'readOnly'    : Write operations are disallowed if this is a non-empty string.
  50          *                     It should be an explanation for the backend being read-only.
  51          *
  52          * @param $config Array
  53          */
  54         public function __construct( array $config ) {
  55                 $this->name = $config['name'];
  56                 $this->wikiId = isset( $config['wikiId'] )
  57                         ? $config['wikiId']
  58                         : wfWikiID(); // e.g. "my_wiki-en_"
  59                 $this->lockManager = ( $config['lockManager'] instanceof LockManager )
  60                         ? $config['lockManager']
  61                         : LockManagerGroup::singleton()->get( $config['lockManager'] );
  62                 $this->readOnly = isset( $config['readOnly'] )
  63                         ? (string)$config['readOnly']
  64                         : '';
  65         }
  66
  67         /**
  68          * Get the unique backend name.
  69          * We may have multiple different backends of the same type.
  70          * For example, we can have two Swift backends using different proxies.
  71          *
  72          * @return string
  73          */
  74         final public function getName() {
  75                 return $this->name;
  76         }
  77
  78         /**
  79          * Check if this backend is read-only
  80          *
  81          * @return bool
  82          */
  83         final public function isReadOnly() {
  84                 return ( $this->readOnly != '' );
  85         }
  86
  87         /**
  88          * Get an explanatory message if this backend is read-only
  89          *
  90          * @return string|false Returns falls if the backend is not read-only
  91          */
  92         final public function getReadOnlyReason() {
  93                 return ( $this->readOnly != '' ) ? $this->readOnly : false;
  94         }
  95
  96         /**
  97          * This is the main entry point into the backend for write operations.
  98          * Callers supply an ordered list of operations to perform as a transaction.
  99          * Files will be locked, the stat cache cleared, and then the operations attempted.
 100          * If any serious errors occur, all attempted operations will be rolled back.
 101          *
 102          * $ops is an array of arrays. The outer array holds a list of operations.
 103          * Each inner array is a set of key value pairs that specify an operation.
 104          *
 105          * Supported operations and their parameters:
 106          * a) Create a new file in storage with the contents of a string
 107          *     array(
 108          *         'op'                  => 'create',
 109          *         'dst'                 => <storage path>,
 110          *         'content'             => <string of new file contents>,
 111          *         'overwrite'           => <boolean>,
 112          *         'overwriteSame'       => <boolean>
 113          *     )
 114          * b) Copy a file system file into storage
 115          *     array(
 116          *         'op'                  => 'store',
 117          *         'src'                 => <file system path>,
 118          *         'dst'                 => <storage path>,
 119          *         'overwrite'           => <boolean>,
 120          *         'overwriteSame'       => <boolean>
 121          *     )
 122          * c) Copy a file within storage
 123          *     array(
 124          *         'op'                  => 'copy',
 125          *         'src'                 => <storage path>,
 126          *         'dst'                 => <storage path>,
 127          *         'overwrite'           => <boolean>,
 128          *         'overwriteSame'       => <boolean>
 129          *     )
 130          * d) Move a file within storage
 131          *     array(
 132          *         'op'                  => 'move',
 133          *         'src'                 => <storage path>,
 134          *         'dst'                 => <storage path>,
 135          *         'overwrite'           => <boolean>,
 136          *         'overwriteSame'       => <boolean>
 137          *     )
 138          * e) Delete a file within storage
 139          *     array(
 140          *         'op'                  => 'delete',
 141          *         'src'                 => <storage path>,
 142          *         'ignoreMissingSource' => <boolean>
 143          *     )
 144          * f) Do nothing (no-op)
 145          *     array(
 146          *         'op'                  => 'null',
 147          *     )
 148          *
 149          * Boolean flags for operations (operation-specific):
 150          * 'ignoreMissingSource' : The operation will simply succeed and do
 151          *                         nothing if the source file does not exist.
 152          * 'overwrite'           : Any destination file will be overwritten.
 153          * 'overwriteSame'       : An error will not be given if a file already
 154          *                         exists at the destination that has the same
 155          *                         contents as the new contents to be written there.
 156          *
 157          * $opts is an associative of boolean flags, including:
 158          * 'force'               : Errors that would normally cause a rollback do not.
 159          *                         The remaining operations are still attempted if any fail.
 160          * 'nonLocking'          : No locks are acquired for the operations.
 161          *                         This can increase performance for non-critical writes.
 162          *                         This has no effect unless the 'force' flag is set.
 163          * 'allowStale'          : Don't require the latest available data.
 164          *                         This can increase performance for non-critical writes.
 165          *                         This has no effect unless the 'force' flag is set.
 166          *
 167          * Remarks on locking:
 168          * File system paths given to operations should refer to files that are
 169          * already locked or otherwise safe from modification from other processes.
 170          * Normally these files will be new temp files, which should be adequate.
 171          *
 172          * Return value:
 173          * This returns a Status, which contains all warnings and fatals that occured
 174          * during the operation. The 'failCount', 'successCount', and 'success' members
 175          * will reflect each operation attempted. The status will be "OK" unless any
 176          * of the operations failed and the 'force' parameter was not set.
 177          *
 178          * @param $ops Array List of operations to execute in order
 179          * @param $opts Array Batch operation options
 180          * @return Status
 181          */
 182         final public function doOperations( array $ops, array $opts = array() ) {
 183                 if ( $this->isReadOnly() ) {
 184                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 185                 }
 186                 if ( empty( $opts['force'] ) ) { // sanity
 187                         unset( $opts['nonLocking'] );
 188                         unset( $opts['allowStale'] );
 189                 }
 190                 return $this->doOperationsInternal( $ops, $opts );
 191         }
 192
 193         /**
 194          * @see FileBackend::doOperations()
 195          */
 196         abstract protected function doOperationsInternal( array $ops, array $opts );
 197
 198         /**
 199          * Same as doOperations() except it takes a single operation.
 200          * If you are doing a batch of operations that should either
 201          * all succeed or all fail, then use that function instead.
 202          *
 203          * @see FileBackend::doOperations()
 204          *
 205          * @param $op Array Operation
 206          * @param $opts Array Operation options
 207          * @return Status
 208          */
 209         final public function doOperation( array $op, array $opts = array() ) {
 210                 return $this->doOperations( array( $op ), $opts );
 211         }
 212
 213         /**
 214          * Performs a single create operation.
 215          * This sets $params['op'] to 'create' and passes it to doOperation().
 216          *
 217          * @see FileBackend::doOperation()
 218          *
 219          * @param $params Array Operation parameters
 220          * @param $opts Array Operation options
 221          * @return Status
 222          */
 223         final public function create( array $params, array $opts = array() ) {
 224                 $params['op'] = 'create';
 225                 return $this->doOperation( $params, $opts );
 226         }
 227
 228         /**
 229          * Performs a single store operation.
 230          * This sets $params['op'] to 'store' and passes it to doOperation().
 231          *
 232          * @see FileBackend::doOperation()
 233          *
 234          * @param $params Array Operation parameters
 235          * @param $opts Array Operation options
 236          * @return Status
 237          */
 238         final public function store( array $params, array $opts = array() ) {
 239                 $params['op'] = 'store';
 240                 return $this->doOperation( $params, $opts );
 241         }
 242
 243         /**
 244          * Performs a single copy operation.
 245          * This sets $params['op'] to 'copy' and passes it to doOperation().
 246          *
 247          * @see FileBackend::doOperation()
 248          *
 249          * @param $params Array Operation parameters
 250          * @param $opts Array Operation options
 251          * @return Status
 252          */
 253         final public function copy( array $params, array $opts = array() ) {
 254                 $params['op'] = 'copy';
 255                 return $this->doOperation( $params, $opts );
 256         }
 257
 258         /**
 259          * Performs a single move operation.
 260          * This sets $params['op'] to 'move' and passes it to doOperation().
 261          *
 262          * @see FileBackend::doOperation()
 263          *
 264          * @param $params Array Operation parameters
 265          * @param $opts Array Operation options
 266          * @return Status
 267          */
 268         final public function move( array $params, array $opts = array() ) {
 269                 $params['op'] = 'move';
 270                 return $this->doOperation( $params, $opts );
 271         }
 272
 273         /**
 274          * Performs a single delete operation.
 275          * This sets $params['op'] to 'delete' and passes it to doOperation().
 276          *
 277          * @see FileBackend::doOperation()
 278          *
 279          * @param $params Array Operation parameters
 280          * @param $opts Array Operation options
 281          * @return Status
 282          */
 283         final public function delete( array $params, array $opts = array() ) {
 284                 $params['op'] = 'delete';
 285                 return $this->doOperation( $params, $opts );
 286         }
 287
 288         /**
 289          * Concatenate a list of storage files into a single file system file.
 290          * The target path should refer to a file that is already locked or
 291          * otherwise safe from modification from other processes. Normally,
 292          * the file will be a new temp file, which should be adequate.
 293          * $params include:
 294          *     srcs          : ordered source storage paths (e.g. chunk1, chunk2, ...)
 295          *     dst           : file system path to 0-byte temp file
 296          *
 297          * @param $params Array Operation parameters
 298          * @return Status
 299          */
 300         abstract public function concatenate( array $params );
 301
 302         /**
 303          * Prepare a storage directory for usage.
 304          * This will create any required containers and parent directories.
 305          * Backends using key/value stores only need to create the container.
 306          *
 307          * $params include:
 308          *     dir : storage directory
 309          *
 310          * @param $params Array
 311          * @return Status
 312          */
 313         final public function prepare( array $params ) {
 314                 if ( $this->isReadOnly() ) {
 315                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 316                 }
 317                 return $this->doPrepare( $params );
 318         }
 319
 320         /**
 321          * @see FileBackend::prepare()
 322          */
 323         abstract protected function doPrepare( array $params );
 324
 325         /**
 326          * Take measures to block web access to a storage directory and
 327          * the container it belongs to. FS backends might add .htaccess
 328          * files whereas key/value store backends might restrict container
 329          * access to the auth user that represents end-users in web request.
 330          * This is not guaranteed to actually do anything.
 331          *
 332          * $params include:
 333          *     dir       : storage directory
 334          *     noAccess  : try to deny file access
 335          *     noListing : try to deny file listing
 336          *
 337          * @param $params Array
 338          * @return Status
 339          */
 340         final public function secure( array $params ) {
 341                 if ( $this->isReadOnly() ) {
 342                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 343                 }
 344                 $status = $this->doPrepare( $params ); // dir must exist to restrict it
 345                 if ( $status->isOK() ) {
 346                         $status->merge( $this->doSecure( $params ) );
 347                 }
 348                 return $status;
 349         }
 350
 351         /**
 352          * @see FileBackend::secure()
 353          */
 354         abstract protected function doSecure( array $params );
 355
 356         /**
 357          * Delete a storage directory if it is empty.
 358          * Backends using key/value stores may do nothing unless the directory
 359          * is that of an empty container, in which case it should be deleted.
 360          *
 361          * $params include:
 362          *     dir : storage directory
 363          *
 364          * @param $params Array
 365          * @return Status
 366          */
 367         final public function clean( array $params ) {
 368                 if ( $this->isReadOnly() ) {
 369                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 370                 }
 371                 return $this->doClean( $params );
 372         }
 373
 374         /**
 375          * @see FileBackend::clean()
 376          */
 377         abstract protected function doClean( array $params );
 378
 379         /**
 380          * Check if a file exists at a storage path in the backend.
 381          * This returns false if only a directory exists at the path.
 382          *
 383          * $params include:
 384          *     src    : source storage path
 385          *     latest : use the latest available data
 386          *
 387          * @param $params Array
 388          * @return bool|null Returns null on failure
 389          */
 390         abstract public function fileExists( array $params );
 391
 392         /**
 393          * Get the last-modified timestamp of the file at a storage path.
 394          *
 395          * $params include:
 396          *     src    : source storage path
 397          *     latest : use the latest available data
 398          *
 399          * @param $params Array
 400          * @return string|false TS_MW timestamp or false on failure
 401          */
 402         abstract public function getFileTimestamp( array $params );
 403
 404         /**
 405          * Get the contents of a file at a storage path in the backend.
 406          * This should be avoided for potentially large files.
 407          *
 408          * $params include:
 409          *     src    : source storage path
 410          *     latest : use the latest available data
 411          *
 412          * @param $params Array
 413          * @return string|false Returns false on failure
 414          */
 415         abstract public function getFileContents( array $params );
 416
 417         /**
 418          * Get the size (bytes) of a file at a storage path in the backend.
 419          *
 420          * $params include:
 421          *     src    : source storage path
 422          *     latest : use the latest available data
 423          *
 424          * @param $params Array
 425          * @return integer|false Returns false on failure
 426          */
 427         abstract public function getFileSize( array $params );
 428
 429         /**
 430          * Get quick information about a file at a storage path in the backend.
 431          * If the file does not exist, then this returns false.
 432          * Otherwise, the result is an associative array that includes:
 433          *     mtime  : the last-modified timestamp (TS_MW)
 434          *     size   : the file size (bytes)
 435          * Additional values may be included for internal use only.
 436          *
 437          * $params include:
 438          *     src    : source storage path
 439          *     latest : use the latest available data
 440          *
 441          * @param $params Array
 442          * @return Array|false|null Returns null on failure
 443          */
 444         abstract public function getFileStat( array $params );
 445
 446         /**
 447          * Get a SHA-1 hash of the file at a storage path in the backend.
 448          *
 449          * $params include:
 450          *     src    : source storage path
 451          *     latest : use the latest available data
 452          *
 453          * @param $params Array
 454          * @return string|false Hash string or false on failure
 455          */
 456         abstract public function getFileSha1Base36( array $params );
 457
 458         /**
 459          * Get the properties of the file at a storage path in the backend.
 460          * Returns FSFile::placeholderProps() on failure.
 461          *
 462          * $params include:
 463          *     src    : source storage path
 464          *     latest : use the latest available data
 465          *
 466          * @param $params Array
 467          * @return Array
 468          */
 469         abstract public function getFileProps( array $params );
 470
 471         /**
 472          * Stream the file at a storage path in the backend.
 473          * If the file does not exists, a 404 error will be given.
 474          * Appropriate HTTP headers (Status, Content-Type, Content-Length)
 475          * must be sent if streaming began, while none should be sent otherwise.
 476          * Implementations should flush the output buffer before sending data.
 477          *
 478          * $params include:
 479          *     src     : source storage path
 480          *     headers : additional HTTP headers to send on success
 481          *     latest  : use the latest available data
 482          *
 483          * @param $params Array
 484          * @return Status
 485          */
 486         abstract public function streamFile( array $params );
 487
 488         /**
 489          * Returns a file system file, identical to the file at a storage path.
 490          * The file returned is either:
 491          * a) A local copy of the file at a storage path in the backend.
 492          *    The temporary copy will have the same extension as the source.
 493          * b) An original of the file at a storage path in the backend.
 494          * Temporary files may be purged when the file object falls out of scope.
 495          *
 496          * Write operations should *never* be done on this file as some backends
 497          * may do internal tracking or may be instances of FileBackendMultiWrite.
 498          * In that later case, there are copies of the file that must stay in sync.
 499          * Additionally, further calls to this function may return the same file.
 500          *
 501          * $params include:
 502          *     src    : source storage path
 503          *     latest : use the latest available data
 504          *
 505          * @param $params Array
 506          * @return FSFile|null Returns null on failure
 507          */
 508         abstract public function getLocalReference( array $params );
 509
 510         /**
 511          * Get a local copy on disk of the file at a storage path in the backend.
 512          * The temporary copy will have the same file extension as the source.
 513          * Temporary files may be purged when the file object falls out of scope.
 514          *
 515          * $params include:
 516          *     src    : source storage path
 517          *     latest : use the latest available data
 518          *
 519          * @param $params Array
 520          * @return TempFSFile|null Returns null on failure
 521          */
 522         abstract public function getLocalCopy( array $params );
 523
 524         /**
 525          * Get an iterator to list out all stored files under a storage directory.
 526          * If the directory is of the form "mwstore://container", then all items in
 527          * the container should be listed. If of the form "mwstore://container/dir",
 528          * then all items under that container directory should be listed.
 529          * Results should be storage paths relative to the given directory.
 530          *
 531          * $params include:
 532          *     dir : storage path directory
 533          *
 534          * @return Traversable|Array|null Returns null on failure
 535          */
 536         abstract public function getFileList( array $params );
 537
 538         /**
 539          * Invalidate any in-process file existence and property cache.
 540          * If $paths is given, then only the cache for those files will be cleared.
 541          *
 542          * @param $paths Array Storage paths (optional)
 543          * @return void
 544          */
 545         public function clearCache( array $paths = null ) {}
 546
 547         /**
 548          * Lock the files at the given storage paths in the backend.
 549          * This will either lock all the files or none (on failure).
 550          *
 551          * Callers should consider using getScopedFileLocks() instead.
 552          *
 553          * @param $paths Array Storage paths
 554          * @param $type integer LockManager::LOCK_* constant
 555          * @return Status
 556          */
 557         final public function lockFiles( array $paths, $type ) {
 558                 return $this->lockManager->lock( $paths, $type );
 559         }
 560
 561         /**
 562          * Unlock the files at the given storage paths in the backend.
 563          *
 564          * @param $paths Array Storage paths
 565          * @param $type integer LockManager::LOCK_* constant
 566          * @return Status
 567          */
 568         final public function unlockFiles( array $paths, $type ) {
 569                 return $this->lockManager->unlock( $paths, $type );
 570         }
 571
 572         /**
 573          * Lock the files at the given storage paths in the backend.
 574          * This will either lock all the files or none (on failure).
 575          * On failure, the status object will be updated with errors.
 576          *
 577          * Once the return value goes out scope, the locks will be released and
 578          * the status updated. Unlock fatals will not change the status "OK" value.
 579          *
 580          * @param $paths Array Storage paths
 581          * @param $type integer LockManager::LOCK_* constant
 582          * @param $status Status Status to update on lock/unlock
 583          * @return ScopedLock|null Returns null on failure
 584          */
 585         final public function getScopedFileLocks( array $paths, $type, Status $status ) {
 586                 return ScopedLock::factory( $this->lockManager, $paths, $type, $status );
 587         }
 588
 589         /**
 590          * Check if a given path is a "mwstore://" path.
 591          * This does not do any further validation or any existence checks.
 592          *
 593          * @param $path string
 594          * @return bool
 595          */
 596         final public static function isStoragePath( $path ) {
 597                 return ( strpos( $path, 'mwstore://' ) === 0 );
 598         }
 599
 600         /**
 601          * Split a storage path into a backend name, a container name,
 602          * and a relative file path. The relative path may be the empty string.
 603          * This does not do any path normalization or traversal checks.
 604          *
 605          * @param $storagePath string
 606          * @return Array (backend, container, rel object) or (null, null, null)
 607          */
 608         final public static function splitStoragePath( $storagePath ) {
 609                 if ( self::isStoragePath( $storagePath ) ) {
 610                         // Remove the "mwstore://" prefix and split the path
 611                         $parts = explode( '/', substr( $storagePath, 10 ), 3 );
 612                         if ( count( $parts ) >= 2 && $parts[0] != '' && $parts[1] != '' ) {
 613                                 if ( count( $parts ) == 3 ) {
 614                                         return $parts; // e.g. "backend/container/path"
 615                                 } else {
 616                                         return array( $parts[0], $parts[1], '' ); // e.g. "backend/container"
 617                                 }
 618                         }
 619                 }
 620                 return array( null, null, null );
 621         }
 622
 623         /**
 624          * Normalize a storage path by cleaning up directory separators.
 625          * Returns null if the path is not of the format of a valid storage path.
 626          *
 627          * @param $storagePath string
 628          * @return string|null
 629          */
 630         final public static function normalizeStoragePath( $storagePath ) {
 631                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
 632                 if ( $relPath !== null ) { // must be for this backend
 633                         $relPath = self::normalizeContainerPath( $relPath );
 634                         if ( $relPath !== null ) {
 635                                 return ( $relPath != '' )
 636                                         ? "mwstore://{$backend}/{$container}/{$relPath}"
 637                                         : "mwstore://{$backend}/{$container}";
 638                         }
 639                 }
 640                 return null;
 641         }
 642
 643         /**
 644          * Validate and normalize a relative storage path.
 645          * Null is returned if the path involves directory traversal.
 646          * Traversal is insecure for FS backends and broken for others.
 647          *
 648          * @param $path string Storage path relative to a container
 649          * @return string|null
 650          */
 651         final protected static function normalizeContainerPath( $path ) {
 652                 // Normalize directory separators
 653                 $path = strtr( $path, '\\', '/' );
 654                 // Collapse any consecutive directory separators
 655                 $path = preg_replace( '![/]{2,}!', '/', $path );
 656                 // Remove any leading directory separator
 657                 $path = ltrim( $path, '/' );
 658                 // Use the same traversal protection as Title::secureAndSplit()
 659                 if ( strpos( $path, '.' ) !== false ) {
 660                         if (
 661                                 $path === '.' ||
 662                                 $path === '..' ||
 663                                 strpos( $path, './' ) === 0 ||
 664                                 strpos( $path, '../' ) === 0 ||
 665                                 strpos( $path, '/./' ) !== false ||
 666                                 strpos( $path, '/../' ) !== false
 667                         ) {
 668                                 return null;
 669                         }
 670                 }
 671                 return $path;
 672         }
 673
 674         /**
 675          * Get the parent storage directory of a storage path.
 676          * This returns a path like "mwstore://backend/container",
 677          * "mwstore://backend/container/...", or null if there is no parent.
 678          *
 679          * @param $storagePath string
 680          * @return string|null
 681          */
 682         final public static function parentStoragePath( $storagePath ) {
 683                 $storagePath = dirname( $storagePath );
 684                 list( $b, $cont, $rel ) = self::splitStoragePath( $storagePath );
 685                 return ( $rel === null ) ? null : $storagePath;
 686         }
 687
 688         /**
 689          * Get the final extension from a storage or FS path
 690          *
 691          * @param $path string
 692          * @return string
 693          */
 694         final public static function extensionFromPath( $path ) {
 695                 $i = strrpos( $path, '.' );
 696                 return strtolower( $i ? substr( $path, $i + 1 ) : '' );
 697         }
 698 }
 699
 700 /**
 701  * Base class for all backends associated with a particular storage medium.
 702  * This class defines the methods as abstract that subclasses must implement.
 703  * Outside callers should *not* use functions with "Internal" in the name.
 704  *
 705  * The FileBackend operations are implemented using basic functions
 706  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
 707  * This class is also responsible for path resolution and sanitization.
 708  *
 709  * @ingroup FileBackend
 710  * @since 1.19
 711  */
 712 abstract class FileBackendStore extends FileBackend {
 713         /** @var Array Map of paths to small (RAM/disk) cache items */
 714         protected $cache = array(); // (storage path => key => value)
 715         protected $maxCacheSize = 100; // integer; max paths with entries
 716         /** @var Array Map of paths to large (RAM/disk) cache items */
 717         protected $expensiveCache = array(); // (storage path => key => value)
 718         protected $maxExpensiveCacheSize = 10; // integer; max paths with entries
 719
 720         /** @var Array Map of container names to sharding settings */
 721         protected $shardViaHashLevels = array(); // (container name => config array)
 722
 723         protected $maxFileSize = 1000000000; // integer bytes (1GB)
 724
 725         /**
 726          * Get the maximum allowable file size given backend
 727          * medium restrictions and basic performance constraints.
 728          * Do not call this function from places outside FileBackend and FileOp.
 729          *
 730          * @return integer Bytes
 731          */
 732         final public function maxFileSizeInternal() {
 733                 return $this->maxFileSize;
 734         }
 735
 736         /**
 737          * Check if a file can be created at a given storage path.
 738          * FS backends should check if the parent directory exists and the file is writable.
 739          * Backends using key/value stores should check if the container exists.
 740          *
 741          * @param $storagePath string
 742          * @return bool
 743          */
 744         abstract public function isPathUsableInternal( $storagePath );
 745
 746         /**
 747          * Create a file in the backend with the given contents.
 748          * Do not call this function from places outside FileBackend and FileOp.
 749          *
 750          * $params include:
 751          *     content       : the raw file contents
 752          *     dst           : destination storage path
 753          *     overwrite     : overwrite any file that exists at the destination
 754          *
 755          * @param $params Array
 756          * @return Status
 757          */
 758         final public function createInternal( array $params ) {
 759                 wfProfileIn( __METHOD__ );
 760                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 761                         $status = Status::newFatal( 'backend-fail-create', $params['dst'] );
 762                 } else {
 763                         $status = $this->doCreateInternal( $params );
 764                         $this->clearCache( array( $params['dst'] ) );
 765                 }
 766                 wfProfileOut( __METHOD__ );
 767                 return $status;
 768         }
 769
 770         /**
 771          * @see FileBackendStore::createInternal()
 772          */
 773         abstract protected function doCreateInternal( array $params );
 774
 775         /**
 776          * Store a file into the backend from a file on disk.
 777          * Do not call this function from places outside FileBackend and FileOp.
 778          *
 779          * $params include:
 780          *     src           : source path on disk
 781          *     dst           : destination storage path
 782          *     overwrite     : overwrite any file that exists at the destination
 783          *
 784          * @param $params Array
 785          * @return Status
 786          */
 787         final public function storeInternal( array $params ) {
 788                 wfProfileIn( __METHOD__ );
 789                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 790                         $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
 791                 } else {
 792                         $status = $this->doStoreInternal( $params );
 793                         $this->clearCache( array( $params['dst'] ) );
 794                 }
 795                 wfProfileOut( __METHOD__ );
 796                 return $status;
 797         }
 798
 799         /**
 800          * @see FileBackendStore::storeInternal()
 801          */
 802         abstract protected function doStoreInternal( array $params );
 803
 804         /**
 805          * Copy a file from one storage path to another in the backend.
 806          * Do not call this function from places outside FileBackend and FileOp.
 807          *
 808          * $params include:
 809          *     src           : source storage path
 810          *     dst           : destination storage path
 811          *     overwrite     : overwrite any file that exists at the destination
 812          *
 813          * @param $params Array
 814          * @return Status
 815          */
 816         final public function copyInternal( array $params ) {
 817                 wfProfileIn( __METHOD__ );
 818                 $status = $this->doCopyInternal( $params );
 819                 $this->clearCache( array( $params['dst'] ) );
 820                 wfProfileOut( __METHOD__ );
 821                 return $status;
 822         }
 823
 824         /**
 825          * @see FileBackendStore::copyInternal()
 826          */
 827         abstract protected function doCopyInternal( array $params );
 828
 829         /**
 830          * Delete a file at the storage path.
 831          * Do not call this function from places outside FileBackend and FileOp.
 832          *
 833          * $params include:
 834          *     src                 : source storage path
 835          *     ignoreMissingSource : do nothing if the source file does not exist
 836          *
 837          * @param $params Array
 838          * @return Status
 839          */
 840         final public function deleteInternal( array $params ) {
 841                 wfProfileIn( __METHOD__ );
 842                 $status = $this->doDeleteInternal( $params );
 843                 $this->clearCache( array( $params['src'] ) );
 844                 wfProfileOut( __METHOD__ );
 845                 return $status;
 846         }
 847
 848         /**
 849          * @see FileBackendStore::deleteInternal()
 850          */
 851         abstract protected function doDeleteInternal( array $params );
 852
 853         /**
 854          * Move a file from one storage path to another in the backend.
 855          * Do not call this function from places outside FileBackend and FileOp.
 856          *
 857          * $params include:
 858          *     src           : source storage path
 859          *     dst           : destination storage path
 860          *     overwrite     : overwrite any file that exists at the destination
 861          *
 862          * @param $params Array
 863          * @return Status
 864          */
 865         final public function moveInternal( array $params ) {
 866                 wfProfileIn( __METHOD__ );
 867                 $status = $this->doMoveInternal( $params );
 868                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 869                 wfProfileOut( __METHOD__ );
 870                 return $status;
 871         }
 872
 873         /**
 874          * @see FileBackendStore::moveInternal()
 875          */
 876         protected function doMoveInternal( array $params ) {
 877                 // Copy source to dest
 878                 $status = $this->copyInternal( $params );
 879                 if ( $status->isOK() ) {
 880                         // Delete source (only fails due to races or medium going down)
 881                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 882                         $status->setResult( true, $status->value ); // ignore delete() errors
 883                 }
 884                 return $status;
 885         }
 886
 887         /**
 888          * @see FileBackend::concatenate()
 889          */
 890         final public function concatenate( array $params ) {
 891                 wfProfileIn( __METHOD__ );
 892                 $status = Status::newGood();
 893
 894                 // Try to lock the source files for the scope of this function
 895                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 896                 if ( $status->isOK() ) {
 897                         // Actually do the concatenation
 898                         $status->merge( $this->doConcatenate( $params ) );
 899                 }
 900
 901                 wfProfileOut( __METHOD__ );
 902                 return $status;
 903         }
 904
 905         /**
 906          * @see FileBackendStore::concatenate()
 907          */
 908         protected function doConcatenate( array $params ) {
 909                 $status = Status::newGood();
 910                 $tmpPath = $params['dst']; // convenience
 911
 912                 // Check that the specified temp file is valid...
 913                 wfSuppressWarnings();
 914                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 915                 wfRestoreWarnings();
 916                 if ( !$ok ) { // not present or not empty
 917                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 918                         return $status;
 919                 }
 920
 921                 // Build up the temp file using the source chunks (in order)...
 922                 $tmpHandle = fopen( $tmpPath, 'ab' );
 923                 if ( $tmpHandle === false ) {
 924                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 925                         return $status;
 926                 }
 927                 foreach ( $params['srcs'] as $virtualSource ) {
 928                         // Get a local FS version of the chunk
 929                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 930                         if ( !$tmpFile ) {
 931                                 $status->fatal( 'backend-fail-read', $virtualSource );
 932                                 return $status;
 933                         }
 934                         // Get a handle to the local FS version
 935                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 936                         if ( $sourceHandle === false ) {
 937                                 fclose( $tmpHandle );
 938                                 $status->fatal( 'backend-fail-read', $virtualSource );
 939                                 return $status;
 940                         }
 941                         // Append chunk to file (pass chunk size to avoid magic quotes)
 942                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 943                                 fclose( $sourceHandle );
 944                                 fclose( $tmpHandle );
 945                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 946                                 return $status;
 947                         }
 948                         fclose( $sourceHandle );
 949                 }
 950                 if ( !fclose( $tmpHandle ) ) {
 951                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 952                         return $status;
 953                 }
 954
 955                 clearstatcache(); // temp file changed
 956
 957                 return $status;
 958         }
 959
 960         /**
 961          * @see FileBackend::doPrepare()
 962          */
 963         final protected function doPrepare( array $params ) {
 964                 wfProfileIn( __METHOD__ );
 965
 966                 $status = Status::newGood();
 967                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 968                 if ( $dir === null ) {
 969                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 970                         wfProfileOut( __METHOD__ );
 971                         return $status; // invalid storage path
 972                 }
 973
 974                 if ( $shard !== null ) { // confined to a single container/shard
 975                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 976                 } else { // directory is on several shards
 977                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 978                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 979                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 980                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 981                         }
 982                 }
 983
 984                 wfProfileOut( __METHOD__ );
 985                 return $status;
 986         }
 987
 988         /**
 989          * @see FileBackendStore::doPrepare()
 990          */
 991         protected function doPrepareInternal( $container, $dir, array $params ) {
 992                 return Status::newGood();
 993         }
 994
 995         /**
 996          * @see FileBackend::doSecure()
 997          */
 998         final protected function doSecure( array $params ) {
 999                 wfProfileIn( __METHOD__ );
1000                 $status = Status::newGood();
1001
1002                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1003                 if ( $dir === null ) {
1004                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
1005                         wfProfileOut( __METHOD__ );
1006                         return $status; // invalid storage path
1007                 }
1008
1009                 if ( $shard !== null ) { // confined to a single container/shard
1010                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
1011                 } else { // directory is on several shards
1012                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1013                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1014                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1015                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
1016                         }
1017                 }
1018
1019                 wfProfileOut( __METHOD__ );
1020                 return $status;
1021         }
1022
1023         /**
1024          * @see FileBackendStore::doSecure()
1025          */
1026         protected function doSecureInternal( $container, $dir, array $params ) {
1027                 return Status::newGood();
1028         }
1029
1030         /**
1031          * @see FileBackend::doClean()
1032          */
1033         final protected function doClean( array $params ) {
1034                 wfProfileIn( __METHOD__ );
1035                 $status = Status::newGood();
1036
1037                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1038                 if ( $dir === null ) {
1039                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
1040                         wfProfileOut( __METHOD__ );
1041                         return $status; // invalid storage path
1042                 }
1043
1044                 // Attempt to lock this directory...
1045                 $filesLockEx = array( $params['dir'] );
1046                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
1047                 if ( !$status->isOK() ) {
1048                         wfProfileOut( __METHOD__ );
1049                         return $status; // abort
1050                 }
1051
1052                 if ( $shard !== null ) { // confined to a single container/shard
1053                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
1054                 } else { // directory is on several shards
1055                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1056                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1057                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1058                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
1059                         }
1060                 }
1061
1062                 wfProfileOut( __METHOD__ );
1063                 return $status;
1064         }
1065
1066         /**
1067          * @see FileBackendStore::doClean()
1068          */
1069         protected function doCleanInternal( $container, $dir, array $params ) {
1070                 return Status::newGood();
1071         }
1072
1073         /**
1074          * @see FileBackend::fileExists()
1075          */
1076         final public function fileExists( array $params ) {
1077                 wfProfileIn( __METHOD__ );
1078                 $stat = $this->getFileStat( $params );
1079                 wfProfileOut( __METHOD__ );
1080                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
1081         }
1082
1083         /**
1084          * @see FileBackend::getFileTimestamp()
1085          */
1086         final public function getFileTimestamp( array $params ) {
1087                 wfProfileIn( __METHOD__ );
1088                 $stat = $this->getFileStat( $params );
1089                 wfProfileOut( __METHOD__ );
1090                 return $stat ? $stat['mtime'] : false;
1091         }
1092
1093         /**
1094          * @see FileBackend::getFileSize()
1095          */
1096         final public function getFileSize( array $params ) {
1097                 wfProfileIn( __METHOD__ );
1098                 $stat = $this->getFileStat( $params );
1099                 wfProfileOut( __METHOD__ );
1100                 return $stat ? $stat['size'] : false;
1101         }
1102
1103         /**
1104          * @see FileBackend::getFileStat()
1105          */
1106         final public function getFileStat( array $params ) {
1107                 wfProfileIn( __METHOD__ );
1108                 $path = self::normalizeStoragePath( $params['src'] );
1109                 if ( $path === null ) {
1110                         return false; // invalid storage path
1111                 }
1112                 $latest = !empty( $params['latest'] );
1113                 if ( isset( $this->cache[$path]['stat'] ) ) {
1114                         // If we want the latest data, check that this cached
1115                         // value was in fact fetched with the latest available data.
1116                         if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
1117                                 wfProfileOut( __METHOD__ );
1118                                 return $this->cache[$path]['stat'];
1119                         }
1120                 }
1121                 $stat = $this->doGetFileStat( $params );
1122                 if ( is_array( $stat ) ) { // don't cache negatives
1123                         $this->trimCache(); // limit memory
1124                         $this->cache[$path]['stat'] = $stat;
1125                         $this->cache[$path]['stat']['latest'] = $latest;
1126                 }
1127                 wfProfileOut( __METHOD__ );
1128                 return $stat;
1129         }
1130
1131         /**
1132          * @see FileBackendStore::getFileStat()
1133          */
1134         abstract protected function doGetFileStat( array $params );
1135
1136         /**
1137          * @see FileBackend::getFileContents()
1138          */
1139         public function getFileContents( array $params ) {
1140                 wfProfileIn( __METHOD__ );
1141                 $tmpFile = $this->getLocalReference( $params );
1142                 if ( !$tmpFile ) {
1143                         wfProfileOut( __METHOD__ );
1144                         return false;
1145                 }
1146                 wfSuppressWarnings();
1147                 $data = file_get_contents( $tmpFile->getPath() );
1148                 wfRestoreWarnings();
1149                 wfProfileOut( __METHOD__ );
1150                 return $data;
1151         }
1152
1153         /**
1154          * @see FileBackend::getFileSha1Base36()
1155          */
1156         final public function getFileSha1Base36( array $params ) {
1157                 wfProfileIn( __METHOD__ );
1158                 $path = $params['src'];
1159                 if ( isset( $this->cache[$path]['sha1'] ) ) {
1160                         wfProfileOut( __METHOD__ );
1161                         return $this->cache[$path]['sha1'];
1162                 }
1163                 $hash = $this->doGetFileSha1Base36( $params );
1164                 if ( $hash ) { // don't cache negatives
1165                         $this->trimCache(); // limit memory
1166                         $this->cache[$path]['sha1'] = $hash;
1167                 }
1168                 wfProfileOut( __METHOD__ );
1169                 return $hash;
1170         }
1171
1172         /**
1173          * @see FileBackendStore::getFileSha1Base36()
1174          */
1175         protected function doGetFileSha1Base36( array $params ) {
1176                 $fsFile = $this->getLocalReference( $params );
1177                 if ( !$fsFile ) {
1178                         return false;
1179                 } else {
1180                         return $fsFile->getSha1Base36();
1181                 }
1182         }
1183
1184         /**
1185          * @see FileBackend::getFileProps()
1186          */
1187         final public function getFileProps( array $params ) {
1188                 wfProfileIn( __METHOD__ );
1189                 $fsFile = $this->getLocalReference( $params );
1190                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
1191                 wfProfileOut( __METHOD__ );
1192                 return $props;
1193         }
1194
1195         /**
1196          * @see FileBackend::getLocalReference()
1197          */
1198         public function getLocalReference( array $params ) {
1199                 wfProfileIn( __METHOD__ );
1200                 $path = $params['src'];
1201                 if ( isset( $this->expensiveCache[$path]['localRef'] ) ) {
1202                         wfProfileOut( __METHOD__ );
1203                         return $this->expensiveCache[$path]['localRef'];
1204                 }
1205                 $tmpFile = $this->getLocalCopy( $params );
1206                 if ( $tmpFile ) { // don't cache negatives
1207                         $this->trimExpensiveCache(); // limit memory
1208                         $this->expensiveCache[$path]['localRef'] = $tmpFile;
1209                 }
1210                 wfProfileOut( __METHOD__ );
1211                 return $tmpFile;
1212         }
1213
1214         /**
1215          * @see FileBackend::streamFile()
1216          */
1217         final public function streamFile( array $params ) {
1218                 wfProfileIn( __METHOD__ );
1219                 $status = Status::newGood();
1220
1221                 $info = $this->getFileStat( $params );
1222                 if ( !$info ) { // let StreamFile handle the 404
1223                         $status->fatal( 'backend-fail-notexists', $params['src'] );
1224                 }
1225
1226                 // Set output buffer and HTTP headers for stream
1227                 $extraHeaders = $params['headers'] ? $params['headers'] : array();
1228                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
1229                 if ( $res == StreamFile::NOT_MODIFIED ) {
1230                         // do nothing; client cache is up to date
1231                 } elseif ( $res == StreamFile::READY_STREAM ) {
1232                         $status = $this->doStreamFile( $params );
1233                 } else {
1234                         $status->fatal( 'backend-fail-stream', $params['src'] );
1235                 }
1236
1237                 wfProfileOut( __METHOD__ );
1238                 return $status;
1239         }
1240
1241         /**
1242          * @see FileBackendStore::streamFile()
1243          */
1244         protected function doStreamFile( array $params ) {
1245                 $status = Status::newGood();
1246
1247                 $fsFile = $this->getLocalReference( $params );
1248                 if ( !$fsFile ) {
1249                         $status->fatal( 'backend-fail-stream', $params['src'] );
1250                 } elseif ( !readfile( $fsFile->getPath() ) ) {
1251                         $status->fatal( 'backend-fail-stream', $params['src'] );
1252                 }
1253
1254                 return $status;
1255         }
1256
1257         /**
1258          * @see FileBackend::getFileList()
1259          */
1260         final public function getFileList( array $params ) {
1261                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1262                 if ( $dir === null ) { // invalid storage path
1263                         return null;
1264                 }
1265                 if ( $shard !== null ) {
1266                         // File listing is confined to a single container/shard
1267                         return $this->getFileListInternal( $fullCont, $dir, $params );
1268                 } else {
1269                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1270                         // File listing spans multiple containers/shards
1271                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1272                         return new FileBackendStoreShardListIterator( $this,
1273                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1274                 }
1275         }
1276
1277         /**
1278          * Do not call this function from places outside FileBackend
1279          *
1280          * @see FileBackendStore::getFileList()
1281          *
1282          * @param $container string Resolved container name
1283          * @param $dir string Resolved path relative to container
1284          * @param $params Array
1285          * @return Traversable|Array|null
1286          */
1287         abstract public function getFileListInternal( $container, $dir, array $params );
1288
1289         /**
1290          * Get the list of supported operations and their corresponding FileOp classes.
1291          *
1292          * @return Array
1293          */
1294         protected function supportedOperations() {
1295                 return array(
1296                         'store'       => 'StoreFileOp',
1297                         'copy'        => 'CopyFileOp',
1298                         'move'        => 'MoveFileOp',
1299                         'delete'      => 'DeleteFileOp',
1300                         'create'      => 'CreateFileOp',
1301                         'null'        => 'NullFileOp'
1302                 );
1303         }
1304
1305         /**
1306          * Return a list of FileOp objects from a list of operations.
1307          * Do not call this function from places outside FileBackend.
1308          *
1309          * The result must have the same number of items as the input.
1310          * An exception is thrown if an unsupported operation is requested.
1311          *
1312          * @param $ops Array Same format as doOperations()
1313          * @return Array List of FileOp objects
1314          * @throws MWException
1315          */
1316         final public function getOperations( array $ops ) {
1317                 $supportedOps = $this->supportedOperations();
1318
1319                 $performOps = array(); // array of FileOp objects
1320                 // Build up ordered array of FileOps...
1321                 foreach ( $ops as $operation ) {
1322                         $opName = $operation['op'];
1323                         if ( isset( $supportedOps[$opName] ) ) {
1324                                 $class = $supportedOps[$opName];
1325                                 // Get params for this operation
1326                                 $params = $operation;
1327                                 // Append the FileOp class
1328                                 $performOps[] = new $class( $this, $params );
1329                         } else {
1330                                 throw new MWException( "Operation `$opName` is not supported." );
1331                         }
1332                 }
1333
1334                 return $performOps;
1335         }
1336
1337         /**
1338          * @see FileBackend::doOperationsInternal()
1339          */
1340         protected function doOperationsInternal( array $ops, array $opts ) {
1341                 wfProfileIn( __METHOD__ );
1342                 $status = Status::newGood();
1343
1344                 // Build up a list of FileOps...
1345                 $performOps = $this->getOperations( $ops );
1346
1347                 // Acquire any locks as needed...
1348                 if ( empty( $opts['nonLocking'] ) ) {
1349                         // Build up a list of files to lock...
1350                         $filesLockEx = $filesLockSh = array();
1351                         foreach ( $performOps as $fileOp ) {
1352                                 $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
1353                                 $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
1354                         }
1355                         // Optimization: if doing an EX lock anyway, don't also set an SH one
1356                         $filesLockSh = array_diff( $filesLockSh, $filesLockEx );
1357                         // Get a shared lock on the parent directory of each path changed
1358                         $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
1359                         // Try to lock those files for the scope of this function...
1360                         $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
1361                         $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
1362                         if ( !$status->isOK() ) {
1363                                 wfProfileOut( __METHOD__ );
1364                                 return $status; // abort
1365                         }
1366                 }
1367
1368                 // Clear any cache entries (after locks acquired)
1369                 $this->clearCache();
1370
1371                 // Actually attempt the operation batch...
1372                 $subStatus = FileOp::attemptBatch( $performOps, $opts );
1373
1374                 // Merge errors into status fields
1375                 $status->merge( $subStatus );
1376                 $status->success = $subStatus->success; // not done in merge()
1377
1378                 wfProfileOut( __METHOD__ );
1379                 return $status;
1380         }
1381
1382         /**
1383          * @see FileBackend::clearCache()
1384          */
1385         final public function clearCache( array $paths = null ) {
1386                 if ( is_array( $paths ) ) {
1387                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1388                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
1389                 }
1390                 if ( $paths === null ) {
1391                         $this->cache = array();
1392                         $this->expensiveCache = array();
1393                 } else {
1394                         foreach ( $paths as $path ) {
1395                                 unset( $this->cache[$path] );
1396                                 unset( $this->expensiveCache[$path] );
1397                         }
1398                 }
1399                 $this->doClearCache( $paths );
1400         }
1401
1402         /**
1403          * Clears any additional stat caches for storage paths
1404          *
1405          * @see FileBackend::clearCache()
1406          *
1407          * @param $paths Array Storage paths (optional)
1408          * @return void
1409          */
1410         protected function doClearCache( array $paths = null ) {}
1411
1412         /**
1413          * Prune the inexpensive cache if it is too big to add an item
1414          *
1415          * @return void
1416          */
1417         protected function trimCache() {
1418                 if ( count( $this->cache ) >= $this->maxCacheSize ) {
1419                         reset( $this->cache );
1420                         unset( $this->cache[key( $this->cache )] );
1421                 }
1422         }
1423
1424         /**
1425          * Prune the expensive cache if it is too big to add an item
1426          *
1427          * @return void
1428          */
1429         protected function trimExpensiveCache() {
1430                 if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) {
1431                         reset( $this->expensiveCache );
1432                         unset( $this->expensiveCache[key( $this->expensiveCache )] );
1433                 }
1434         }
1435
1436         /**
1437          * Check if a container name is valid.
1438          * This checks for for length and illegal characters.
1439          *
1440          * @param $container string
1441          * @return bool
1442          */
1443         final protected static function isValidContainerName( $container ) {
1444                 // This accounts for Swift and S3 restrictions while leaving room
1445                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1446                 // This disallows directory separators or traversal characters.
1447                 // Note that matching strings URL encode to the same string;
1448                 // in Swift, the length restriction is *after* URL encoding.
1449                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1450         }
1451
1452         /**
1453          * Splits a storage path into an internal container name,
1454          * an internal relative file name, and a container shard suffix.
1455          * Any shard suffix is already appended to the internal container name.
1456          * This also checks that the storage path is valid and within this backend.
1457          *
1458          * If the container is sharded but a suffix could not be determined,
1459          * this means that the path can only refer to a directory and can only
1460          * be scanned by looking in all the container shards.
1461          *
1462          * @param $storagePath string
1463          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1464          */
1465         final protected function resolveStoragePath( $storagePath ) {
1466                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1467                 if ( $backend === $this->name ) { // must be for this backend
1468                         $relPath = self::normalizeContainerPath( $relPath );
1469                         if ( $relPath !== null ) {
1470                                 // Get shard for the normalized path if this container is sharded
1471                                 $cShard = $this->getContainerShard( $container, $relPath );
1472                                 // Validate and sanitize the relative path (backend-specific)
1473                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1474                                 if ( $relPath !== null ) {
1475                                         // Prepend any wiki ID prefix to the container name
1476                                         $container = $this->fullContainerName( $container );
1477                                         if ( self::isValidContainerName( $container ) ) {
1478                                                 // Validate and sanitize the container name (backend-specific)
1479                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1480                                                 if ( $container !== null ) {
1481                                                         return array( $container, $relPath, $cShard );
1482                                                 }
1483                                         }
1484                                 }
1485                         }
1486                 }
1487                 return array( null, null, null );
1488         }
1489
1490         /**
1491          * Like resolveStoragePath() except null values are returned if
1492          * the container is sharded and the shard could not be determined.
1493          *
1494          * @see FileBackendStore::resolveStoragePath()
1495          *
1496          * @param $storagePath string
1497          * @return Array (container, path) or (null, null) if invalid
1498          */
1499         final protected function resolveStoragePathReal( $storagePath ) {
1500                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1501                 if ( $cShard !== null ) {
1502                         return array( $container, $relPath );
1503                 }
1504                 return array( null, null );
1505         }
1506
1507         /**
1508          * Get the container name shard suffix for a given path.
1509          * Any empty suffix means the container is not sharded.
1510          *
1511          * @param $container string Container name
1512          * @param $relStoragePath string Storage path relative to the container
1513          * @return string|null Returns null if shard could not be determined
1514          */
1515         final protected function getContainerShard( $container, $relPath ) {
1516                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1517                 if ( $levels == 1 || $levels == 2 ) {
1518                         // Hash characters are either base 16 or 36
1519                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1520                         // Get a regex that represents the shard portion of paths.
1521                         // The concatenation of the captures gives us the shard.
1522                         if ( $levels === 1 ) { // 16 or 36 shards per container
1523                                 $hashDirRegex = '(' . $char . ')';
1524                         } else { // 256 or 1296 shards per container
1525                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1526                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1527                                 } else { // short hash dir format (e.g. "a/b/c")
1528                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1529                                 }
1530                         }
1531                         // Allow certain directories to be above the hash dirs so as
1532                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1533                         // They must be 2+ chars to avoid any hash directory ambiguity.
1534                         $m = array();
1535                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1536                                 return '.' . implode( '', array_slice( $m, 1 ) );
1537                         }
1538                         return null; // failed to match
1539                 }
1540                 return ''; // no sharding
1541         }
1542
1543         /**
1544          * Get the sharding config for a container.
1545          * If greater than 0, then all file storage paths within
1546          * the container are required to be hashed accordingly.
1547          *
1548          * @param $container string
1549          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1550          */
1551         final protected function getContainerHashLevels( $container ) {
1552                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1553                         $config = $this->shardViaHashLevels[$container];
1554                         $hashLevels = (int)$config['levels'];
1555                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1556                                 $hashBase = (int)$config['base'];
1557                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1558                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1559                                 }
1560                         }
1561                 }
1562                 return array( 0, 0, false ); // no sharding
1563         }
1564
1565         /**
1566          * Get a list of full container shard suffixes for a container
1567          *
1568          * @param $container string
1569          * @return Array
1570          */
1571         final protected function getContainerSuffixes( $container ) {
1572                 $shards = array();
1573                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1574                 if ( $digits > 0 ) {
1575                         $numShards = pow( $base, $digits );
1576                         for ( $index = 0; $index < $numShards; $index++ ) {
1577                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1578                         }
1579                 }
1580                 return $shards;
1581         }
1582
1583         /**
1584          * Get the full container name, including the wiki ID prefix
1585          *
1586          * @param $container string
1587          * @return string
1588          */
1589         final protected function fullContainerName( $container ) {
1590                 if ( $this->wikiId != '' ) {
1591                         return "{$this->wikiId}-$container";
1592                 } else {
1593                         return $container;
1594                 }
1595         }
1596
1597         /**
1598          * Resolve a container name, checking if it's allowed by the backend.
1599          * This is intended for internal use, such as encoding illegal chars.
1600          * Subclasses can override this to be more restrictive.
1601          *
1602          * @param $container string
1603          * @return string|null
1604          */
1605         protected function resolveContainerName( $container ) {
1606                 return $container;
1607         }
1608
1609         /**
1610          * Resolve a relative storage path, checking if it's allowed by the backend.
1611          * This is intended for internal use, such as encoding illegal chars or perhaps
1612          * getting absolute paths (e.g. FS based backends). Note that the relative path
1613          * may be the empty string (e.g. the path is simply to the container).
1614          *
1615          * @param $container string Container name
1616          * @param $relStoragePath string Storage path relative to the container
1617          * @return string|null Path or null if not valid
1618          */
1619         protected function resolveContainerPath( $container, $relStoragePath ) {
1620                 return $relStoragePath;
1621         }
1622 }
1623
1624 /**
1625  * FileBackendStore helper function to handle file listings that span container shards.
1626  * Do not use this class from places outside of FileBackendStore.
1627  *
1628  * @ingroup FileBackendStore
1629  */
1630 class FileBackendStoreShardListIterator implements Iterator {
1631         /* @var FileBackendStore */
1632         protected $backend;
1633         /* @var Array */
1634         protected $params;
1635         /* @var Array */
1636         protected $shardSuffixes;
1637         protected $container; // string
1638         protected $directory; // string
1639
1640         /* @var Traversable */
1641         protected $iter;
1642         protected $curShard = 0; // integer
1643         protected $pos = 0; // integer
1644
1645         /**
1646          * @param $backend FileBackendStore
1647          * @param $container string Full storage container name
1648          * @param $dir string Storage directory relative to container
1649          * @param $suffixes Array List of container shard suffixes
1650          * @param $params Array
1651          */
1652         public function __construct(
1653                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1654         ) {
1655                 $this->backend = $backend;
1656                 $this->container = $container;
1657                 $this->directory = $dir;
1658                 $this->shardSuffixes = $suffixes;
1659                 $this->params = $params;
1660         }
1661
1662         public function current() {
1663                 if ( is_array( $this->iter ) ) {
1664                         return current( $this->iter );
1665                 } else {
1666                         return $this->iter->current();
1667                 }
1668         }
1669
1670         public function key() {
1671                 return $this->pos;
1672         }
1673
1674         public function next() {
1675                 ++$this->pos;
1676                 if ( is_array( $this->iter ) ) {
1677                         next( $this->iter );
1678                 } else {
1679                         $this->iter->next();
1680                 }
1681                 // Find the next non-empty shard if no elements are left
1682                 $this->nextShardIteratorIfNotValid();
1683         }
1684
1685         /**
1686          * If the iterator for this container shard is out of items,
1687          * then move on to the next container that has items.
1688          * If there are none, then it advances to the last container.
1689          */
1690         protected function nextShardIteratorIfNotValid() {
1691                 while ( !$this->valid() ) {
1692                         if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1693                                 break; // no more container shards
1694                         }
1695                         $this->setIteratorFromCurrentShard();
1696                 }
1697         }
1698
1699         protected function setIteratorFromCurrentShard() {
1700                 $suffix = $this->shardSuffixes[$this->curShard];
1701                 $this->iter = $this->backend->getFileListInternal(
1702                         "{$this->container}{$suffix}", $this->directory, $this->params );
1703         }
1704
1705         public function rewind() {
1706                 $this->pos = 0;
1707                 $this->curShard = 0;
1708                 $this->setIteratorFromCurrentShard();
1709                 // Find the next non-empty shard if this one has no elements
1710                 $this->nextShardIteratorIfNotValid();
1711         }
1712
1713         public function valid() {
1714                 if ( $this->iter == null ) {
1715                         return false; // some failure?
1716                 } elseif ( is_array( $this->iter ) ) {
1717                         return ( current( $this->iter ) !== false ); // no paths can have this value
1718                 } else {
1719                         return $this->iter->valid();
1720                 }
1721         }
1722 }