includes/filerepo/backend/FileBackend.php

   1 <?php
   2 /**
   3  * @defgroup FileBackend File backend
   4  * @ingroup  FileRepo
   5  *
   6  * File backend is used to interact with file storage systems,
   7  * such as the local file system, NFS, or cloud storage systems.
   8  */
   9
  10 /**
  11  * @file
  12  * @ingroup FileBackend
  13  * @author Aaron Schulz
  14  */
  15
  16 /**
  17  * Base class for all file backend classes (including multi-write backends).
  18  *
  19  * This class defines the methods as abstract that subclasses must implement.
  20  * Outside callers can assume that all backends will have these functions.
  21  *
  22  * All "storage paths" are of the format "mwstore://<backend>/<container>/<path>".
  23  * The <path> portion is a relative path that uses UNIX file system (FS) notation,
  24  * though any particular backend may not actually be using a local filesystem.
  25  * Therefore, the relative paths are only virtual.
  26  *
  27  * Backend contents are stored under wiki-specific container names by default.
  28  * For legacy reasons, this has no effect for the FS backend class, and per-wiki
  29  * segregation must be done by setting the container paths appropriately.
  30  *
  31  * FS-based backends are somewhat more restrictive due to the existence of real
  32  * directory files; a regular file cannot have the same name as a directory. Other
  33  * backends with virtual directories may not have this limitation. Callers should
  34  * store files in such a way that no files and directories are under the same path.
  35  *
  36  * Methods should avoid throwing exceptions at all costs.
  37  * As a corollary, external dependencies should be kept to a minimum.
  38  *
  39  * @ingroup FileBackend
  40  * @since 1.19
  41  */
  42 abstract class FileBackend {
  43         protected $name; // string; unique backend name
  44         protected $wikiId; // string; unique wiki name
  45         protected $readOnly; // string; read-only explanation message
  46         /** @var LockManager */
  47         protected $lockManager;
  48
  49         /**
  50          * Create a new backend instance from configuration.
  51          * This should only be called from within FileBackendGroup.
  52          *
  53          * $config includes:
  54          *     'name'        : The unique name of this backend.
  55          *                     This should consist of alphanumberic, '-', and '_' characters.
  56          *     'wikiId'      : Prefix to container names that is unique to this wiki.
  57          *                     This should consist of alphanumberic, '-', and '_' characters.
  58          *     'lockManager' : Registered name of a file lock manager to use.
  59          *     'readOnly'    : Write operations are disallowed if this is a non-empty string.
  60          *                     It should be an explanation for the backend being read-only.
  61          *
  62          * @param $config Array
  63          */
  64         public function __construct( array $config ) {
  65                 $this->name = $config['name'];
  66                 $this->wikiId = isset( $config['wikiId'] )
  67                         ? $config['wikiId']
  68                         : wfWikiID(); // e.g. "my_wiki-en_"
  69                 $this->lockManager = ( $config['lockManager'] instanceof LockManager )
  70                         ? $config['lockManager']
  71                         : LockManagerGroup::singleton()->get( $config['lockManager'] );
  72                 $this->readOnly = isset( $config['readOnly'] )
  73                         ? (string)$config['readOnly']
  74                         : '';
  75         }
  76
  77         /**
  78          * Get the unique backend name.
  79          * We may have multiple different backends of the same type.
  80          * For example, we can have two Swift backends using different proxies.
  81          *
  82          * @return string
  83          */
  84         final public function getName() {
  85                 return $this->name;
  86         }
  87
  88         /**
  89          * Check if this backend is read-only
  90          *
  91          * @return bool
  92          */
  93         final public function isReadOnly() {
  94                 return ( $this->readOnly != '' );
  95         }
  96
  97         /**
  98          * Get an explanatory message if this backend is read-only
  99          *
 100          * @return string|bool Returns falls if the backend is not read-only
 101          */
 102         final public function getReadOnlyReason() {
 103                 return ( $this->readOnly != '' ) ? $this->readOnly : false;
 104         }
 105
 106         /**
 107          * This is the main entry point into the backend for write operations.
 108          * Callers supply an ordered list of operations to perform as a transaction.
 109          * Files will be locked, the stat cache cleared, and then the operations attempted.
 110          * If any serious errors occur, all attempted operations will be rolled back.
 111          *
 112          * $ops is an array of arrays. The outer array holds a list of operations.
 113          * Each inner array is a set of key value pairs that specify an operation.
 114          *
 115          * Supported operations and their parameters:
 116          * a) Create a new file in storage with the contents of a string
 117          *     array(
 118          *         'op'                  => 'create',
 119          *         'dst'                 => <storage path>,
 120          *         'content'             => <string of new file contents>,
 121          *         'overwrite'           => <boolean>,
 122          *         'overwriteSame'       => <boolean>
 123          *     )
 124          * b) Copy a file system file into storage
 125          *     array(
 126          *         'op'                  => 'store',
 127          *         'src'                 => <file system path>,
 128          *         'dst'                 => <storage path>,
 129          *         'overwrite'           => <boolean>,
 130          *         'overwriteSame'       => <boolean>
 131          *     )
 132          * c) Copy a file within storage
 133          *     array(
 134          *         'op'                  => 'copy',
 135          *         'src'                 => <storage path>,
 136          *         'dst'                 => <storage path>,
 137          *         'overwrite'           => <boolean>,
 138          *         'overwriteSame'       => <boolean>
 139          *     )
 140          * d) Move a file within storage
 141          *     array(
 142          *         'op'                  => 'move',
 143          *         'src'                 => <storage path>,
 144          *         'dst'                 => <storage path>,
 145          *         'overwrite'           => <boolean>,
 146          *         'overwriteSame'       => <boolean>
 147          *     )
 148          * e) Delete a file within storage
 149          *     array(
 150          *         'op'                  => 'delete',
 151          *         'src'                 => <storage path>,
 152          *         'ignoreMissingSource' => <boolean>
 153          *     )
 154          * f) Do nothing (no-op)
 155          *     array(
 156          *         'op'                  => 'null',
 157          *     )
 158          *
 159          * Boolean flags for operations (operation-specific):
 160          * 'ignoreMissingSource' : The operation will simply succeed and do
 161          *                         nothing if the source file does not exist.
 162          * 'overwrite'           : Any destination file will be overwritten.
 163          * 'overwriteSame'       : An error will not be given if a file already
 164          *                         exists at the destination that has the same
 165          *                         contents as the new contents to be written there.
 166          *
 167          * $opts is an associative of boolean flags, including:
 168          * 'force'               : Errors that would normally cause a rollback do not.
 169          *                         The remaining operations are still attempted if any fail.
 170          * 'nonLocking'          : No locks are acquired for the operations.
 171          *                         This can increase performance for non-critical writes.
 172          *                         This has no effect unless the 'force' flag is set.
 173          * 'allowStale'          : Don't require the latest available data.
 174          *                         This can increase performance for non-critical writes.
 175          *                         This has no effect unless the 'force' flag is set.
 176          *
 177          * Remarks on locking:
 178          * File system paths given to operations should refer to files that are
 179          * already locked or otherwise safe from modification from other processes.
 180          * Normally these files will be new temp files, which should be adequate.
 181          *
 182          * Return value:
 183          * This returns a Status, which contains all warnings and fatals that occured
 184          * during the operation. The 'failCount', 'successCount', and 'success' members
 185          * will reflect each operation attempted. The status will be "OK" unless any
 186          * of the operations failed and the 'force' parameter was not set.
 187          *
 188          * @param $ops Array List of operations to execute in order
 189          * @param $opts Array Batch operation options
 190          * @return Status
 191          */
 192         final public function doOperations( array $ops, array $opts = array() ) {
 193                 if ( $this->isReadOnly() ) {
 194                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 195                 }
 196                 if ( empty( $opts['force'] ) ) { // sanity
 197                         unset( $opts['nonLocking'] );
 198                         unset( $opts['allowStale'] );
 199                 }
 200                 return $this->doOperationsInternal( $ops, $opts );
 201         }
 202
 203         /**
 204          * @see FileBackend::doOperations()
 205          */
 206         abstract protected function doOperationsInternal( array $ops, array $opts );
 207
 208         /**
 209          * Same as doOperations() except it takes a single operation.
 210          * If you are doing a batch of operations that should either
 211          * all succeed or all fail, then use that function instead.
 212          *
 213          * @see FileBackend::doOperations()
 214          *
 215          * @param $op Array Operation
 216          * @param $opts Array Operation options
 217          * @return Status
 218          */
 219         final public function doOperation( array $op, array $opts = array() ) {
 220                 return $this->doOperations( array( $op ), $opts );
 221         }
 222
 223         /**
 224          * Performs a single create operation.
 225          * This sets $params['op'] to 'create' and passes it to doOperation().
 226          *
 227          * @see FileBackend::doOperation()
 228          *
 229          * @param $params Array Operation parameters
 230          * @param $opts Array Operation options
 231          * @return Status
 232          */
 233         final public function create( array $params, array $opts = array() ) {
 234                 $params['op'] = 'create';
 235                 return $this->doOperation( $params, $opts );
 236         }
 237
 238         /**
 239          * Performs a single store operation.
 240          * This sets $params['op'] to 'store' and passes it to doOperation().
 241          *
 242          * @see FileBackend::doOperation()
 243          *
 244          * @param $params Array Operation parameters
 245          * @param $opts Array Operation options
 246          * @return Status
 247          */
 248         final public function store( array $params, array $opts = array() ) {
 249                 $params['op'] = 'store';
 250                 return $this->doOperation( $params, $opts );
 251         }
 252
 253         /**
 254          * Performs a single copy operation.
 255          * This sets $params['op'] to 'copy' and passes it to doOperation().
 256          *
 257          * @see FileBackend::doOperation()
 258          *
 259          * @param $params Array Operation parameters
 260          * @param $opts Array Operation options
 261          * @return Status
 262          */
 263         final public function copy( array $params, array $opts = array() ) {
 264                 $params['op'] = 'copy';
 265                 return $this->doOperation( $params, $opts );
 266         }
 267
 268         /**
 269          * Performs a single move operation.
 270          * This sets $params['op'] to 'move' and passes it to doOperation().
 271          *
 272          * @see FileBackend::doOperation()
 273          *
 274          * @param $params Array Operation parameters
 275          * @param $opts Array Operation options
 276          * @return Status
 277          */
 278         final public function move( array $params, array $opts = array() ) {
 279                 $params['op'] = 'move';
 280                 return $this->doOperation( $params, $opts );
 281         }
 282
 283         /**
 284          * Performs a single delete operation.
 285          * This sets $params['op'] to 'delete' and passes it to doOperation().
 286          *
 287          * @see FileBackend::doOperation()
 288          *
 289          * @param $params Array Operation parameters
 290          * @param $opts Array Operation options
 291          * @return Status
 292          */
 293         final public function delete( array $params, array $opts = array() ) {
 294                 $params['op'] = 'delete';
 295                 return $this->doOperation( $params, $opts );
 296         }
 297
 298         /**
 299          * Concatenate a list of storage files into a single file system file.
 300          * The target path should refer to a file that is already locked or
 301          * otherwise safe from modification from other processes. Normally,
 302          * the file will be a new temp file, which should be adequate.
 303          * $params include:
 304          *     srcs          : ordered source storage paths (e.g. chunk1, chunk2, ...)
 305          *     dst           : file system path to 0-byte temp file
 306          *
 307          * @param $params Array Operation parameters
 308          * @return Status
 309          */
 310         abstract public function concatenate( array $params );
 311
 312         /**
 313          * Prepare a storage directory for usage.
 314          * This will create any required containers and parent directories.
 315          * Backends using key/value stores only need to create the container.
 316          *
 317          * $params include:
 318          *     dir : storage directory
 319          *
 320          * @param $params Array
 321          * @return Status
 322          */
 323         final public function prepare( array $params ) {
 324                 if ( $this->isReadOnly() ) {
 325                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 326                 }
 327                 return $this->doPrepare( $params );
 328         }
 329
 330         /**
 331          * @see FileBackend::prepare()
 332          */
 333         abstract protected function doPrepare( array $params );
 334
 335         /**
 336          * Take measures to block web access to a storage directory and
 337          * the container it belongs to. FS backends might add .htaccess
 338          * files whereas key/value store backends might restrict container
 339          * access to the auth user that represents end-users in web request.
 340          * This is not guaranteed to actually do anything.
 341          *
 342          * $params include:
 343          *     dir       : storage directory
 344          *     noAccess  : try to deny file access
 345          *     noListing : try to deny file listing
 346          *
 347          * @param $params Array
 348          * @return Status
 349          */
 350         final public function secure( array $params ) {
 351                 if ( $this->isReadOnly() ) {
 352                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 353                 }
 354                 $status = $this->doPrepare( $params ); // dir must exist to restrict it
 355                 if ( $status->isOK() ) {
 356                         $status->merge( $this->doSecure( $params ) );
 357                 }
 358                 return $status;
 359         }
 360
 361         /**
 362          * @see FileBackend::secure()
 363          */
 364         abstract protected function doSecure( array $params );
 365
 366         /**
 367          * Delete a storage directory if it is empty.
 368          * Backends using key/value stores may do nothing unless the directory
 369          * is that of an empty container, in which case it should be deleted.
 370          *
 371          * $params include:
 372          *     dir : storage directory
 373          *
 374          * @param $params Array
 375          * @return Status
 376          */
 377         final public function clean( array $params ) {
 378                 if ( $this->isReadOnly() ) {
 379                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 380                 }
 381                 return $this->doClean( $params );
 382         }
 383
 384         /**
 385          * @see FileBackend::clean()
 386          */
 387         abstract protected function doClean( array $params );
 388
 389         /**
 390          * Check if a file exists at a storage path in the backend.
 391          * This returns false if only a directory exists at the path.
 392          *
 393          * $params include:
 394          *     src    : source storage path
 395          *     latest : use the latest available data
 396          *
 397          * @param $params Array
 398          * @return bool|null Returns null on failure
 399          */
 400         abstract public function fileExists( array $params );
 401
 402         /**
 403          * Get the last-modified timestamp of the file at a storage path.
 404          *
 405          * $params include:
 406          *     src    : source storage path
 407          *     latest : use the latest available data
 408          *
 409          * @param $params Array
 410          * @return string|bool TS_MW timestamp or false on failure
 411          */
 412         abstract public function getFileTimestamp( array $params );
 413
 414         /**
 415          * Get the contents of a file at a storage path in the backend.
 416          * This should be avoided for potentially large files.
 417          *
 418          * $params include:
 419          *     src    : source storage path
 420          *     latest : use the latest available data
 421          *
 422          * @param $params Array
 423          * @return string|bool Returns false on failure
 424          */
 425         abstract public function getFileContents( array $params );
 426
 427         /**
 428          * Get the size (bytes) of a file at a storage path in the backend.
 429          *
 430          * $params include:
 431          *     src    : source storage path
 432          *     latest : use the latest available data
 433          *
 434          * @param $params Array
 435          * @return integer|bool Returns false on failure
 436          */
 437         abstract public function getFileSize( array $params );
 438
 439         /**
 440          * Get quick information about a file at a storage path in the backend.
 441          * If the file does not exist, then this returns false.
 442          * Otherwise, the result is an associative array that includes:
 443          *     mtime  : the last-modified timestamp (TS_MW)
 444          *     size   : the file size (bytes)
 445          * Additional values may be included for internal use only.
 446          *
 447          * $params include:
 448          *     src    : source storage path
 449          *     latest : use the latest available data
 450          *
 451          * @param $params Array
 452          * @return Array|bool|null Returns null on failure
 453          */
 454         abstract public function getFileStat( array $params );
 455
 456         /**
 457          * Get a SHA-1 hash of the file at a storage path in the backend.
 458          *
 459          * $params include:
 460          *     src    : source storage path
 461          *     latest : use the latest available data
 462          *
 463          * @param $params Array
 464          * @return string|bool Hash string or false on failure
 465          */
 466         abstract public function getFileSha1Base36( array $params );
 467
 468         /**
 469          * Get the properties of the file at a storage path in the backend.
 470          * Returns FSFile::placeholderProps() on failure.
 471          *
 472          * $params include:
 473          *     src    : source storage path
 474          *     latest : use the latest available data
 475          *
 476          * @param $params Array
 477          * @return Array
 478          */
 479         abstract public function getFileProps( array $params );
 480
 481         /**
 482          * Stream the file at a storage path in the backend.
 483          * If the file does not exists, a 404 error will be given.
 484          * Appropriate HTTP headers (Status, Content-Type, Content-Length)
 485          * must be sent if streaming began, while none should be sent otherwise.
 486          * Implementations should flush the output buffer before sending data.
 487          *
 488          * $params include:
 489          *     src     : source storage path
 490          *     headers : additional HTTP headers to send on success
 491          *     latest  : use the latest available data
 492          *
 493          * @param $params Array
 494          * @return Status
 495          */
 496         abstract public function streamFile( array $params );
 497
 498         /**
 499          * Returns a file system file, identical to the file at a storage path.
 500          * The file returned is either:
 501          * a) A local copy of the file at a storage path in the backend.
 502          *    The temporary copy will have the same extension as the source.
 503          * b) An original of the file at a storage path in the backend.
 504          * Temporary files may be purged when the file object falls out of scope.
 505          *
 506          * Write operations should *never* be done on this file as some backends
 507          * may do internal tracking or may be instances of FileBackendMultiWrite.
 508          * In that later case, there are copies of the file that must stay in sync.
 509          * Additionally, further calls to this function may return the same file.
 510          *
 511          * $params include:
 512          *     src    : source storage path
 513          *     latest : use the latest available data
 514          *
 515          * @param $params Array
 516          * @return FSFile|null Returns null on failure
 517          */
 518         abstract public function getLocalReference( array $params );
 519
 520         /**
 521          * Get a local copy on disk of the file at a storage path in the backend.
 522          * The temporary copy will have the same file extension as the source.
 523          * Temporary files may be purged when the file object falls out of scope.
 524          *
 525          * $params include:
 526          *     src    : source storage path
 527          *     latest : use the latest available data
 528          *
 529          * @param $params Array
 530          * @return TempFSFile|null Returns null on failure
 531          */
 532         abstract public function getLocalCopy( array $params );
 533
 534         /**
 535          * Get an iterator to list out all stored files under a storage directory.
 536          * If the directory is of the form "mwstore://backend/container",
 537          * then all files in the container should be listed.
 538          * If the directory is of form "mwstore://backend/container/dir",
 539          * then all files under that container directory should be listed.
 540          * Results should be storage paths relative to the given directory.
 541          *
 542          * Storage backends with eventual consistency might return stale data.
 543          *
 544          * $params include:
 545          *     dir : storage path directory
 546          *
 547          * @return Traversable|Array|null Returns null on failure
 548          */
 549         abstract public function getFileList( array $params );
 550
 551         /**
 552          * Invalidate any in-process file existence and property cache.
 553          * If $paths is given, then only the cache for those files will be cleared.
 554          *
 555          * @param $paths Array Storage paths (optional)
 556          * @return void
 557          */
 558         public function clearCache( array $paths = null ) {}
 559
 560         /**
 561          * Lock the files at the given storage paths in the backend.
 562          * This will either lock all the files or none (on failure).
 563          *
 564          * Callers should consider using getScopedFileLocks() instead.
 565          *
 566          * @param $paths Array Storage paths
 567          * @param $type integer LockManager::LOCK_* constant
 568          * @return Status
 569          */
 570         final public function lockFiles( array $paths, $type ) {
 571                 return $this->lockManager->lock( $paths, $type );
 572         }
 573
 574         /**
 575          * Unlock the files at the given storage paths in the backend.
 576          *
 577          * @param $paths Array Storage paths
 578          * @param $type integer LockManager::LOCK_* constant
 579          * @return Status
 580          */
 581         final public function unlockFiles( array $paths, $type ) {
 582                 return $this->lockManager->unlock( $paths, $type );
 583         }
 584
 585         /**
 586          * Lock the files at the given storage paths in the backend.
 587          * This will either lock all the files or none (on failure).
 588          * On failure, the status object will be updated with errors.
 589          *
 590          * Once the return value goes out scope, the locks will be released and
 591          * the status updated. Unlock fatals will not change the status "OK" value.
 592          *
 593          * @param $paths Array Storage paths
 594          * @param $type integer LockManager::LOCK_* constant
 595          * @param $status Status Status to update on lock/unlock
 596          * @return ScopedLock|null Returns null on failure
 597          */
 598         final public function getScopedFileLocks( array $paths, $type, Status $status ) {
 599                 return ScopedLock::factory( $this->lockManager, $paths, $type, $status );
 600         }
 601
 602         /**
 603          * Check if a given path is a "mwstore://" path.
 604          * This does not do any further validation or any existence checks.
 605          *
 606          * @param $path string
 607          * @return bool
 608          */
 609         final public static function isStoragePath( $path ) {
 610                 return ( strpos( $path, 'mwstore://' ) === 0 );
 611         }
 612
 613         /**
 614          * Split a storage path into a backend name, a container name,
 615          * and a relative file path. The relative path may be the empty string.
 616          * This does not do any path normalization or traversal checks.
 617          *
 618          * @param $storagePath string
 619          * @return Array (backend, container, rel object) or (null, null, null)
 620          */
 621         final public static function splitStoragePath( $storagePath ) {
 622                 if ( self::isStoragePath( $storagePath ) ) {
 623                         // Remove the "mwstore://" prefix and split the path
 624                         $parts = explode( '/', substr( $storagePath, 10 ), 3 );
 625                         if ( count( $parts ) >= 2 && $parts[0] != '' && $parts[1] != '' ) {
 626                                 if ( count( $parts ) == 3 ) {
 627                                         return $parts; // e.g. "backend/container/path"
 628                                 } else {
 629                                         return array( $parts[0], $parts[1], '' ); // e.g. "backend/container"
 630                                 }
 631                         }
 632                 }
 633                 return array( null, null, null );
 634         }
 635
 636         /**
 637          * Normalize a storage path by cleaning up directory separators.
 638          * Returns null if the path is not of the format of a valid storage path.
 639          *
 640          * @param $storagePath string
 641          * @return string|null
 642          */
 643         final public static function normalizeStoragePath( $storagePath ) {
 644                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
 645                 if ( $relPath !== null ) { // must be for this backend
 646                         $relPath = self::normalizeContainerPath( $relPath );
 647                         if ( $relPath !== null ) {
 648                                 return ( $relPath != '' )
 649                                         ? "mwstore://{$backend}/{$container}/{$relPath}"
 650                                         : "mwstore://{$backend}/{$container}";
 651                         }
 652                 }
 653                 return null;
 654         }
 655
 656         /**
 657          * Validate and normalize a relative storage path.
 658          * Null is returned if the path involves directory traversal.
 659          * Traversal is insecure for FS backends and broken for others.
 660          *
 661          * @param $path string Storage path relative to a container
 662          * @return string|null
 663          */
 664         final protected static function normalizeContainerPath( $path ) {
 665                 // Normalize directory separators
 666                 $path = strtr( $path, '\\', '/' );
 667                 // Collapse any consecutive directory separators
 668                 $path = preg_replace( '![/]{2,}!', '/', $path );
 669                 // Remove any leading directory separator
 670                 $path = ltrim( $path, '/' );
 671                 // Use the same traversal protection as Title::secureAndSplit()
 672                 if ( strpos( $path, '.' ) !== false ) {
 673                         if (
 674                                 $path === '.' ||
 675                                 $path === '..' ||
 676                                 strpos( $path, './' ) === 0 ||
 677                                 strpos( $path, '../' ) === 0 ||
 678                                 strpos( $path, '/./' ) !== false ||
 679                                 strpos( $path, '/../' ) !== false
 680                         ) {
 681                                 return null;
 682                         }
 683                 }
 684                 return $path;
 685         }
 686
 687         /**
 688          * Get the parent storage directory of a storage path.
 689          * This returns a path like "mwstore://backend/container",
 690          * "mwstore://backend/container/...", or null if there is no parent.
 691          *
 692          * @param $storagePath string
 693          * @return string|null
 694          */
 695         final public static function parentStoragePath( $storagePath ) {
 696                 $storagePath = dirname( $storagePath );
 697                 list( $b, $cont, $rel ) = self::splitStoragePath( $storagePath );
 698                 return ( $rel === null ) ? null : $storagePath;
 699         }
 700
 701         /**
 702          * Get the final extension from a storage or FS path
 703          *
 704          * @param $path string
 705          * @return string
 706          */
 707         final public static function extensionFromPath( $path ) {
 708                 $i = strrpos( $path, '.' );
 709                 return strtolower( $i ? substr( $path, $i + 1 ) : '' );
 710         }
 711 }
 712
 713 /**
 714  * @brief Base class for all backends using particular storage medium.
 715  *
 716  * This class defines the methods as abstract that subclasses must implement.
 717  * Outside callers should *not* use functions with "Internal" in the name.
 718  *
 719  * The FileBackend operations are implemented using basic functions
 720  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
 721  * This class is also responsible for path resolution and sanitization.
 722  *
 723  * @ingroup FileBackend
 724  * @since 1.19
 725  */
 726 abstract class FileBackendStore extends FileBackend {
 727         /** @var Array Map of paths to small (RAM/disk) cache items */
 728         protected $cache = array(); // (storage path => key => value)
 729         protected $maxCacheSize = 100; // integer; max paths with entries
 730         /** @var Array Map of paths to large (RAM/disk) cache items */
 731         protected $expensiveCache = array(); // (storage path => key => value)
 732         protected $maxExpensiveCacheSize = 10; // integer; max paths with entries
 733
 734         /** @var Array Map of container names to sharding settings */
 735         protected $shardViaHashLevels = array(); // (container name => config array)
 736
 737         protected $maxFileSize = 1000000000; // integer bytes (1GB)
 738
 739         /**
 740          * Get the maximum allowable file size given backend
 741          * medium restrictions and basic performance constraints.
 742          * Do not call this function from places outside FileBackend and FileOp.
 743          *
 744          * @return integer Bytes
 745          */
 746         final public function maxFileSizeInternal() {
 747                 return $this->maxFileSize;
 748         }
 749
 750         /**
 751          * Check if a file can be created at a given storage path.
 752          * FS backends should check if the parent directory exists and the file is writable.
 753          * Backends using key/value stores should check if the container exists.
 754          *
 755          * @param $storagePath string
 756          * @return bool
 757          */
 758         abstract public function isPathUsableInternal( $storagePath );
 759
 760         /**
 761          * Create a file in the backend with the given contents.
 762          * Do not call this function from places outside FileBackend and FileOp.
 763          *
 764          * $params include:
 765          *     content       : the raw file contents
 766          *     dst           : destination storage path
 767          *     overwrite     : overwrite any file that exists at the destination
 768          *
 769          * @param $params Array
 770          * @return Status
 771          */
 772         final public function createInternal( array $params ) {
 773                 wfProfileIn( __METHOD__ );
 774                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 775                         $status = Status::newFatal( 'backend-fail-create', $params['dst'] );
 776                 } else {
 777                         $status = $this->doCreateInternal( $params );
 778                         $this->clearCache( array( $params['dst'] ) );
 779                 }
 780                 wfProfileOut( __METHOD__ );
 781                 return $status;
 782         }
 783
 784         /**
 785          * @see FileBackendStore::createInternal()
 786          */
 787         abstract protected function doCreateInternal( array $params );
 788
 789         /**
 790          * Store a file into the backend from a file on disk.
 791          * Do not call this function from places outside FileBackend and FileOp.
 792          *
 793          * $params include:
 794          *     src           : source path on disk
 795          *     dst           : destination storage path
 796          *     overwrite     : overwrite any file that exists at the destination
 797          *
 798          * @param $params Array
 799          * @return Status
 800          */
 801         final public function storeInternal( array $params ) {
 802                 wfProfileIn( __METHOD__ );
 803                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 804                         $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
 805                 } else {
 806                         $status = $this->doStoreInternal( $params );
 807                         $this->clearCache( array( $params['dst'] ) );
 808                 }
 809                 wfProfileOut( __METHOD__ );
 810                 return $status;
 811         }
 812
 813         /**
 814          * @see FileBackendStore::storeInternal()
 815          */
 816         abstract protected function doStoreInternal( array $params );
 817
 818         /**
 819          * Copy a file from one storage path to another in the backend.
 820          * Do not call this function from places outside FileBackend and FileOp.
 821          *
 822          * $params include:
 823          *     src           : source storage path
 824          *     dst           : destination storage path
 825          *     overwrite     : overwrite any file that exists at the destination
 826          *
 827          * @param $params Array
 828          * @return Status
 829          */
 830         final public function copyInternal( array $params ) {
 831                 wfProfileIn( __METHOD__ );
 832                 $status = $this->doCopyInternal( $params );
 833                 $this->clearCache( array( $params['dst'] ) );
 834                 wfProfileOut( __METHOD__ );
 835                 return $status;
 836         }
 837
 838         /**
 839          * @see FileBackendStore::copyInternal()
 840          */
 841         abstract protected function doCopyInternal( array $params );
 842
 843         /**
 844          * Delete a file at the storage path.
 845          * Do not call this function from places outside FileBackend and FileOp.
 846          *
 847          * $params include:
 848          *     src                 : source storage path
 849          *     ignoreMissingSource : do nothing if the source file does not exist
 850          *
 851          * @param $params Array
 852          * @return Status
 853          */
 854         final public function deleteInternal( array $params ) {
 855                 wfProfileIn( __METHOD__ );
 856                 $status = $this->doDeleteInternal( $params );
 857                 $this->clearCache( array( $params['src'] ) );
 858                 wfProfileOut( __METHOD__ );
 859                 return $status;
 860         }
 861
 862         /**
 863          * @see FileBackendStore::deleteInternal()
 864          */
 865         abstract protected function doDeleteInternal( array $params );
 866
 867         /**
 868          * Move a file from one storage path to another in the backend.
 869          * Do not call this function from places outside FileBackend and FileOp.
 870          *
 871          * $params include:
 872          *     src           : source storage path
 873          *     dst           : destination storage path
 874          *     overwrite     : overwrite any file that exists at the destination
 875          *
 876          * @param $params Array
 877          * @return Status
 878          */
 879         final public function moveInternal( array $params ) {
 880                 wfProfileIn( __METHOD__ );
 881                 $status = $this->doMoveInternal( $params );
 882                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 883                 wfProfileOut( __METHOD__ );
 884                 return $status;
 885         }
 886
 887         /**
 888          * @see FileBackendStore::moveInternal()
 889          * @return Status
 890          */
 891         protected function doMoveInternal( array $params ) {
 892                 // Copy source to dest
 893                 $status = $this->copyInternal( $params );
 894                 if ( $status->isOK() ) {
 895                         // Delete source (only fails due to races or medium going down)
 896                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 897                         $status->setResult( true, $status->value ); // ignore delete() errors
 898                 }
 899                 return $status;
 900         }
 901
 902         /**
 903          * @see FileBackend::concatenate()
 904          * @return Status
 905          */
 906         final public function concatenate( array $params ) {
 907                 wfProfileIn( __METHOD__ );
 908                 $status = Status::newGood();
 909
 910                 // Try to lock the source files for the scope of this function
 911                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 912                 if ( $status->isOK() ) {
 913                         // Actually do the concatenation
 914                         $status->merge( $this->doConcatenate( $params ) );
 915                 }
 916
 917                 wfProfileOut( __METHOD__ );
 918                 return $status;
 919         }
 920
 921         /**
 922          * @see FileBackendStore::concatenate()
 923          * @return Status
 924          */
 925         protected function doConcatenate( array $params ) {
 926                 $status = Status::newGood();
 927                 $tmpPath = $params['dst']; // convenience
 928
 929                 // Check that the specified temp file is valid...
 930                 wfSuppressWarnings();
 931                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 932                 wfRestoreWarnings();
 933                 if ( !$ok ) { // not present or not empty
 934                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 935                         return $status;
 936                 }
 937
 938                 // Build up the temp file using the source chunks (in order)...
 939                 $tmpHandle = fopen( $tmpPath, 'ab' );
 940                 if ( $tmpHandle === false ) {
 941                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 942                         return $status;
 943                 }
 944                 foreach ( $params['srcs'] as $virtualSource ) {
 945                         // Get a local FS version of the chunk
 946                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 947                         if ( !$tmpFile ) {
 948                                 $status->fatal( 'backend-fail-read', $virtualSource );
 949                                 return $status;
 950                         }
 951                         // Get a handle to the local FS version
 952                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 953                         if ( $sourceHandle === false ) {
 954                                 fclose( $tmpHandle );
 955                                 $status->fatal( 'backend-fail-read', $virtualSource );
 956                                 return $status;
 957                         }
 958                         // Append chunk to file (pass chunk size to avoid magic quotes)
 959                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 960                                 fclose( $sourceHandle );
 961                                 fclose( $tmpHandle );
 962                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 963                                 return $status;
 964                         }
 965                         fclose( $sourceHandle );
 966                 }
 967                 if ( !fclose( $tmpHandle ) ) {
 968                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 969                         return $status;
 970                 }
 971
 972                 clearstatcache(); // temp file changed
 973
 974                 return $status;
 975         }
 976
 977         /**
 978          * @see FileBackend::doPrepare()
 979          * @return Status
 980          */
 981         final protected function doPrepare( array $params ) {
 982                 wfProfileIn( __METHOD__ );
 983
 984                 $status = Status::newGood();
 985                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 986                 if ( $dir === null ) {
 987                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 988                         wfProfileOut( __METHOD__ );
 989                         return $status; // invalid storage path
 990                 }
 991
 992                 if ( $shard !== null ) { // confined to a single container/shard
 993                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 994                 } else { // directory is on several shards
 995                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 996                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 997                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 998                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 999                         }
1000                 }
1001
1002                 wfProfileOut( __METHOD__ );
1003                 return $status;
1004         }
1005
1006         /**
1007          * @see FileBackendStore::doPrepare()
1008          * @return Status
1009          */
1010         protected function doPrepareInternal( $container, $dir, array $params ) {
1011                 return Status::newGood();
1012         }
1013
1014         /**
1015          * @see FileBackend::doSecure()
1016          * @return Status
1017          */
1018         final protected function doSecure( array $params ) {
1019                 wfProfileIn( __METHOD__ );
1020                 $status = Status::newGood();
1021
1022                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1023                 if ( $dir === null ) {
1024                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
1025                         wfProfileOut( __METHOD__ );
1026                         return $status; // invalid storage path
1027                 }
1028
1029                 if ( $shard !== null ) { // confined to a single container/shard
1030                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
1031                 } else { // directory is on several shards
1032                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1033                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1034                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1035                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
1036                         }
1037                 }
1038
1039                 wfProfileOut( __METHOD__ );
1040                 return $status;
1041         }
1042
1043         /**
1044          * @see FileBackendStore::doSecure()
1045          * @return Status
1046          */
1047         protected function doSecureInternal( $container, $dir, array $params ) {
1048                 return Status::newGood();
1049         }
1050
1051         /**
1052          * @see FileBackend::doClean()
1053          * @return Status
1054          */
1055         final protected function doClean( array $params ) {
1056                 wfProfileIn( __METHOD__ );
1057                 $status = Status::newGood();
1058
1059                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1060                 if ( $dir === null ) {
1061                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
1062                         wfProfileOut( __METHOD__ );
1063                         return $status; // invalid storage path
1064                 }
1065
1066                 // Attempt to lock this directory...
1067                 $filesLockEx = array( $params['dir'] );
1068                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
1069                 if ( !$status->isOK() ) {
1070                         wfProfileOut( __METHOD__ );
1071                         return $status; // abort
1072                 }
1073
1074                 if ( $shard !== null ) { // confined to a single container/shard
1075                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
1076                 } else { // directory is on several shards
1077                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1078                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1079                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
1080                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
1081                         }
1082                 }
1083
1084                 wfProfileOut( __METHOD__ );
1085                 return $status;
1086         }
1087
1088         /**
1089          * @see FileBackendStore::doClean()
1090          * @return Status
1091          */
1092         protected function doCleanInternal( $container, $dir, array $params ) {
1093                 return Status::newGood();
1094         }
1095
1096         /**
1097          * @see FileBackend::fileExists()
1098          * @return bool|null
1099          */
1100         final public function fileExists( array $params ) {
1101                 wfProfileIn( __METHOD__ );
1102                 $stat = $this->getFileStat( $params );
1103                 wfProfileOut( __METHOD__ );
1104                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
1105         }
1106
1107         /**
1108          * @see FileBackend::getFileTimestamp()
1109          * @return bool
1110          */
1111         final public function getFileTimestamp( array $params ) {
1112                 wfProfileIn( __METHOD__ );
1113                 $stat = $this->getFileStat( $params );
1114                 wfProfileOut( __METHOD__ );
1115                 return $stat ? $stat['mtime'] : false;
1116         }
1117
1118         /**
1119          * @see FileBackend::getFileSize()
1120          * @return bool
1121          */
1122         final public function getFileSize( array $params ) {
1123                 wfProfileIn( __METHOD__ );
1124                 $stat = $this->getFileStat( $params );
1125                 wfProfileOut( __METHOD__ );
1126                 return $stat ? $stat['size'] : false;
1127         }
1128
1129         /**
1130          * @see FileBackend::getFileStat()
1131          * @return bool
1132          */
1133         final public function getFileStat( array $params ) {
1134                 wfProfileIn( __METHOD__ );
1135                 $path = self::normalizeStoragePath( $params['src'] );
1136                 if ( $path === null ) {
1137                         wfProfileOut( __METHOD__ );
1138                         return false; // invalid storage path
1139                 }
1140                 $latest = !empty( $params['latest'] );
1141                 if ( isset( $this->cache[$path]['stat'] ) ) {
1142                         // If we want the latest data, check that this cached
1143                         // value was in fact fetched with the latest available data.
1144                         if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
1145                                 wfProfileOut( __METHOD__ );
1146                                 return $this->cache[$path]['stat'];
1147                         }
1148                 }
1149                 $stat = $this->doGetFileStat( $params );
1150                 if ( is_array( $stat ) ) { // don't cache negatives
1151                         $this->trimCache(); // limit memory
1152                         $this->cache[$path]['stat'] = $stat;
1153                         $this->cache[$path]['stat']['latest'] = $latest;
1154                 }
1155                 wfProfileOut( __METHOD__ );
1156                 return $stat;
1157         }
1158
1159         /**
1160          * @see FileBackendStore::getFileStat()
1161          */
1162         abstract protected function doGetFileStat( array $params );
1163
1164         /**
1165          * @see FileBackend::getFileContents()
1166          * @return bool|string
1167          */
1168         public function getFileContents( array $params ) {
1169                 wfProfileIn( __METHOD__ );
1170                 $tmpFile = $this->getLocalReference( $params );
1171                 if ( !$tmpFile ) {
1172                         wfProfileOut( __METHOD__ );
1173                         return false;
1174                 }
1175                 wfSuppressWarnings();
1176                 $data = file_get_contents( $tmpFile->getPath() );
1177                 wfRestoreWarnings();
1178                 wfProfileOut( __METHOD__ );
1179                 return $data;
1180         }
1181
1182         /**
1183          * @see FileBackend::getFileSha1Base36()
1184          * @return bool|string
1185          */
1186         final public function getFileSha1Base36( array $params ) {
1187                 wfProfileIn( __METHOD__ );
1188                 $path = $params['src'];
1189                 if ( isset( $this->cache[$path]['sha1'] ) ) {
1190                         wfProfileOut( __METHOD__ );
1191                         return $this->cache[$path]['sha1'];
1192                 }
1193                 $hash = $this->doGetFileSha1Base36( $params );
1194                 if ( $hash ) { // don't cache negatives
1195                         $this->trimCache(); // limit memory
1196                         $this->cache[$path]['sha1'] = $hash;
1197                 }
1198                 wfProfileOut( __METHOD__ );
1199                 return $hash;
1200         }
1201
1202         /**
1203          * @see FileBackendStore::getFileSha1Base36()
1204          * @return bool
1205          */
1206         protected function doGetFileSha1Base36( array $params ) {
1207                 $fsFile = $this->getLocalReference( $params );
1208                 if ( !$fsFile ) {
1209                         return false;
1210                 } else {
1211                         return $fsFile->getSha1Base36();
1212                 }
1213         }
1214
1215         /**
1216          * @see FileBackend::getFileProps()
1217          * @return Array
1218          */
1219         final public function getFileProps( array $params ) {
1220                 wfProfileIn( __METHOD__ );
1221                 $fsFile = $this->getLocalReference( $params );
1222                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
1223                 wfProfileOut( __METHOD__ );
1224                 return $props;
1225         }
1226
1227         /**
1228          * @see FileBackend::getLocalReference()
1229          * @return TempFSFile|null
1230          */
1231         public function getLocalReference( array $params ) {
1232                 wfProfileIn( __METHOD__ );
1233                 $path = $params['src'];
1234                 if ( isset( $this->expensiveCache[$path]['localRef'] ) ) {
1235                         wfProfileOut( __METHOD__ );
1236                         return $this->expensiveCache[$path]['localRef'];
1237                 }
1238                 $tmpFile = $this->getLocalCopy( $params );
1239                 if ( $tmpFile ) { // don't cache negatives
1240                         $this->trimExpensiveCache(); // limit memory
1241                         $this->expensiveCache[$path]['localRef'] = $tmpFile;
1242                 }
1243                 wfProfileOut( __METHOD__ );
1244                 return $tmpFile;
1245         }
1246
1247         /**
1248          * @see FileBackend::streamFile()
1249          * @return Status
1250          */
1251         final public function streamFile( array $params ) {
1252                 wfProfileIn( __METHOD__ );
1253                 $status = Status::newGood();
1254
1255                 $info = $this->getFileStat( $params );
1256                 if ( !$info ) { // let StreamFile handle the 404
1257                         $status->fatal( 'backend-fail-notexists', $params['src'] );
1258                 }
1259
1260                 // Set output buffer and HTTP headers for stream
1261                 $extraHeaders = $params['headers'] ? $params['headers'] : array();
1262                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
1263                 if ( $res == StreamFile::NOT_MODIFIED ) {
1264                         // do nothing; client cache is up to date
1265                 } elseif ( $res == StreamFile::READY_STREAM ) {
1266                         $status = $this->doStreamFile( $params );
1267                 } else {
1268                         $status->fatal( 'backend-fail-stream', $params['src'] );
1269                 }
1270
1271                 wfProfileOut( __METHOD__ );
1272                 return $status;
1273         }
1274
1275         /**
1276          * @see FileBackendStore::streamFile()
1277          * @return Status
1278          */
1279         protected function doStreamFile( array $params ) {
1280                 $status = Status::newGood();
1281
1282                 $fsFile = $this->getLocalReference( $params );
1283                 if ( !$fsFile ) {
1284                         $status->fatal( 'backend-fail-stream', $params['src'] );
1285                 } elseif ( !readfile( $fsFile->getPath() ) ) {
1286                         $status->fatal( 'backend-fail-stream', $params['src'] );
1287                 }
1288
1289                 return $status;
1290         }
1291
1292         /**
1293          * @copydoc FileBackend::getFileList()
1294          * @return Array|null|Traversable
1295          */
1296         final public function getFileList( array $params ) {
1297                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1298                 if ( $dir === null ) { // invalid storage path
1299                         return null;
1300                 }
1301                 if ( $shard !== null ) {
1302                         // File listing is confined to a single container/shard
1303                         return $this->getFileListInternal( $fullCont, $dir, $params );
1304                 } else {
1305                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1306                         // File listing spans multiple containers/shards
1307                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1308                         return new FileBackendStoreShardListIterator( $this,
1309                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1310                 }
1311         }
1312
1313         /**
1314          * Do not call this function from places outside FileBackend
1315          *
1316          * @see FileBackendStore::getFileList()
1317          *
1318          * @param $container string Resolved container name
1319          * @param $dir string Resolved path relative to container
1320          * @param $params Array
1321          * @return Traversable|Array|null
1322          */
1323         abstract public function getFileListInternal( $container, $dir, array $params );
1324
1325         /**
1326          * Get the list of supported operations and their corresponding FileOp classes.
1327          *
1328          * @return Array
1329          */
1330         protected function supportedOperations() {
1331                 return array(
1332                         'store'       => 'StoreFileOp',
1333                         'copy'        => 'CopyFileOp',
1334                         'move'        => 'MoveFileOp',
1335                         'delete'      => 'DeleteFileOp',
1336                         'create'      => 'CreateFileOp',
1337                         'null'        => 'NullFileOp'
1338                 );
1339         }
1340
1341         /**
1342          * Return a list of FileOp objects from a list of operations.
1343          * Do not call this function from places outside FileBackend.
1344          *
1345          * The result must have the same number of items as the input.
1346          * An exception is thrown if an unsupported operation is requested.
1347          *
1348          * @param $ops Array Same format as doOperations()
1349          * @return Array List of FileOp objects
1350          * @throws MWException
1351          */
1352         final public function getOperations( array $ops ) {
1353                 $supportedOps = $this->supportedOperations();
1354
1355                 $performOps = array(); // array of FileOp objects
1356                 // Build up ordered array of FileOps...
1357                 foreach ( $ops as $operation ) {
1358                         $opName = $operation['op'];
1359                         if ( isset( $supportedOps[$opName] ) ) {
1360                                 $class = $supportedOps[$opName];
1361                                 // Get params for this operation
1362                                 $params = $operation;
1363                                 // Append the FileOp class
1364                                 $performOps[] = new $class( $this, $params );
1365                         } else {
1366                                 throw new MWException( "Operation `$opName` is not supported." );
1367                         }
1368                 }
1369
1370                 return $performOps;
1371         }
1372
1373         /**
1374          * @see FileBackend::doOperationsInternal()
1375          * @return Status
1376          */
1377         protected function doOperationsInternal( array $ops, array $opts ) {
1378                 wfProfileIn( __METHOD__ );
1379                 $status = Status::newGood();
1380
1381                 // Build up a list of FileOps...
1382                 $performOps = $this->getOperations( $ops );
1383
1384                 // Acquire any locks as needed...
1385                 if ( empty( $opts['nonLocking'] ) ) {
1386                         // Build up a list of files to lock...
1387                         $filesLockEx = $filesLockSh = array();
1388                         foreach ( $performOps as $fileOp ) {
1389                                 $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
1390                                 $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
1391                         }
1392                         // Optimization: if doing an EX lock anyway, don't also set an SH one
1393                         $filesLockSh = array_diff( $filesLockSh, $filesLockEx );
1394                         // Get a shared lock on the parent directory of each path changed
1395                         $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
1396                         // Try to lock those files for the scope of this function...
1397                         $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
1398                         $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
1399                         if ( !$status->isOK() ) {
1400                                 wfProfileOut( __METHOD__ );
1401                                 return $status; // abort
1402                         }
1403                 }
1404
1405                 // Clear any cache entries (after locks acquired)
1406                 $this->clearCache();
1407
1408                 // Actually attempt the operation batch...
1409                 $subStatus = FileOp::attemptBatch( $performOps, $opts );
1410
1411                 // Merge errors into status fields
1412                 $status->merge( $subStatus );
1413                 $status->success = $subStatus->success; // not done in merge()
1414
1415                 wfProfileOut( __METHOD__ );
1416                 return $status;
1417         }
1418
1419         /**
1420          * @see FileBackend::clearCache()
1421          */
1422         final public function clearCache( array $paths = null ) {
1423                 if ( is_array( $paths ) ) {
1424                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1425                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
1426                 }
1427                 if ( $paths === null ) {
1428                         $this->cache = array();
1429                         $this->expensiveCache = array();
1430                 } else {
1431                         foreach ( $paths as $path ) {
1432                                 unset( $this->cache[$path] );
1433                                 unset( $this->expensiveCache[$path] );
1434                         }
1435                 }
1436                 $this->doClearCache( $paths );
1437         }
1438
1439         /**
1440          * Clears any additional stat caches for storage paths
1441          *
1442          * @see FileBackend::clearCache()
1443          *
1444          * @param $paths Array Storage paths (optional)
1445          * @return void
1446          */
1447         protected function doClearCache( array $paths = null ) {}
1448
1449         /**
1450          * Prune the inexpensive cache if it is too big to add an item
1451          *
1452          * @return void
1453          */
1454         protected function trimCache() {
1455                 if ( count( $this->cache ) >= $this->maxCacheSize ) {
1456                         reset( $this->cache );
1457                         unset( $this->cache[key( $this->cache )] );
1458                 }
1459         }
1460
1461         /**
1462          * Prune the expensive cache if it is too big to add an item
1463          *
1464          * @return void
1465          */
1466         protected function trimExpensiveCache() {
1467                 if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) {
1468                         reset( $this->expensiveCache );
1469                         unset( $this->expensiveCache[key( $this->expensiveCache )] );
1470                 }
1471         }
1472
1473         /**
1474          * Check if a container name is valid.
1475          * This checks for for length and illegal characters.
1476          *
1477          * @param $container string
1478          * @return bool
1479          */
1480         final protected static function isValidContainerName( $container ) {
1481                 // This accounts for Swift and S3 restrictions while leaving room
1482                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1483                 // This disallows directory separators or traversal characters.
1484                 // Note that matching strings URL encode to the same string;
1485                 // in Swift, the length restriction is *after* URL encoding.
1486                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1487         }
1488
1489         /**
1490          * Splits a storage path into an internal container name,
1491          * an internal relative file name, and a container shard suffix.
1492          * Any shard suffix is already appended to the internal container name.
1493          * This also checks that the storage path is valid and within this backend.
1494          *
1495          * If the container is sharded but a suffix could not be determined,
1496          * this means that the path can only refer to a directory and can only
1497          * be scanned by looking in all the container shards.
1498          *
1499          * @param $storagePath string
1500          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1501          */
1502         final protected function resolveStoragePath( $storagePath ) {
1503                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1504                 if ( $backend === $this->name ) { // must be for this backend
1505                         $relPath = self::normalizeContainerPath( $relPath );
1506                         if ( $relPath !== null ) {
1507                                 // Get shard for the normalized path if this container is sharded
1508                                 $cShard = $this->getContainerShard( $container, $relPath );
1509                                 // Validate and sanitize the relative path (backend-specific)
1510                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1511                                 if ( $relPath !== null ) {
1512                                         // Prepend any wiki ID prefix to the container name
1513                                         $container = $this->fullContainerName( $container );
1514                                         if ( self::isValidContainerName( $container ) ) {
1515                                                 // Validate and sanitize the container name (backend-specific)
1516                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1517                                                 if ( $container !== null ) {
1518                                                         return array( $container, $relPath, $cShard );
1519                                                 }
1520                                         }
1521                                 }
1522                         }
1523                 }
1524                 return array( null, null, null );
1525         }
1526
1527         /**
1528          * Like resolveStoragePath() except null values are returned if
1529          * the container is sharded and the shard could not be determined.
1530          *
1531          * @see FileBackendStore::resolveStoragePath()
1532          *
1533          * @param $storagePath string
1534          * @return Array (container, path) or (null, null) if invalid
1535          */
1536         final protected function resolveStoragePathReal( $storagePath ) {
1537                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1538                 if ( $cShard !== null ) {
1539                         return array( $container, $relPath );
1540                 }
1541                 return array( null, null );
1542         }
1543
1544         /**
1545          * Get the container name shard suffix for a given path.
1546          * Any empty suffix means the container is not sharded.
1547          *
1548          * @param $container string Container name
1549          * @param $relStoragePath string Storage path relative to the container
1550          * @return string|null Returns null if shard could not be determined
1551          */
1552         final protected function getContainerShard( $container, $relPath ) {
1553                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1554                 if ( $levels == 1 || $levels == 2 ) {
1555                         // Hash characters are either base 16 or 36
1556                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1557                         // Get a regex that represents the shard portion of paths.
1558                         // The concatenation of the captures gives us the shard.
1559                         if ( $levels === 1 ) { // 16 or 36 shards per container
1560                                 $hashDirRegex = '(' . $char . ')';
1561                         } else { // 256 or 1296 shards per container
1562                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1563                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1564                                 } else { // short hash dir format (e.g. "a/b/c")
1565                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1566                                 }
1567                         }
1568                         // Allow certain directories to be above the hash dirs so as
1569                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1570                         // They must be 2+ chars to avoid any hash directory ambiguity.
1571                         $m = array();
1572                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1573                                 return '.' . implode( '', array_slice( $m, 1 ) );
1574                         }
1575                         return null; // failed to match
1576                 }
1577                 return ''; // no sharding
1578         }
1579
1580         /**
1581          * Get the sharding config for a container.
1582          * If greater than 0, then all file storage paths within
1583          * the container are required to be hashed accordingly.
1584          *
1585          * @param $container string
1586          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1587          */
1588         final protected function getContainerHashLevels( $container ) {
1589                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1590                         $config = $this->shardViaHashLevels[$container];
1591                         $hashLevels = (int)$config['levels'];
1592                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1593                                 $hashBase = (int)$config['base'];
1594                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1595                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1596                                 }
1597                         }
1598                 }
1599                 return array( 0, 0, false ); // no sharding
1600         }
1601
1602         /**
1603          * Get a list of full container shard suffixes for a container
1604          *
1605          * @param $container string
1606          * @return Array
1607          */
1608         final protected function getContainerSuffixes( $container ) {
1609                 $shards = array();
1610                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1611                 if ( $digits > 0 ) {
1612                         $numShards = pow( $base, $digits );
1613                         for ( $index = 0; $index < $numShards; $index++ ) {
1614                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1615                         }
1616                 }
1617                 return $shards;
1618         }
1619
1620         /**
1621          * Get the full container name, including the wiki ID prefix
1622          *
1623          * @param $container string
1624          * @return string
1625          */
1626         final protected function fullContainerName( $container ) {
1627                 if ( $this->wikiId != '' ) {
1628                         return "{$this->wikiId}-$container";
1629                 } else {
1630                         return $container;
1631                 }
1632         }
1633
1634         /**
1635          * Resolve a container name, checking if it's allowed by the backend.
1636          * This is intended for internal use, such as encoding illegal chars.
1637          * Subclasses can override this to be more restrictive.
1638          *
1639          * @param $container string
1640          * @return string|null
1641          */
1642         protected function resolveContainerName( $container ) {
1643                 return $container;
1644         }
1645
1646         /**
1647          * Resolve a relative storage path, checking if it's allowed by the backend.
1648          * This is intended for internal use, such as encoding illegal chars or perhaps
1649          * getting absolute paths (e.g. FS based backends). Note that the relative path
1650          * may be the empty string (e.g. the path is simply to the container).
1651          *
1652          * @param $container string Container name
1653          * @param $relStoragePath string Storage path relative to the container
1654          * @return string|null Path or null if not valid
1655          */
1656         protected function resolveContainerPath( $container, $relStoragePath ) {
1657                 return $relStoragePath;
1658         }
1659 }
1660
1661 /**
1662  * FileBackendStore helper function to handle file listings that span container shards.
1663  * Do not use this class from places outside of FileBackendStore.
1664  *
1665  * @ingroup FileBackend
1666  */
1667 class FileBackendStoreShardListIterator implements Iterator {
1668         /* @var FileBackendStore */
1669         protected $backend;
1670         /* @var Array */
1671         protected $params;
1672         /* @var Array */
1673         protected $shardSuffixes;
1674         protected $container; // string
1675         protected $directory; // string
1676
1677         /* @var Traversable */
1678         protected $iter;
1679         protected $curShard = 0; // integer
1680         protected $pos = 0; // integer
1681
1682         /**
1683          * @param $backend FileBackendStore
1684          * @param $container string Full storage container name
1685          * @param $dir string Storage directory relative to container
1686          * @param $suffixes Array List of container shard suffixes
1687          * @param $params Array
1688          */
1689         public function __construct(
1690                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1691         ) {
1692                 $this->backend = $backend;
1693                 $this->container = $container;
1694                 $this->directory = $dir;
1695                 $this->shardSuffixes = $suffixes;
1696                 $this->params = $params;
1697         }
1698
1699         public function current() {
1700                 if ( is_array( $this->iter ) ) {
1701                         return current( $this->iter );
1702                 } else {
1703                         return $this->iter->current();
1704                 }
1705         }
1706
1707         public function key() {
1708                 return $this->pos;
1709         }
1710
1711         public function next() {
1712                 ++$this->pos;
1713                 if ( is_array( $this->iter ) ) {
1714                         next( $this->iter );
1715                 } else {
1716                         $this->iter->next();
1717                 }
1718                 // Find the next non-empty shard if no elements are left
1719                 $this->nextShardIteratorIfNotValid();
1720         }
1721
1722         /**
1723          * If the iterator for this container shard is out of items,
1724          * then move on to the next container that has items.
1725          * If there are none, then it advances to the last container.
1726          */
1727         protected function nextShardIteratorIfNotValid() {
1728                 while ( !$this->valid() ) {
1729                         if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1730                                 break; // no more container shards
1731                         }
1732                         $this->setIteratorFromCurrentShard();
1733                 }
1734         }
1735
1736         protected function setIteratorFromCurrentShard() {
1737                 $suffix = $this->shardSuffixes[$this->curShard];
1738                 $this->iter = $this->backend->getFileListInternal(
1739                         "{$this->container}{$suffix}", $this->directory, $this->params );
1740         }
1741
1742         public function rewind() {
1743                 $this->pos = 0;
1744                 $this->curShard = 0;
1745                 $this->setIteratorFromCurrentShard();
1746                 // Find the next non-empty shard if this one has no elements
1747                 $this->nextShardIteratorIfNotValid();
1748         }
1749
1750         public function valid() {
1751                 if ( $this->iter == null ) {
1752                         return false; // some failure?
1753                 } elseif ( is_array( $this->iter ) ) {
1754                         return ( current( $this->iter ) !== false ); // no paths can have this value
1755                 } else {
1756                         return $this->iter->valid();
1757                 }
1758         }
1759 }