includes/filerepo/backend/FileBackendStore.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * @brief Base class for all backends using particular storage medium.
  10  *
  11  * This class defines the methods as abstract that subclasses must implement.
  12  * Outside callers should *not* use functions with "Internal" in the name.
  13  *
  14  * The FileBackend operations are implemented using basic functions
  15  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  16  * This class is also responsible for path resolution and sanitization.
  17  *
  18  * @ingroup FileBackend
  19  * @since 1.19
  20  */
  21 abstract class FileBackendStore extends FileBackend {
  22         /** @var BagOStuff */
  23         protected $memCache;
  24
  25         /** @var Array Map of paths to small (RAM/disk) cache items */
  26         protected $cache = array(); // (storage path => key => value)
  27         protected $maxCacheSize = 300; // integer; max paths with entries
  28         /** @var Array Map of paths to large (RAM/disk) cache items */
  29         protected $expensiveCache = array(); // (storage path => key => value)
  30         protected $maxExpensiveCacheSize = 5; // integer; max paths with entries
  31
  32         /** @var Array Map of container names to sharding settings */
  33         protected $shardViaHashLevels = array(); // (container name => config array)
  34
  35         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  36
  37         /**
  38          * @see FileBackend::__construct()
  39          *
  40          * @param $config Array
  41          */
  42         public function __construct( array $config ) {
  43                 parent::__construct( $config );
  44                 $this->memCache = new EmptyBagOStuff(); // disabled by default
  45         }
  46
  47         /**
  48          * Get the maximum allowable file size given backend
  49          * medium restrictions and basic performance constraints.
  50          * Do not call this function from places outside FileBackend and FileOp.
  51          *
  52          * @return integer Bytes
  53          */
  54         final public function maxFileSizeInternal() {
  55                 return $this->maxFileSize;
  56         }
  57
  58         /**
  59          * Check if a file can be created at a given storage path.
  60          * FS backends should check if the parent directory exists and the file is writable.
  61          * Backends using key/value stores should check if the container exists.
  62          *
  63          * @param $storagePath string
  64          * @return bool
  65          */
  66         abstract public function isPathUsableInternal( $storagePath );
  67
  68         /**
  69          * Create a file in the backend with the given contents.
  70          * Do not call this function from places outside FileBackend and FileOp.
  71          *
  72          * $params include:
  73          *     content       : the raw file contents
  74          *     dst           : destination storage path
  75          *     overwrite     : overwrite any file that exists at the destination
  76          *
  77          * @param $params Array
  78          * @return Status
  79          */
  80         final public function createInternal( array $params ) {
  81                 wfProfileIn( __METHOD__ );
  82                 wfProfileIn( __METHOD__ . '-' . $this->name );
  83                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
  84                         $status = Status::newFatal( 'backend-fail-maxsize',
  85                                 $params['dst'], $this->maxFileSizeInternal() );
  86                 } else {
  87                         $status = $this->doCreateInternal( $params );
  88                         $this->clearCache( array( $params['dst'] ) );
  89                         $this->deleteFileCache( $params['dst'] ); // persistent cache
  90                 }
  91                 wfProfileOut( __METHOD__ . '-' . $this->name );
  92                 wfProfileOut( __METHOD__ );
  93                 return $status;
  94         }
  95
  96         /**
  97          * @see FileBackendStore::createInternal()
  98          */
  99         abstract protected function doCreateInternal( array $params );
 100
 101         /**
 102          * Store a file into the backend from a file on disk.
 103          * Do not call this function from places outside FileBackend and FileOp.
 104          *
 105          * $params include:
 106          *     src           : source path on disk
 107          *     dst           : destination storage path
 108          *     overwrite     : overwrite any file that exists at the destination
 109          *
 110          * @param $params Array
 111          * @return Status
 112          */
 113         final public function storeInternal( array $params ) {
 114                 wfProfileIn( __METHOD__ );
 115                 wfProfileIn( __METHOD__ . '-' . $this->name );
 116                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 117                         $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
 118                 } else {
 119                         $status = $this->doStoreInternal( $params );
 120                         $this->clearCache( array( $params['dst'] ) );
 121                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 122                 }
 123                 wfProfileOut( __METHOD__ . '-' . $this->name );
 124                 wfProfileOut( __METHOD__ );
 125                 return $status;
 126         }
 127
 128         /**
 129          * @see FileBackendStore::storeInternal()
 130          */
 131         abstract protected function doStoreInternal( array $params );
 132
 133         /**
 134          * Copy a file from one storage path to another in the backend.
 135          * Do not call this function from places outside FileBackend and FileOp.
 136          *
 137          * $params include:
 138          *     src           : source storage path
 139          *     dst           : destination storage path
 140          *     overwrite     : overwrite any file that exists at the destination
 141          *
 142          * @param $params Array
 143          * @return Status
 144          */
 145         final public function copyInternal( array $params ) {
 146                 wfProfileIn( __METHOD__ );
 147                 wfProfileIn( __METHOD__ . '-' . $this->name );
 148                 $status = $this->doCopyInternal( $params );
 149                 $this->clearCache( array( $params['dst'] ) );
 150                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 151                 wfProfileOut( __METHOD__ . '-' . $this->name );
 152                 wfProfileOut( __METHOD__ );
 153                 return $status;
 154         }
 155
 156         /**
 157          * @see FileBackendStore::copyInternal()
 158          */
 159         abstract protected function doCopyInternal( array $params );
 160
 161         /**
 162          * Delete a file at the storage path.
 163          * Do not call this function from places outside FileBackend and FileOp.
 164          *
 165          * $params include:
 166          *     src                 : source storage path
 167          *     ignoreMissingSource : do nothing if the source file does not exist
 168          *
 169          * @param $params Array
 170          * @return Status
 171          */
 172         final public function deleteInternal( array $params ) {
 173                 wfProfileIn( __METHOD__ );
 174                 wfProfileIn( __METHOD__ . '-' . $this->name );
 175                 $status = $this->doDeleteInternal( $params );
 176                 $this->clearCache( array( $params['src'] ) );
 177                 $this->deleteFileCache( $params['src'] ); // persistent cache
 178                 wfProfileOut( __METHOD__ . '-' . $this->name );
 179                 wfProfileOut( __METHOD__ );
 180                 return $status;
 181         }
 182
 183         /**
 184          * @see FileBackendStore::deleteInternal()
 185          */
 186         abstract protected function doDeleteInternal( array $params );
 187
 188         /**
 189          * Move a file from one storage path to another in the backend.
 190          * Do not call this function from places outside FileBackend and FileOp.
 191          *
 192          * $params include:
 193          *     src           : source storage path
 194          *     dst           : destination storage path
 195          *     overwrite     : overwrite any file that exists at the destination
 196          *
 197          * @param $params Array
 198          * @return Status
 199          */
 200         final public function moveInternal( array $params ) {
 201                 wfProfileIn( __METHOD__ );
 202                 wfProfileIn( __METHOD__ . '-' . $this->name );
 203                 $status = $this->doMoveInternal( $params );
 204                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 205                 $this->deleteFileCache( $params['src'] ); // persistent cache
 206                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 207                 wfProfileOut( __METHOD__ . '-' . $this->name );
 208                 wfProfileOut( __METHOD__ );
 209                 return $status;
 210         }
 211
 212         /**
 213          * @see FileBackendStore::moveInternal()
 214          * @return Status
 215          */
 216         protected function doMoveInternal( array $params ) {
 217                 // Copy source to dest
 218                 $status = $this->copyInternal( $params );
 219                 if ( $status->isOK() ) {
 220                         // Delete source (only fails due to races or medium going down)
 221                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 222                         $status->setResult( true, $status->value ); // ignore delete() errors
 223                 }
 224                 return $status;
 225         }
 226
 227         /**
 228          * @see FileBackend::concatenate()
 229          * @return Status
 230          */
 231         final public function concatenate( array $params ) {
 232                 wfProfileIn( __METHOD__ );
 233                 wfProfileIn( __METHOD__ . '-' . $this->name );
 234                 $status = Status::newGood();
 235
 236                 // Try to lock the source files for the scope of this function
 237                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 238                 if ( $status->isOK() ) {
 239                         // Actually do the concatenation
 240                         $status->merge( $this->doConcatenate( $params ) );
 241                 }
 242
 243                 wfProfileOut( __METHOD__ . '-' . $this->name );
 244                 wfProfileOut( __METHOD__ );
 245                 return $status;
 246         }
 247
 248         /**
 249          * @see FileBackendStore::concatenate()
 250          * @return Status
 251          */
 252         protected function doConcatenate( array $params ) {
 253                 $status = Status::newGood();
 254                 $tmpPath = $params['dst']; // convenience
 255
 256                 // Check that the specified temp file is valid...
 257                 wfSuppressWarnings();
 258                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 259                 wfRestoreWarnings();
 260                 if ( !$ok ) { // not present or not empty
 261                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 262                         return $status;
 263                 }
 264
 265                 // Build up the temp file using the source chunks (in order)...
 266                 $tmpHandle = fopen( $tmpPath, 'ab' );
 267                 if ( $tmpHandle === false ) {
 268                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 269                         return $status;
 270                 }
 271                 foreach ( $params['srcs'] as $virtualSource ) {
 272                         // Get a local FS version of the chunk
 273                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 274                         if ( !$tmpFile ) {
 275                                 $status->fatal( 'backend-fail-read', $virtualSource );
 276                                 return $status;
 277                         }
 278                         // Get a handle to the local FS version
 279                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 280                         if ( $sourceHandle === false ) {
 281                                 fclose( $tmpHandle );
 282                                 $status->fatal( 'backend-fail-read', $virtualSource );
 283                                 return $status;
 284                         }
 285                         // Append chunk to file (pass chunk size to avoid magic quotes)
 286                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 287                                 fclose( $sourceHandle );
 288                                 fclose( $tmpHandle );
 289                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 290                                 return $status;
 291                         }
 292                         fclose( $sourceHandle );
 293                 }
 294                 if ( !fclose( $tmpHandle ) ) {
 295                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 296                         return $status;
 297                 }
 298
 299                 clearstatcache(); // temp file changed
 300
 301                 return $status;
 302         }
 303
 304         /**
 305          * @see FileBackend::doPrepare()
 306          * @return Status
 307          */
 308         final protected function doPrepare( array $params ) {
 309                 wfProfileIn( __METHOD__ );
 310                 wfProfileIn( __METHOD__ . '-' . $this->name );
 311
 312                 $status = Status::newGood();
 313                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 314                 if ( $dir === null ) {
 315                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 316                         wfProfileOut( __METHOD__ . '-' . $this->name );
 317                         wfProfileOut( __METHOD__ );
 318                         return $status; // invalid storage path
 319                 }
 320
 321                 if ( $shard !== null ) { // confined to a single container/shard
 322                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 323                 } else { // directory is on several shards
 324                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 325                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 326                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 327                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 328                         }
 329                 }
 330
 331                 wfProfileOut( __METHOD__ . '-' . $this->name );
 332                 wfProfileOut( __METHOD__ );
 333                 return $status;
 334         }
 335
 336         /**
 337          * @see FileBackendStore::doPrepare()
 338          * @return Status
 339          */
 340         protected function doPrepareInternal( $container, $dir, array $params ) {
 341                 return Status::newGood();
 342         }
 343
 344         /**
 345          * @see FileBackend::doSecure()
 346          * @return Status
 347          */
 348         final protected function doSecure( array $params ) {
 349                 wfProfileIn( __METHOD__ );
 350                 wfProfileIn( __METHOD__ . '-' . $this->name );
 351                 $status = Status::newGood();
 352
 353                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 354                 if ( $dir === null ) {
 355                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 356                         wfProfileOut( __METHOD__ . '-' . $this->name );
 357                         wfProfileOut( __METHOD__ );
 358                         return $status; // invalid storage path
 359                 }
 360
 361                 if ( $shard !== null ) { // confined to a single container/shard
 362                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 363                 } else { // directory is on several shards
 364                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 365                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 366                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 367                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 368                         }
 369                 }
 370
 371                 wfProfileOut( __METHOD__ . '-' . $this->name );
 372                 wfProfileOut( __METHOD__ );
 373                 return $status;
 374         }
 375
 376         /**
 377          * @see FileBackendStore::doSecure()
 378          * @return Status
 379          */
 380         protected function doSecureInternal( $container, $dir, array $params ) {
 381                 return Status::newGood();
 382         }
 383
 384         /**
 385          * @see FileBackend::doClean()
 386          * @return Status
 387          */
 388         final protected function doClean( array $params ) {
 389                 wfProfileIn( __METHOD__ );
 390                 wfProfileIn( __METHOD__ . '-' . $this->name );
 391                 $status = Status::newGood();
 392
 393                 // Recursive: first delete all empty subdirs recursively
 394                 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
 395                         $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
 396                         if ( $subDirsRel !== null ) { // no errors
 397                                 foreach ( $subDirsRel as $subDirRel ) {
 398                                         $subDir = $params['dir'] . "/{$subDirRel}"; // full path
 399                                         $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
 400                                 }
 401                         }
 402                 }
 403
 404                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 405                 if ( $dir === null ) {
 406                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 407                         wfProfileOut( __METHOD__ . '-' . $this->name );
 408                         wfProfileOut( __METHOD__ );
 409                         return $status; // invalid storage path
 410                 }
 411
 412                 // Attempt to lock this directory...
 413                 $filesLockEx = array( $params['dir'] );
 414                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 415                 if ( !$status->isOK() ) {
 416                         wfProfileOut( __METHOD__ . '-' . $this->name );
 417                         wfProfileOut( __METHOD__ );
 418                         return $status; // abort
 419                 }
 420
 421                 if ( $shard !== null ) { // confined to a single container/shard
 422                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 423                         $this->deleteContainerCache( $fullCont ); // purge cache
 424                 } else { // directory is on several shards
 425                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 426                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 427                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 428                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 429                                 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
 430                         }
 431                 }
 432
 433                 wfProfileOut( __METHOD__ . '-' . $this->name );
 434                 wfProfileOut( __METHOD__ );
 435                 return $status;
 436         }
 437
 438         /**
 439          * @see FileBackendStore::doClean()
 440          * @return Status
 441          */
 442         protected function doCleanInternal( $container, $dir, array $params ) {
 443                 return Status::newGood();
 444         }
 445
 446         /**
 447          * @see FileBackend::fileExists()
 448          * @return bool|null
 449          */
 450         final public function fileExists( array $params ) {
 451                 wfProfileIn( __METHOD__ );
 452                 wfProfileIn( __METHOD__ . '-' . $this->name );
 453                 $stat = $this->getFileStat( $params );
 454                 wfProfileOut( __METHOD__ . '-' . $this->name );
 455                 wfProfileOut( __METHOD__ );
 456                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 457         }
 458
 459         /**
 460          * @see FileBackend::getFileTimestamp()
 461          * @return bool
 462          */
 463         final public function getFileTimestamp( array $params ) {
 464                 wfProfileIn( __METHOD__ );
 465                 wfProfileIn( __METHOD__ . '-' . $this->name );
 466                 $stat = $this->getFileStat( $params );
 467                 wfProfileOut( __METHOD__ . '-' . $this->name );
 468                 wfProfileOut( __METHOD__ );
 469                 return $stat ? $stat['mtime'] : false;
 470         }
 471
 472         /**
 473          * @see FileBackend::getFileSize()
 474          * @return bool
 475          */
 476         final public function getFileSize( array $params ) {
 477                 wfProfileIn( __METHOD__ );
 478                 wfProfileIn( __METHOD__ . '-' . $this->name );
 479                 $stat = $this->getFileStat( $params );
 480                 wfProfileOut( __METHOD__ . '-' . $this->name );
 481                 wfProfileOut( __METHOD__ );
 482                 return $stat ? $stat['size'] : false;
 483         }
 484
 485         /**
 486          * @see FileBackend::getFileStat()
 487          * @return bool
 488          */
 489         final public function getFileStat( array $params ) {
 490                 wfProfileIn( __METHOD__ );
 491                 wfProfileIn( __METHOD__ . '-' . $this->name );
 492                 $path = self::normalizeStoragePath( $params['src'] );
 493                 if ( $path === null ) {
 494                         wfProfileOut( __METHOD__ . '-' . $this->name );
 495                         wfProfileOut( __METHOD__ );
 496                         return false; // invalid storage path
 497                 }
 498                 $latest = !empty( $params['latest'] ); // use latest data?
 499                 if ( !isset( $this->cache[$path]['stat'] ) ) {
 500                         $this->primeFileCache( array( $path ) ); // check persistent cache
 501                 }
 502                 if ( isset( $this->cache[$path]['stat'] ) ) {
 503                         // If we want the latest data, check that this cached
 504                         // value was in fact fetched with the latest available data.
 505                         if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
 506                                 $this->pingCache( $path ); // LRU
 507                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 508                                 wfProfileOut( __METHOD__ );
 509                                 return $this->cache[$path]['stat'];
 510                         }
 511                 }
 512                 wfProfileIn( __METHOD__ . '-miss' );
 513                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 514                 $stat = $this->doGetFileStat( $params );
 515                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 516                 wfProfileOut( __METHOD__ . '-miss' );
 517                 if ( is_array( $stat ) ) { // don't cache negatives
 518                         $stat['latest'] = $latest;
 519                         $this->trimCache(); // limit memory
 520                         $this->cache[$path]['stat'] = $stat;
 521                         $this->setFileCache( $path, $stat ); // update persistent cache
 522                 }
 523                 wfProfileOut( __METHOD__ . '-' . $this->name );
 524                 wfProfileOut( __METHOD__ );
 525                 return $stat;
 526         }
 527
 528         /**
 529          * @see FileBackendStore::getFileStat()
 530          */
 531         abstract protected function doGetFileStat( array $params );
 532
 533         /**
 534          * @see FileBackend::getFileContents()
 535          * @return bool|string
 536          */
 537         public function getFileContents( array $params ) {
 538                 wfProfileIn( __METHOD__ );
 539                 wfProfileIn( __METHOD__ . '-' . $this->name );
 540                 $tmpFile = $this->getLocalReference( $params );
 541                 if ( !$tmpFile ) {
 542                         wfProfileOut( __METHOD__ . '-' . $this->name );
 543                         wfProfileOut( __METHOD__ );
 544                         return false;
 545                 }
 546                 wfSuppressWarnings();
 547                 $data = file_get_contents( $tmpFile->getPath() );
 548                 wfRestoreWarnings();
 549                 wfProfileOut( __METHOD__ . '-' . $this->name );
 550                 wfProfileOut( __METHOD__ );
 551                 return $data;
 552         }
 553
 554         /**
 555          * @see FileBackend::getFileSha1Base36()
 556          * @return bool|string
 557          */
 558         final public function getFileSha1Base36( array $params ) {
 559                 wfProfileIn( __METHOD__ );
 560                 wfProfileIn( __METHOD__ . '-' . $this->name );
 561                 $path = $params['src'];
 562                 if ( isset( $this->cache[$path]['sha1'] ) ) {
 563                         $this->pingCache( $path ); // LRU
 564                         wfProfileOut( __METHOD__ . '-' . $this->name );
 565                         wfProfileOut( __METHOD__ );
 566                         return $this->cache[$path]['sha1'];
 567                 }
 568                 wfProfileIn( __METHOD__ . '-miss' );
 569                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 570                 $hash = $this->doGetFileSha1Base36( $params );
 571                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 572                 wfProfileOut( __METHOD__ . '-miss' );
 573                 if ( $hash ) { // don't cache negatives
 574                         $this->trimCache(); // limit memory
 575                         $this->cache[$path]['sha1'] = $hash;
 576                 }
 577                 wfProfileOut( __METHOD__ . '-' . $this->name );
 578                 wfProfileOut( __METHOD__ );
 579                 return $hash;
 580         }
 581
 582         /**
 583          * @see FileBackendStore::getFileSha1Base36()
 584          * @return bool
 585          */
 586         protected function doGetFileSha1Base36( array $params ) {
 587                 $fsFile = $this->getLocalReference( $params );
 588                 if ( !$fsFile ) {
 589                         return false;
 590                 } else {
 591                         return $fsFile->getSha1Base36();
 592                 }
 593         }
 594
 595         /**
 596          * @see FileBackend::getFileProps()
 597          * @return Array
 598          */
 599         final public function getFileProps( array $params ) {
 600                 wfProfileIn( __METHOD__ );
 601                 wfProfileIn( __METHOD__ . '-' . $this->name );
 602                 $fsFile = $this->getLocalReference( $params );
 603                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 604                 wfProfileOut( __METHOD__ . '-' . $this->name );
 605                 wfProfileOut( __METHOD__ );
 606                 return $props;
 607         }
 608
 609         /**
 610          * @see FileBackend::getLocalReference()
 611          * @return TempFSFile|null
 612          */
 613         public function getLocalReference( array $params ) {
 614                 wfProfileIn( __METHOD__ );
 615                 wfProfileIn( __METHOD__ . '-' . $this->name );
 616                 $path = $params['src'];
 617                 if ( isset( $this->expensiveCache[$path]['localRef'] ) ) {
 618                         $this->pingExpensiveCache( $path );
 619                         wfProfileOut( __METHOD__ . '-' . $this->name );
 620                         wfProfileOut( __METHOD__ );
 621                         return $this->expensiveCache[$path]['localRef'];
 622                 }
 623                 $tmpFile = $this->getLocalCopy( $params );
 624                 if ( $tmpFile ) { // don't cache negatives
 625                         $this->trimExpensiveCache(); // limit memory
 626                         $this->expensiveCache[$path]['localRef'] = $tmpFile;
 627                 }
 628                 wfProfileOut( __METHOD__ . '-' . $this->name );
 629                 wfProfileOut( __METHOD__ );
 630                 return $tmpFile;
 631         }
 632
 633         /**
 634          * @see FileBackend::streamFile()
 635          * @return Status
 636          */
 637         final public function streamFile( array $params ) {
 638                 wfProfileIn( __METHOD__ );
 639                 wfProfileIn( __METHOD__ . '-' . $this->name );
 640                 $status = Status::newGood();
 641
 642                 $info = $this->getFileStat( $params );
 643                 if ( !$info ) { // let StreamFile handle the 404
 644                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 645                 }
 646
 647                 // Set output buffer and HTTP headers for stream
 648                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 649                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 650                 if ( $res == StreamFile::NOT_MODIFIED ) {
 651                         // do nothing; client cache is up to date
 652                 } elseif ( $res == StreamFile::READY_STREAM ) {
 653                         wfProfileIn( __METHOD__ . '-send' );
 654                         wfProfileIn( __METHOD__ . '-send-' . $this->name );
 655                         $status = $this->doStreamFile( $params );
 656                         wfProfileOut( __METHOD__ . '-send-' . $this->name );
 657                         wfProfileOut( __METHOD__ . '-send' );
 658                 } else {
 659                         $status->fatal( 'backend-fail-stream', $params['src'] );
 660                 }
 661
 662                 wfProfileOut( __METHOD__ . '-' . $this->name );
 663                 wfProfileOut( __METHOD__ );
 664                 return $status;
 665         }
 666
 667         /**
 668          * @see FileBackendStore::streamFile()
 669          * @return Status
 670          */
 671         protected function doStreamFile( array $params ) {
 672                 $status = Status::newGood();
 673
 674                 $fsFile = $this->getLocalReference( $params );
 675                 if ( !$fsFile ) {
 676                         $status->fatal( 'backend-fail-stream', $params['src'] );
 677                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 678                         $status->fatal( 'backend-fail-stream', $params['src'] );
 679                 }
 680
 681                 return $status;
 682         }
 683
 684         /**
 685          * @see FileBackend::directoryExists()
 686          * @return bool|null
 687          */
 688         final public function directoryExists( array $params ) {
 689                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 690                 if ( $dir === null ) {
 691                         return false; // invalid storage path
 692                 }
 693                 if ( $shard !== null ) { // confined to a single container/shard
 694                         return $this->doDirectoryExists( $fullCont, $dir, $params );
 695                 } else { // directory is on several shards
 696                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 697                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 698                         $res = false; // response
 699                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 700                                 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
 701                                 if ( $exists ) {
 702                                         $res = true;
 703                                         break; // found one!
 704                                 } elseif ( $exists === null ) { // error?
 705                                         $res = null; // if we don't find anything, it is indeterminate
 706                                 }
 707                         }
 708                         return $res;
 709                 }
 710         }
 711
 712         /**
 713          * @see FileBackendStore::directoryExists()
 714          *
 715          * @param $container string Resolved container name
 716          * @param $dir string Resolved path relative to container
 717          * @param $params Array
 718          * @return bool|null
 719          */
 720         abstract protected function doDirectoryExists( $container, $dir, array $params );
 721
 722         /**
 723          * @see FileBackend::getDirectoryList()
 724          * @return Traversable|Array|null Returns null on failure
 725          */
 726         final public function getDirectoryList( array $params ) {
 727                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 728                 if ( $dir === null ) { // invalid storage path
 729                         return null;
 730                 }
 731                 if ( $shard !== null ) {
 732                         // File listing is confined to a single container/shard
 733                         return $this->getDirectoryListInternal( $fullCont, $dir, $params );
 734                 } else {
 735                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 736                         // File listing spans multiple containers/shards
 737                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 738                         return new FileBackendStoreShardDirIterator( $this,
 739                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 740                 }
 741         }
 742
 743         /**
 744          * Do not call this function from places outside FileBackend
 745          *
 746          * @see FileBackendStore::getDirectoryList()
 747          *
 748          * @param $container string Resolved container name
 749          * @param $dir string Resolved path relative to container
 750          * @param $params Array
 751          * @return Traversable|Array|null Returns null on failure
 752          */
 753         abstract public function getDirectoryListInternal( $container, $dir, array $params );
 754
 755         /**
 756          * @see FileBackend::getFileList()
 757          * @return Traversable|Array|null Returns null on failure
 758          */
 759         final public function getFileList( array $params ) {
 760                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 761                 if ( $dir === null ) { // invalid storage path
 762                         return null;
 763                 }
 764                 if ( $shard !== null ) {
 765                         // File listing is confined to a single container/shard
 766                         return $this->getFileListInternal( $fullCont, $dir, $params );
 767                 } else {
 768                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 769                         // File listing spans multiple containers/shards
 770                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 771                         return new FileBackendStoreShardFileIterator( $this,
 772                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 773                 }
 774         }
 775
 776         /**
 777          * Do not call this function from places outside FileBackend
 778          *
 779          * @see FileBackendStore::getFileList()
 780          *
 781          * @param $container string Resolved container name
 782          * @param $dir string Resolved path relative to container
 783          * @param $params Array
 784          * @return Traversable|Array|null Returns null on failure
 785          */
 786         abstract public function getFileListInternal( $container, $dir, array $params );
 787
 788         /**
 789          * Get the list of supported operations and their corresponding FileOp classes.
 790          *
 791          * @return Array
 792          */
 793         protected function supportedOperations() {
 794                 return array(
 795                         'store'       => 'StoreFileOp',
 796                         'copy'        => 'CopyFileOp',
 797                         'move'        => 'MoveFileOp',
 798                         'delete'      => 'DeleteFileOp',
 799                         'create'      => 'CreateFileOp',
 800                         'null'        => 'NullFileOp'
 801                 );
 802         }
 803
 804         /**
 805          * Return a list of FileOp objects from a list of operations.
 806          * Do not call this function from places outside FileBackend.
 807          *
 808          * The result must have the same number of items as the input.
 809          * An exception is thrown if an unsupported operation is requested.
 810          *
 811          * @param $ops Array Same format as doOperations()
 812          * @return Array List of FileOp objects
 813          * @throws MWException
 814          */
 815         final public function getOperationsInternal( array $ops ) {
 816                 $supportedOps = $this->supportedOperations();
 817
 818                 $performOps = array(); // array of FileOp objects
 819                 // Build up ordered array of FileOps...
 820                 foreach ( $ops as $operation ) {
 821                         $opName = $operation['op'];
 822                         if ( isset( $supportedOps[$opName] ) ) {
 823                                 $class = $supportedOps[$opName];
 824                                 // Get params for this operation
 825                                 $params = $operation;
 826                                 // Append the FileOp class
 827                                 $performOps[] = new $class( $this, $params );
 828                         } else {
 829                                 throw new MWException( "Operation '$opName' is not supported." );
 830                         }
 831                 }
 832
 833                 return $performOps;
 834         }
 835
 836         /**
 837          * Get a list of storage paths to lock for a list of operations
 838          * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
 839          * each corresponding to a list of storage paths to be locked.
 840          *
 841          * @param $performOps Array List of FileOp objects
 842          * @return Array ('sh' => list of paths, 'ex' => list of paths)
 843          */
 844         final public function getPathsToLockForOpsInternal( array $performOps ) {
 845                 // Build up a list of files to lock...
 846                 $paths = array( 'sh' => array(), 'ex' => array() );
 847                 foreach ( $performOps as $fileOp ) {
 848                         $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
 849                         $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
 850                 }
 851                 // Optimization: if doing an EX lock anyway, don't also set an SH one
 852                 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
 853                 // Get a shared lock on the parent directory of each path changed
 854                 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
 855
 856                 return $paths;
 857         }
 858
 859         /**
 860          * @see FileBackend::doOperationsInternal()
 861          * @return Status
 862          */
 863         protected function doOperationsInternal( array $ops, array $opts ) {
 864                 wfProfileIn( __METHOD__ );
 865                 wfProfileIn( __METHOD__ . '-' . $this->name );
 866                 $status = Status::newGood();
 867
 868                 // Build up a list of FileOps...
 869                 $performOps = $this->getOperationsInternal( $ops );
 870
 871                 // Acquire any locks as needed...
 872                 if ( empty( $opts['nonLocking'] ) ) {
 873                         // Build up a list of files to lock...
 874                         $paths = $this->getPathsToLockForOpsInternal( $performOps );
 875                         // Try to lock those files for the scope of this function...
 876                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 877                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 878                         if ( !$status->isOK() ) {
 879                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 880                                 wfProfileOut( __METHOD__ );
 881                                 return $status; // abort
 882                         }
 883                 }
 884
 885                 // Clear any file cache entries (after locks acquired)
 886                 $this->clearCache();
 887
 888                 // Load from the persistent file and container caches
 889                 $this->primeFileCache( $performOps );
 890                 $this->primeContainerCache( $performOps );
 891
 892                 // Actually attempt the operation batch...
 893                 $subStatus = FileOp::attemptBatch( $performOps, $opts, $this->fileJournal );
 894
 895                 // Merge errors into status fields
 896                 $status->merge( $subStatus );
 897                 $status->success = $subStatus->success; // not done in merge()
 898
 899                 wfProfileOut( __METHOD__ . '-' . $this->name );
 900                 wfProfileOut( __METHOD__ );
 901                 return $status;
 902         }
 903
 904         /**
 905          * @see FileBackend::clearCache()
 906          */
 907         final public function clearCache( array $paths = null ) {
 908                 if ( is_array( $paths ) ) {
 909                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
 910                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
 911                 }
 912                 if ( $paths === null ) {
 913                         $this->cache = array();
 914                         $this->expensiveCache = array();
 915                 } else {
 916                         foreach ( $paths as $path ) {
 917                                 unset( $this->cache[$path] );
 918                                 unset( $this->expensiveCache[$path] );
 919                         }
 920                 }
 921                 $this->doClearCache( $paths );
 922         }
 923
 924         /**
 925          * Clears any additional stat caches for storage paths
 926          *
 927          * @see FileBackend::clearCache()
 928          *
 929          * @param $paths Array Storage paths (optional)
 930          * @return void
 931          */
 932         protected function doClearCache( array $paths = null ) {}
 933
 934         /**
 935          * Is this a key/value store where directories are just virtual?
 936          * Virtual directories exists in so much as files exists that are
 937          * prefixed with the directory path followed by a forward slash.
 938          *
 939          * @return bool
 940          */
 941         abstract protected function directoriesAreVirtual();
 942
 943         /**
 944          * Move a cache entry to the top (such as when accessed)
 945          *
 946          * @param $path string Storage path
 947          * @return void
 948          */
 949         protected function pingCache( $path ) {
 950                 if ( isset( $this->cache[$path] ) ) {
 951                         $tmp = $this->cache[$path];
 952                         unset( $this->cache[$path] );
 953                         $this->cache[$path] = $tmp;
 954                 }
 955         }
 956
 957         /**
 958          * Prune the inexpensive cache if it is too big to add an item
 959          *
 960          * @return void
 961          */
 962         protected function trimCache() {
 963                 if ( count( $this->cache ) >= $this->maxCacheSize ) {
 964                         reset( $this->cache );
 965                         unset( $this->cache[key( $this->cache )] );
 966                 }
 967         }
 968
 969         /**
 970          * Move a cache entry to the top (such as when accessed)
 971          *
 972          * @param $path string Storage path
 973          * @return void
 974          */
 975         protected function pingExpensiveCache( $path ) {
 976                 if ( isset( $this->expensiveCache[$path] ) ) {
 977                         $tmp = $this->expensiveCache[$path];
 978                         unset( $this->expensiveCache[$path] );
 979                         $this->expensiveCache[$path] = $tmp;
 980                 }
 981         }
 982
 983         /**
 984          * Prune the expensive cache if it is too big to add an item
 985          *
 986          * @return void
 987          */
 988         protected function trimExpensiveCache() {
 989                 if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) {
 990                         reset( $this->expensiveCache );
 991                         unset( $this->expensiveCache[key( $this->expensiveCache )] );
 992                 }
 993         }
 994
 995         /**
 996          * Check if a container name is valid.
 997          * This checks for for length and illegal characters.
 998          *
 999          * @param $container string
1000          * @return bool
1001          */
1002         final protected static function isValidContainerName( $container ) {
1003                 // This accounts for Swift and S3 restrictions while leaving room
1004                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1005                 // This disallows directory separators or traversal characters.
1006                 // Note that matching strings URL encode to the same string;
1007                 // in Swift, the length restriction is *after* URL encoding.
1008                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1009         }
1010
1011         /**
1012          * Splits a storage path into an internal container name,
1013          * an internal relative file name, and a container shard suffix.
1014          * Any shard suffix is already appended to the internal container name.
1015          * This also checks that the storage path is valid and within this backend.
1016          *
1017          * If the container is sharded but a suffix could not be determined,
1018          * this means that the path can only refer to a directory and can only
1019          * be scanned by looking in all the container shards.
1020          *
1021          * @param $storagePath string
1022          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1023          */
1024         final protected function resolveStoragePath( $storagePath ) {
1025                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1026                 if ( $backend === $this->name ) { // must be for this backend
1027                         $relPath = self::normalizeContainerPath( $relPath );
1028                         if ( $relPath !== null ) {
1029                                 // Get shard for the normalized path if this container is sharded
1030                                 $cShard = $this->getContainerShard( $container, $relPath );
1031                                 // Validate and sanitize the relative path (backend-specific)
1032                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1033                                 if ( $relPath !== null ) {
1034                                         // Prepend any wiki ID prefix to the container name
1035                                         $container = $this->fullContainerName( $container );
1036                                         if ( self::isValidContainerName( $container ) ) {
1037                                                 // Validate and sanitize the container name (backend-specific)
1038                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1039                                                 if ( $container !== null ) {
1040                                                         return array( $container, $relPath, $cShard );
1041                                                 }
1042                                         }
1043                                 }
1044                         }
1045                 }
1046                 return array( null, null, null );
1047         }
1048
1049         /**
1050          * Like resolveStoragePath() except null values are returned if
1051          * the container is sharded and the shard could not be determined.
1052          *
1053          * @see FileBackendStore::resolveStoragePath()
1054          *
1055          * @param $storagePath string
1056          * @return Array (container, path) or (null, null) if invalid
1057          */
1058         final protected function resolveStoragePathReal( $storagePath ) {
1059                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1060                 if ( $cShard !== null ) {
1061                         return array( $container, $relPath );
1062                 }
1063                 return array( null, null );
1064         }
1065
1066         /**
1067          * Get the container name shard suffix for a given path.
1068          * Any empty suffix means the container is not sharded.
1069          *
1070          * @param $container string Container name
1071          * @param $relStoragePath string Storage path relative to the container
1072          * @return string|null Returns null if shard could not be determined
1073          */
1074         final protected function getContainerShard( $container, $relPath ) {
1075                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1076                 if ( $levels == 1 || $levels == 2 ) {
1077                         // Hash characters are either base 16 or 36
1078                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1079                         // Get a regex that represents the shard portion of paths.
1080                         // The concatenation of the captures gives us the shard.
1081                         if ( $levels === 1 ) { // 16 or 36 shards per container
1082                                 $hashDirRegex = '(' . $char . ')';
1083                         } else { // 256 or 1296 shards per container
1084                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1085                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1086                                 } else { // short hash dir format (e.g. "a/b/c")
1087                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1088                                 }
1089                         }
1090                         // Allow certain directories to be above the hash dirs so as
1091                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1092                         // They must be 2+ chars to avoid any hash directory ambiguity.
1093                         $m = array();
1094                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1095                                 return '.' . implode( '', array_slice( $m, 1 ) );
1096                         }
1097                         return null; // failed to match
1098                 }
1099                 return ''; // no sharding
1100         }
1101
1102         /**
1103          * Check if a storage path maps to a single shard.
1104          * Container dirs like "a", where the container shards on "x/xy",
1105          * can reside on several shards. Such paths are tricky to handle.
1106          *
1107          * @param $storagePath string Storage path
1108          * @return bool
1109          */
1110         final public function isSingleShardPathInternal( $storagePath ) {
1111                 list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
1112                 return ( $shard !== null );
1113         }
1114
1115         /**
1116          * Get the sharding config for a container.
1117          * If greater than 0, then all file storage paths within
1118          * the container are required to be hashed accordingly.
1119          *
1120          * @param $container string
1121          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1122          */
1123         final protected function getContainerHashLevels( $container ) {
1124                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1125                         $config = $this->shardViaHashLevels[$container];
1126                         $hashLevels = (int)$config['levels'];
1127                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1128                                 $hashBase = (int)$config['base'];
1129                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1130                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1131                                 }
1132                         }
1133                 }
1134                 return array( 0, 0, false ); // no sharding
1135         }
1136
1137         /**
1138          * Get a list of full container shard suffixes for a container
1139          *
1140          * @param $container string
1141          * @return Array
1142          */
1143         final protected function getContainerSuffixes( $container ) {
1144                 $shards = array();
1145                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1146                 if ( $digits > 0 ) {
1147                         $numShards = pow( $base, $digits );
1148                         for ( $index = 0; $index < $numShards; $index++ ) {
1149                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1150                         }
1151                 }
1152                 return $shards;
1153         }
1154
1155         /**
1156          * Get the full container name, including the wiki ID prefix
1157          *
1158          * @param $container string
1159          * @return string
1160          */
1161         final protected function fullContainerName( $container ) {
1162                 if ( $this->wikiId != '' ) {
1163                         return "{$this->wikiId}-$container";
1164                 } else {
1165                         return $container;
1166                 }
1167         }
1168
1169         /**
1170          * Resolve a container name, checking if it's allowed by the backend.
1171          * This is intended for internal use, such as encoding illegal chars.
1172          * Subclasses can override this to be more restrictive.
1173          *
1174          * @param $container string
1175          * @return string|null
1176          */
1177         protected function resolveContainerName( $container ) {
1178                 return $container;
1179         }
1180
1181         /**
1182          * Resolve a relative storage path, checking if it's allowed by the backend.
1183          * This is intended for internal use, such as encoding illegal chars or perhaps
1184          * getting absolute paths (e.g. FS based backends). Note that the relative path
1185          * may be the empty string (e.g. the path is simply to the container).
1186          *
1187          * @param $container string Container name
1188          * @param $relStoragePath string Storage path relative to the container
1189          * @return string|null Path or null if not valid
1190          */
1191         protected function resolveContainerPath( $container, $relStoragePath ) {
1192                 return $relStoragePath;
1193         }
1194
1195         /**
1196          * Get the cache key for a container
1197          *
1198          * @param $container Resolved container name
1199          * @return string
1200          */
1201         private function containerCacheKey( $container ) {
1202                 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1203         }
1204
1205         /**
1206          * Set the cached info for a container
1207          *
1208          * @param $container Resolved container name
1209          * @param $val mixed Information to cache
1210          * @return void
1211          */
1212         final protected function setContainerCache( $container, $val ) {
1213                 $this->memCache->set( $this->containerCacheKey( $container ), $val, 14*86400 );
1214         }
1215
1216         /**
1217          * Delete the cached info for a container
1218          *
1219          * @param $container Resolved container name
1220          * @return void
1221          */
1222         final protected function deleteContainerCache( $container ) {
1223                 for ( $attempts=1; $attempts <= 3; $attempts++ ) {
1224                         if ( $this->memCache->delete( $this->containerCacheKey( $container ) ) ) {
1225                                 return; // done!
1226                         }
1227                 }
1228                 trigger_error( "Unable to delete stat cache for container $container." );
1229         }
1230
1231         /**
1232          * Do a batch lookup from cache for container stats for all containers
1233          * used in a list of container names, storage paths, or FileOp objects.
1234          *
1235          * @param $items Array
1236          * @return void
1237          */
1238         final protected function primeContainerCache( array $items ) {
1239                 wfProfileIn( __METHOD__ );
1240                 wfProfileIn( __METHOD__ . '-' . $this->name );
1241                 $paths = array(); // list of storage paths
1242                 $contNames = array(); // (cache key => resolved container name)
1243                 // Get all the paths/containers from the items...
1244                 foreach ( $items as $item ) {
1245                         if ( $item instanceof FileOp ) {
1246                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1247                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1248                         } elseif ( self::isStoragePath( $item ) ) {
1249                                 $paths[] = $item;
1250                         } elseif ( is_string( $item ) ) { // full container name
1251                                 $contNames[$this->containerCacheKey( $item )] = $item;
1252                         }
1253                 }
1254                 // Get all the corresponding cache keys for paths...
1255                 foreach ( $paths as $path ) {
1256                         list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
1257                         if ( $fullCont !== null ) { // valid path for this backend
1258                                 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1259                         }
1260                 }
1261
1262                 $contInfo = array(); // (resolved container name => cache value)
1263                 // Get all cache entries for these container cache keys...
1264                 $values = $this->memCache->getBatch( array_keys( $contNames ) );
1265                 foreach ( $values as $cacheKey => $val ) {
1266                         $contInfo[$contNames[$cacheKey]] = $val;
1267                 }
1268
1269                 // Populate the container process cache for the backend...
1270                 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1271                 wfProfileOut( __METHOD__ . '-' . $this->name );
1272                 wfProfileOut( __METHOD__ );
1273         }
1274
1275         /**
1276          * Fill the backend-specific process cache given an array of
1277          * resolved container names and their corresponding cached info.
1278          * Only containers that actually exist should appear in the map.
1279          *
1280          * @param $containerInfo Array Map of resolved container names to cached info
1281          * @return void
1282          */
1283         protected function doPrimeContainerCache( array $containerInfo ) {}
1284
1285         /**
1286          * Get the cache key for a file path
1287          *
1288          * @param $path Storage path
1289          * @return string
1290          */
1291         private function fileCacheKey( $path ) {
1292                 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1293         }
1294
1295         /**
1296          * Set the cached stat info for a file path
1297          *
1298          * @param $path Storage path
1299          * @param $val mixed Information to cache
1300          * @return void
1301          */
1302         final protected function setFileCache( $path, $val ) {
1303                 $this->memCache->set( $this->fileCacheKey( $path ), $val, 7*86400 );
1304         }
1305
1306         /**
1307          * Delete the cached stat info for a file path
1308          *
1309          * @param $path Storage path
1310          * @return void
1311          */
1312         final protected function deleteFileCache( $path ) {
1313                 for ( $attempts=1; $attempts <= 3; $attempts++ ) {
1314                         if ( $this->memCache->delete( $this->fileCacheKey( $path ) ) ) {
1315                                 return; // done!
1316                         }
1317                 }
1318                 trigger_error( "Unable to delete stat cache for file $path." );
1319         }
1320
1321         /**
1322          * Do a batch lookup from cache for file stats for all paths
1323          * used in a list of storage paths or FileOp objects.
1324          *
1325          * @param $items Array List of storage paths or FileOps
1326          * @return void
1327          */
1328         final protected function primeFileCache( array $items ) {
1329                 wfProfileIn( __METHOD__ );
1330                 wfProfileIn( __METHOD__ . '-' . $this->name );
1331                 $paths = array(); // list of storage paths
1332                 $pathNames = array(); // (cache key => storage path)
1333                 // Get all the paths/containers from the items...
1334                 foreach ( $items as $item ) {
1335                         if ( $item instanceof FileOp ) {
1336                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1337                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1338                         } elseif ( self::isStoragePath( $item ) ) {
1339                                 $paths[] = $item;
1340                         }
1341                 }
1342                 // Get all the corresponding cache keys for paths...
1343                 foreach ( $paths as $path ) {
1344                         list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
1345                         if ( $rel !== null ) { // valid path for this backend
1346                                 $pathNames[$this->fileCacheKey( $path )] = $path;
1347                         }
1348                 }
1349                 // Get all cache entries for these container cache keys...
1350                 $values = $this->memCache->getBatch( array_keys( $pathNames ) );
1351                 foreach ( $values as $cacheKey => $val ) {
1352                         if ( is_array( $val ) ) {
1353                                 $this->trimCache(); // limit memory
1354                                 $this->cache[$pathNames[$cacheKey]]['stat'] = $val;
1355                         }
1356                 }
1357                 wfProfileOut( __METHOD__ . '-' . $this->name );
1358                 wfProfileOut( __METHOD__ );
1359         }
1360 }
1361
1362 /**
1363  * FileBackendStore helper function to handle listings that span container shards.
1364  * Do not use this class from places outside of FileBackendStore.
1365  *
1366  * @ingroup FileBackend
1367  */
1368 abstract class FileBackendStoreShardListIterator implements Iterator {
1369         /** @var FileBackendStore */
1370         protected $backend;
1371         /** @var Array */
1372         protected $params;
1373         /** @var Array */
1374         protected $shardSuffixes;
1375         protected $container; // string; full container name
1376         protected $directory; // string; resolved relative path
1377
1378         /** @var Traversable */
1379         protected $iter;
1380         protected $curShard = 0; // integer
1381         protected $pos = 0; // integer
1382
1383         /** @var Array */
1384         protected $multiShardPaths = array(); // (rel path => 1)
1385
1386         /**
1387          * @param $backend FileBackendStore
1388          * @param $container string Full storage container name
1389          * @param $dir string Storage directory relative to container
1390          * @param $suffixes Array List of container shard suffixes
1391          * @param $params Array
1392          */
1393         public function __construct(
1394                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1395         ) {
1396                 $this->backend = $backend;
1397                 $this->container = $container;
1398                 $this->directory = $dir;
1399                 $this->shardSuffixes = $suffixes;
1400                 $this->params = $params;
1401         }
1402
1403         /**
1404          * @see Iterator::current()
1405          * @return string|bool String or false
1406          */
1407         public function current() {
1408                 if ( is_array( $this->iter ) ) {
1409                         return current( $this->iter );
1410                 } else {
1411                         return $this->iter->current();
1412                 }
1413         }
1414
1415         /**
1416          * @see Iterator::key()
1417          * @return integer
1418          */
1419         public function key() {
1420                 return $this->pos;
1421         }
1422
1423         /**
1424          * @see Iterator::next()
1425          * @return void
1426          */
1427         public function next() {
1428                 ++$this->pos;
1429                 if ( is_array( $this->iter ) ) {
1430                         next( $this->iter );
1431                 } else {
1432                         $this->iter->next();
1433                 }
1434                 // Filter out items that we already listed
1435                 $this->filterViaNext();
1436                 // Find the next non-empty shard if no elements are left
1437                 $this->nextShardIteratorIfNotValid();
1438         }
1439
1440         /**
1441          * @see Iterator::rewind()
1442          * @return void
1443          */
1444         public function rewind() {
1445                 $this->pos = 0;
1446                 $this->curShard = 0;
1447                 $this->setIteratorFromCurrentShard();
1448                 // Filter out items that we already listed
1449                 $this->filterViaNext();
1450                 // Find the next non-empty shard if this one has no elements
1451                 $this->nextShardIteratorIfNotValid();
1452         }
1453
1454         /**
1455          * @see Iterator::valid()
1456          * @return bool
1457          */
1458         public function valid() {
1459                 if ( $this->iter === null ) {
1460                         return false; // some failure?
1461                 } elseif ( is_array( $this->iter ) ) {
1462                         return ( current( $this->iter ) !== false ); // no paths can have this value
1463                 } else {
1464                         return $this->iter->valid();
1465                 }
1466         }
1467
1468         /**
1469          * Filter out duplicate items by advancing to the next ones
1470          */
1471         protected function filterViaNext() {
1472                 while ( $this->iter->valid() ) {
1473                         $rel = $this->iter->current(); // path relative to given directory
1474                         $path = $this->params['dir'] . "/{$rel}"; // full storage path
1475                         if ( !$this->backend->isSingleShardPathInternal( $path ) ) {
1476                                 // Don't keep listing paths that are on multiple shards
1477                                 if ( isset( $this->multiShardPaths[$rel] ) ) {
1478                                         $this->iter->next(); // we already listed this path
1479                                 } else {
1480                                         $this->multiShardPaths[$rel] = 1;
1481                                         break;
1482                                 }
1483                         }
1484                 }
1485         }
1486
1487         /**
1488          * If the list iterator for this container shard is out of items,
1489          * then move on to the next container that has items.
1490          * If there are none, then it advances to the last container.
1491          */
1492         protected function nextShardIteratorIfNotValid() {
1493                 while ( !$this->valid() ) {
1494                         if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1495                                 break; // no more container shards
1496                         }
1497                         $this->setIteratorFromCurrentShard();
1498                 }
1499         }
1500
1501         /**
1502          * Set the list iterator to that of the current container shard
1503          */
1504         protected function setIteratorFromCurrentShard() {
1505                 $suffix = $this->shardSuffixes[$this->curShard];
1506                 $this->iter = $this->listFromShard(
1507                         "{$this->container}{$suffix}", $this->directory, $this->params );
1508         }
1509
1510         /**
1511          * Get the list for a given container shard
1512          *
1513          * @param $container string Resolved container name
1514          * @param $dir string Resolved path relative to container
1515          * @param $params Array
1516          * @return Traversable|Array|null
1517          */
1518         abstract protected function listFromShard( $container, $dir, array $params );
1519 }
1520
1521 /**
1522  * Iterator for listing directories
1523  */
1524 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1525         protected function listFromShard( $container, $dir, array $params ) {
1526                 return $this->backend->getDirectoryListInternal( $container, $dir, $params );
1527         }
1528 }
1529
1530 /**
1531  * Iterator for listing regular files
1532  */
1533 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1534         protected function listFromShard( $container, $dir, array $params ) {
1535                 return $this->backend->getFileListInternal( $container, $dir, $params );
1536         }
1537 }