includes/filerepo/backend/FileBackendStore.php

   1 <?php
   2 /**
   3  * Base class for all backends using particular storage medium.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Base class for all backends using particular storage medium.
  27  *
  28  * This class defines the methods as abstract that subclasses must implement.
  29  * Outside callers should *not* use functions with "Internal" in the name.
  30  *
  31  * The FileBackend operations are implemented using basic functions
  32  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  33  * This class is also responsible for path resolution and sanitization.
  34  *
  35  * @ingroup FileBackend
  36  * @since 1.19
  37  */
  38 abstract class FileBackendStore extends FileBackend {
  39         /** @var BagOStuff */
  40         protected $memCache;
  41
  42         /** @var Array Map of paths to small (RAM/disk) cache items */
  43         protected $cache = array(); // (storage path => key => value)
  44         protected $maxCacheSize = 300; // integer; max paths with entries
  45         /** @var Array Map of paths to large (RAM/disk) cache items */
  46         protected $expensiveCache = array(); // (storage path => key => value)
  47         protected $maxExpensiveCacheSize = 5; // integer; max paths with entries
  48
  49         /** @var Array Map of container names to sharding settings */
  50         protected $shardViaHashLevels = array(); // (container name => config array)
  51
  52         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  53
  54         /**
  55          * @see FileBackend::__construct()
  56          *
  57          * @param $config Array
  58          */
  59         public function __construct( array $config ) {
  60                 parent::__construct( $config );
  61                 $this->memCache = new EmptyBagOStuff(); // disabled by default
  62         }
  63
  64         /**
  65          * Get the maximum allowable file size given backend
  66          * medium restrictions and basic performance constraints.
  67          * Do not call this function from places outside FileBackend and FileOp.
  68          *
  69          * @return integer Bytes
  70          */
  71         final public function maxFileSizeInternal() {
  72                 return $this->maxFileSize;
  73         }
  74
  75         /**
  76          * Check if a file can be created at a given storage path.
  77          * FS backends should check if the parent directory exists and the file is writable.
  78          * Backends using key/value stores should check if the container exists.
  79          *
  80          * @param $storagePath string
  81          * @return bool
  82          */
  83         abstract public function isPathUsableInternal( $storagePath );
  84
  85         /**
  86          * Create a file in the backend with the given contents.
  87          * Do not call this function from places outside FileBackend and FileOp.
  88          *
  89          * $params include:
  90          *     content       : the raw file contents
  91          *     dst           : destination storage path
  92          *     overwrite     : overwrite any file that exists at the destination
  93          *
  94          * @param $params Array
  95          * @return Status
  96          */
  97         final public function createInternal( array $params ) {
  98                 wfProfileIn( __METHOD__ );
  99                 wfProfileIn( __METHOD__ . '-' . $this->name );
 100                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 101                         $status = Status::newFatal( 'backend-fail-maxsize',
 102                                 $params['dst'], $this->maxFileSizeInternal() );
 103                 } else {
 104                         $status = $this->doCreateInternal( $params );
 105                         $this->clearCache( array( $params['dst'] ) );
 106                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 107                 }
 108                 wfProfileOut( __METHOD__ . '-' . $this->name );
 109                 wfProfileOut( __METHOD__ );
 110                 return $status;
 111         }
 112
 113         /**
 114          * @see FileBackendStore::createInternal()
 115          */
 116         abstract protected function doCreateInternal( array $params );
 117
 118         /**
 119          * Store a file into the backend from a file on disk.
 120          * Do not call this function from places outside FileBackend and FileOp.
 121          *
 122          * $params include:
 123          *     src           : source path on disk
 124          *     dst           : destination storage path
 125          *     overwrite     : overwrite any file that exists at the destination
 126          *
 127          * @param $params Array
 128          * @return Status
 129          */
 130         final public function storeInternal( array $params ) {
 131                 wfProfileIn( __METHOD__ );
 132                 wfProfileIn( __METHOD__ . '-' . $this->name );
 133                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 134                         $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
 135                 } else {
 136                         $status = $this->doStoreInternal( $params );
 137                         $this->clearCache( array( $params['dst'] ) );
 138                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 139                 }
 140                 wfProfileOut( __METHOD__ . '-' . $this->name );
 141                 wfProfileOut( __METHOD__ );
 142                 return $status;
 143         }
 144
 145         /**
 146          * @see FileBackendStore::storeInternal()
 147          */
 148         abstract protected function doStoreInternal( array $params );
 149
 150         /**
 151          * Copy a file from one storage path to another in the backend.
 152          * Do not call this function from places outside FileBackend and FileOp.
 153          *
 154          * $params include:
 155          *     src           : source storage path
 156          *     dst           : destination storage path
 157          *     overwrite     : overwrite any file that exists at the destination
 158          *
 159          * @param $params Array
 160          * @return Status
 161          */
 162         final public function copyInternal( array $params ) {
 163                 wfProfileIn( __METHOD__ );
 164                 wfProfileIn( __METHOD__ . '-' . $this->name );
 165                 $status = $this->doCopyInternal( $params );
 166                 $this->clearCache( array( $params['dst'] ) );
 167                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 168                 wfProfileOut( __METHOD__ . '-' . $this->name );
 169                 wfProfileOut( __METHOD__ );
 170                 return $status;
 171         }
 172
 173         /**
 174          * @see FileBackendStore::copyInternal()
 175          */
 176         abstract protected function doCopyInternal( array $params );
 177
 178         /**
 179          * Delete a file at the storage path.
 180          * Do not call this function from places outside FileBackend and FileOp.
 181          *
 182          * $params include:
 183          *     src                 : source storage path
 184          *     ignoreMissingSource : do nothing if the source file does not exist
 185          *
 186          * @param $params Array
 187          * @return Status
 188          */
 189         final public function deleteInternal( array $params ) {
 190                 wfProfileIn( __METHOD__ );
 191                 wfProfileIn( __METHOD__ . '-' . $this->name );
 192                 $status = $this->doDeleteInternal( $params );
 193                 $this->clearCache( array( $params['src'] ) );
 194                 $this->deleteFileCache( $params['src'] ); // persistent cache
 195                 wfProfileOut( __METHOD__ . '-' . $this->name );
 196                 wfProfileOut( __METHOD__ );
 197                 return $status;
 198         }
 199
 200         /**
 201          * @see FileBackendStore::deleteInternal()
 202          */
 203         abstract protected function doDeleteInternal( array $params );
 204
 205         /**
 206          * Move a file from one storage path to another in the backend.
 207          * Do not call this function from places outside FileBackend and FileOp.
 208          *
 209          * $params include:
 210          *     src           : source storage path
 211          *     dst           : destination storage path
 212          *     overwrite     : overwrite any file that exists at the destination
 213          *
 214          * @param $params Array
 215          * @return Status
 216          */
 217         final public function moveInternal( array $params ) {
 218                 wfProfileIn( __METHOD__ );
 219                 wfProfileIn( __METHOD__ . '-' . $this->name );
 220                 $status = $this->doMoveInternal( $params );
 221                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 222                 $this->deleteFileCache( $params['src'] ); // persistent cache
 223                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 224                 wfProfileOut( __METHOD__ . '-' . $this->name );
 225                 wfProfileOut( __METHOD__ );
 226                 return $status;
 227         }
 228
 229         /**
 230          * @see FileBackendStore::moveInternal()
 231          * @return Status
 232          */
 233         protected function doMoveInternal( array $params ) {
 234                 // Copy source to dest
 235                 $status = $this->copyInternal( $params );
 236                 if ( $status->isOK() ) {
 237                         // Delete source (only fails due to races or medium going down)
 238                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 239                         $status->setResult( true, $status->value ); // ignore delete() errors
 240                 }
 241                 return $status;
 242         }
 243
 244         /**
 245          * @see FileBackend::concatenate()
 246          * @return Status
 247          */
 248         final public function concatenate( array $params ) {
 249                 wfProfileIn( __METHOD__ );
 250                 wfProfileIn( __METHOD__ . '-' . $this->name );
 251                 $status = Status::newGood();
 252
 253                 // Try to lock the source files for the scope of this function
 254                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 255                 if ( $status->isOK() ) {
 256                         // Actually do the concatenation
 257                         $status->merge( $this->doConcatenate( $params ) );
 258                 }
 259
 260                 wfProfileOut( __METHOD__ . '-' . $this->name );
 261                 wfProfileOut( __METHOD__ );
 262                 return $status;
 263         }
 264
 265         /**
 266          * @see FileBackendStore::concatenate()
 267          * @return Status
 268          */
 269         protected function doConcatenate( array $params ) {
 270                 $status = Status::newGood();
 271                 $tmpPath = $params['dst']; // convenience
 272
 273                 // Check that the specified temp file is valid...
 274                 wfSuppressWarnings();
 275                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 276                 wfRestoreWarnings();
 277                 if ( !$ok ) { // not present or not empty
 278                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 279                         return $status;
 280                 }
 281
 282                 // Build up the temp file using the source chunks (in order)...
 283                 $tmpHandle = fopen( $tmpPath, 'ab' );
 284                 if ( $tmpHandle === false ) {
 285                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 286                         return $status;
 287                 }
 288                 foreach ( $params['srcs'] as $virtualSource ) {
 289                         // Get a local FS version of the chunk
 290                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 291                         if ( !$tmpFile ) {
 292                                 $status->fatal( 'backend-fail-read', $virtualSource );
 293                                 return $status;
 294                         }
 295                         // Get a handle to the local FS version
 296                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 297                         if ( $sourceHandle === false ) {
 298                                 fclose( $tmpHandle );
 299                                 $status->fatal( 'backend-fail-read', $virtualSource );
 300                                 return $status;
 301                         }
 302                         // Append chunk to file (pass chunk size to avoid magic quotes)
 303                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 304                                 fclose( $sourceHandle );
 305                                 fclose( $tmpHandle );
 306                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 307                                 return $status;
 308                         }
 309                         fclose( $sourceHandle );
 310                 }
 311                 if ( !fclose( $tmpHandle ) ) {
 312                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 313                         return $status;
 314                 }
 315
 316                 clearstatcache(); // temp file changed
 317
 318                 return $status;
 319         }
 320
 321         /**
 322          * @see FileBackend::doPrepare()
 323          * @return Status
 324          */
 325         final protected function doPrepare( array $params ) {
 326                 wfProfileIn( __METHOD__ );
 327                 wfProfileIn( __METHOD__ . '-' . $this->name );
 328
 329                 $status = Status::newGood();
 330                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 331                 if ( $dir === null ) {
 332                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 333                         wfProfileOut( __METHOD__ . '-' . $this->name );
 334                         wfProfileOut( __METHOD__ );
 335                         return $status; // invalid storage path
 336                 }
 337
 338                 if ( $shard !== null ) { // confined to a single container/shard
 339                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 340                 } else { // directory is on several shards
 341                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 342                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 343                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 344                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 345                         }
 346                 }
 347
 348                 wfProfileOut( __METHOD__ . '-' . $this->name );
 349                 wfProfileOut( __METHOD__ );
 350                 return $status;
 351         }
 352
 353         /**
 354          * @see FileBackendStore::doPrepare()
 355          * @return Status
 356          */
 357         protected function doPrepareInternal( $container, $dir, array $params ) {
 358                 return Status::newGood();
 359         }
 360
 361         /**
 362          * @see FileBackend::doSecure()
 363          * @return Status
 364          */
 365         final protected function doSecure( array $params ) {
 366                 wfProfileIn( __METHOD__ );
 367                 wfProfileIn( __METHOD__ . '-' . $this->name );
 368                 $status = Status::newGood();
 369
 370                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 371                 if ( $dir === null ) {
 372                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 373                         wfProfileOut( __METHOD__ . '-' . $this->name );
 374                         wfProfileOut( __METHOD__ );
 375                         return $status; // invalid storage path
 376                 }
 377
 378                 if ( $shard !== null ) { // confined to a single container/shard
 379                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 380                 } else { // directory is on several shards
 381                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 382                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 383                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 384                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 385                         }
 386                 }
 387
 388                 wfProfileOut( __METHOD__ . '-' . $this->name );
 389                 wfProfileOut( __METHOD__ );
 390                 return $status;
 391         }
 392
 393         /**
 394          * @see FileBackendStore::doSecure()
 395          * @return Status
 396          */
 397         protected function doSecureInternal( $container, $dir, array $params ) {
 398                 return Status::newGood();
 399         }
 400
 401         /**
 402          * @see FileBackend::doClean()
 403          * @return Status
 404          */
 405         final protected function doClean( array $params ) {
 406                 wfProfileIn( __METHOD__ );
 407                 wfProfileIn( __METHOD__ . '-' . $this->name );
 408                 $status = Status::newGood();
 409
 410                 // Recursive: first delete all empty subdirs recursively
 411                 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
 412                         $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
 413                         if ( $subDirsRel !== null ) { // no errors
 414                                 foreach ( $subDirsRel as $subDirRel ) {
 415                                         $subDir = $params['dir'] . "/{$subDirRel}"; // full path
 416                                         $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
 417                                 }
 418                         }
 419                 }
 420
 421                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 422                 if ( $dir === null ) {
 423                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 424                         wfProfileOut( __METHOD__ . '-' . $this->name );
 425                         wfProfileOut( __METHOD__ );
 426                         return $status; // invalid storage path
 427                 }
 428
 429                 // Attempt to lock this directory...
 430                 $filesLockEx = array( $params['dir'] );
 431                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 432                 if ( !$status->isOK() ) {
 433                         wfProfileOut( __METHOD__ . '-' . $this->name );
 434                         wfProfileOut( __METHOD__ );
 435                         return $status; // abort
 436                 }
 437
 438                 if ( $shard !== null ) { // confined to a single container/shard
 439                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 440                         $this->deleteContainerCache( $fullCont ); // purge cache
 441                 } else { // directory is on several shards
 442                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 443                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 444                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 445                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 446                                 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
 447                         }
 448                 }
 449
 450                 wfProfileOut( __METHOD__ . '-' . $this->name );
 451                 wfProfileOut( __METHOD__ );
 452                 return $status;
 453         }
 454
 455         /**
 456          * @see FileBackendStore::doClean()
 457          * @return Status
 458          */
 459         protected function doCleanInternal( $container, $dir, array $params ) {
 460                 return Status::newGood();
 461         }
 462
 463         /**
 464          * @see FileBackend::fileExists()
 465          * @return bool|null
 466          */
 467         final public function fileExists( array $params ) {
 468                 wfProfileIn( __METHOD__ );
 469                 wfProfileIn( __METHOD__ . '-' . $this->name );
 470                 $stat = $this->getFileStat( $params );
 471                 wfProfileOut( __METHOD__ . '-' . $this->name );
 472                 wfProfileOut( __METHOD__ );
 473                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 474         }
 475
 476         /**
 477          * @see FileBackend::getFileTimestamp()
 478          * @return bool
 479          */
 480         final public function getFileTimestamp( array $params ) {
 481                 wfProfileIn( __METHOD__ );
 482                 wfProfileIn( __METHOD__ . '-' . $this->name );
 483                 $stat = $this->getFileStat( $params );
 484                 wfProfileOut( __METHOD__ . '-' . $this->name );
 485                 wfProfileOut( __METHOD__ );
 486                 return $stat ? $stat['mtime'] : false;
 487         }
 488
 489         /**
 490          * @see FileBackend::getFileSize()
 491          * @return bool
 492          */
 493         final public function getFileSize( array $params ) {
 494                 wfProfileIn( __METHOD__ );
 495                 wfProfileIn( __METHOD__ . '-' . $this->name );
 496                 $stat = $this->getFileStat( $params );
 497                 wfProfileOut( __METHOD__ . '-' . $this->name );
 498                 wfProfileOut( __METHOD__ );
 499                 return $stat ? $stat['size'] : false;
 500         }
 501
 502         /**
 503          * @see FileBackend::getFileStat()
 504          * @return bool
 505          */
 506         final public function getFileStat( array $params ) {
 507                 wfProfileIn( __METHOD__ );
 508                 wfProfileIn( __METHOD__ . '-' . $this->name );
 509                 $path = self::normalizeStoragePath( $params['src'] );
 510                 if ( $path === null ) {
 511                         wfProfileOut( __METHOD__ . '-' . $this->name );
 512                         wfProfileOut( __METHOD__ );
 513                         return false; // invalid storage path
 514                 }
 515                 $latest = !empty( $params['latest'] ); // use latest data?
 516                 if ( !isset( $this->cache[$path]['stat'] ) ) {
 517                         $this->primeFileCache( array( $path ) ); // check persistent cache
 518                 }
 519                 if ( isset( $this->cache[$path]['stat'] ) ) {
 520                         // If we want the latest data, check that this cached
 521                         // value was in fact fetched with the latest available data.
 522                         if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
 523                                 $this->pingCache( $path ); // LRU
 524                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 525                                 wfProfileOut( __METHOD__ );
 526                                 return $this->cache[$path]['stat'];
 527                         }
 528                 }
 529                 wfProfileIn( __METHOD__ . '-miss' );
 530                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 531                 $stat = $this->doGetFileStat( $params );
 532                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 533                 wfProfileOut( __METHOD__ . '-miss' );
 534                 if ( is_array( $stat ) ) { // don't cache negatives
 535                         $stat['latest'] = $latest;
 536                         $this->trimCache(); // limit memory
 537                         $this->cache[$path]['stat'] = $stat;
 538                         $this->setFileCache( $path, $stat ); // update persistent cache
 539                 }
 540                 wfProfileOut( __METHOD__ . '-' . $this->name );
 541                 wfProfileOut( __METHOD__ );
 542                 return $stat;
 543         }
 544
 545         /**
 546          * @see FileBackendStore::getFileStat()
 547          */
 548         abstract protected function doGetFileStat( array $params );
 549
 550         /**
 551          * @see FileBackend::getFileContents()
 552          * @return bool|string
 553          */
 554         public function getFileContents( array $params ) {
 555                 wfProfileIn( __METHOD__ );
 556                 wfProfileIn( __METHOD__ . '-' . $this->name );
 557                 $tmpFile = $this->getLocalReference( $params );
 558                 if ( !$tmpFile ) {
 559                         wfProfileOut( __METHOD__ . '-' . $this->name );
 560                         wfProfileOut( __METHOD__ );
 561                         return false;
 562                 }
 563                 wfSuppressWarnings();
 564                 $data = file_get_contents( $tmpFile->getPath() );
 565                 wfRestoreWarnings();
 566                 wfProfileOut( __METHOD__ . '-' . $this->name );
 567                 wfProfileOut( __METHOD__ );
 568                 return $data;
 569         }
 570
 571         /**
 572          * @see FileBackend::getFileSha1Base36()
 573          * @return bool|string
 574          */
 575         final public function getFileSha1Base36( array $params ) {
 576                 wfProfileIn( __METHOD__ );
 577                 wfProfileIn( __METHOD__ . '-' . $this->name );
 578                 $path = $params['src'];
 579                 if ( isset( $this->cache[$path]['sha1'] ) ) {
 580                         $this->pingCache( $path ); // LRU
 581                         wfProfileOut( __METHOD__ . '-' . $this->name );
 582                         wfProfileOut( __METHOD__ );
 583                         return $this->cache[$path]['sha1'];
 584                 }
 585                 wfProfileIn( __METHOD__ . '-miss' );
 586                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 587                 $hash = $this->doGetFileSha1Base36( $params );
 588                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 589                 wfProfileOut( __METHOD__ . '-miss' );
 590                 if ( $hash ) { // don't cache negatives
 591                         $this->trimCache(); // limit memory
 592                         $this->cache[$path]['sha1'] = $hash;
 593                 }
 594                 wfProfileOut( __METHOD__ . '-' . $this->name );
 595                 wfProfileOut( __METHOD__ );
 596                 return $hash;
 597         }
 598
 599         /**
 600          * @see FileBackendStore::getFileSha1Base36()
 601          * @return bool
 602          */
 603         protected function doGetFileSha1Base36( array $params ) {
 604                 $fsFile = $this->getLocalReference( $params );
 605                 if ( !$fsFile ) {
 606                         return false;
 607                 } else {
 608                         return $fsFile->getSha1Base36();
 609                 }
 610         }
 611
 612         /**
 613          * @see FileBackend::getFileProps()
 614          * @return Array
 615          */
 616         final public function getFileProps( array $params ) {
 617                 wfProfileIn( __METHOD__ );
 618                 wfProfileIn( __METHOD__ . '-' . $this->name );
 619                 $fsFile = $this->getLocalReference( $params );
 620                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 621                 wfProfileOut( __METHOD__ . '-' . $this->name );
 622                 wfProfileOut( __METHOD__ );
 623                 return $props;
 624         }
 625
 626         /**
 627          * @see FileBackend::getLocalReference()
 628          * @return TempFSFile|null
 629          */
 630         public function getLocalReference( array $params ) {
 631                 wfProfileIn( __METHOD__ );
 632                 wfProfileIn( __METHOD__ . '-' . $this->name );
 633                 $path = $params['src'];
 634                 if ( isset( $this->expensiveCache[$path]['localRef'] ) ) {
 635                         $this->pingExpensiveCache( $path );
 636                         wfProfileOut( __METHOD__ . '-' . $this->name );
 637                         wfProfileOut( __METHOD__ );
 638                         return $this->expensiveCache[$path]['localRef'];
 639                 }
 640                 $tmpFile = $this->getLocalCopy( $params );
 641                 if ( $tmpFile ) { // don't cache negatives
 642                         $this->trimExpensiveCache(); // limit memory
 643                         $this->expensiveCache[$path]['localRef'] = $tmpFile;
 644                 }
 645                 wfProfileOut( __METHOD__ . '-' . $this->name );
 646                 wfProfileOut( __METHOD__ );
 647                 return $tmpFile;
 648         }
 649
 650         /**
 651          * @see FileBackend::streamFile()
 652          * @return Status
 653          */
 654         final public function streamFile( array $params ) {
 655                 wfProfileIn( __METHOD__ );
 656                 wfProfileIn( __METHOD__ . '-' . $this->name );
 657                 $status = Status::newGood();
 658
 659                 $info = $this->getFileStat( $params );
 660                 if ( !$info ) { // let StreamFile handle the 404
 661                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 662                 }
 663
 664                 // Set output buffer and HTTP headers for stream
 665                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 666                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 667                 if ( $res == StreamFile::NOT_MODIFIED ) {
 668                         // do nothing; client cache is up to date
 669                 } elseif ( $res == StreamFile::READY_STREAM ) {
 670                         wfProfileIn( __METHOD__ . '-send' );
 671                         wfProfileIn( __METHOD__ . '-send-' . $this->name );
 672                         $status = $this->doStreamFile( $params );
 673                         wfProfileOut( __METHOD__ . '-send-' . $this->name );
 674                         wfProfileOut( __METHOD__ . '-send' );
 675                 } else {
 676                         $status->fatal( 'backend-fail-stream', $params['src'] );
 677                 }
 678
 679                 wfProfileOut( __METHOD__ . '-' . $this->name );
 680                 wfProfileOut( __METHOD__ );
 681                 return $status;
 682         }
 683
 684         /**
 685          * @see FileBackendStore::streamFile()
 686          * @return Status
 687          */
 688         protected function doStreamFile( array $params ) {
 689                 $status = Status::newGood();
 690
 691                 $fsFile = $this->getLocalReference( $params );
 692                 if ( !$fsFile ) {
 693                         $status->fatal( 'backend-fail-stream', $params['src'] );
 694                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 695                         $status->fatal( 'backend-fail-stream', $params['src'] );
 696                 }
 697
 698                 return $status;
 699         }
 700
 701         /**
 702          * @see FileBackend::directoryExists()
 703          * @return bool|null
 704          */
 705         final public function directoryExists( array $params ) {
 706                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 707                 if ( $dir === null ) {
 708                         return false; // invalid storage path
 709                 }
 710                 if ( $shard !== null ) { // confined to a single container/shard
 711                         return $this->doDirectoryExists( $fullCont, $dir, $params );
 712                 } else { // directory is on several shards
 713                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 714                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 715                         $res = false; // response
 716                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 717                                 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
 718                                 if ( $exists ) {
 719                                         $res = true;
 720                                         break; // found one!
 721                                 } elseif ( $exists === null ) { // error?
 722                                         $res = null; // if we don't find anything, it is indeterminate
 723                                 }
 724                         }
 725                         return $res;
 726                 }
 727         }
 728
 729         /**
 730          * @see FileBackendStore::directoryExists()
 731          *
 732          * @param $container string Resolved container name
 733          * @param $dir string Resolved path relative to container
 734          * @param $params Array
 735          * @return bool|null
 736          */
 737         abstract protected function doDirectoryExists( $container, $dir, array $params );
 738
 739         /**
 740          * @see FileBackend::getDirectoryList()
 741          * @return Traversable|Array|null Returns null on failure
 742          */
 743         final public function getDirectoryList( array $params ) {
 744                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 745                 if ( $dir === null ) { // invalid storage path
 746                         return null;
 747                 }
 748                 if ( $shard !== null ) {
 749                         // File listing is confined to a single container/shard
 750                         return $this->getDirectoryListInternal( $fullCont, $dir, $params );
 751                 } else {
 752                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 753                         // File listing spans multiple containers/shards
 754                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 755                         return new FileBackendStoreShardDirIterator( $this,
 756                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 757                 }
 758         }
 759
 760         /**
 761          * Do not call this function from places outside FileBackend
 762          *
 763          * @see FileBackendStore::getDirectoryList()
 764          *
 765          * @param $container string Resolved container name
 766          * @param $dir string Resolved path relative to container
 767          * @param $params Array
 768          * @return Traversable|Array|null Returns null on failure
 769          */
 770         abstract public function getDirectoryListInternal( $container, $dir, array $params );
 771
 772         /**
 773          * @see FileBackend::getFileList()
 774          * @return Traversable|Array|null Returns null on failure
 775          */
 776         final public function getFileList( array $params ) {
 777                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 778                 if ( $dir === null ) { // invalid storage path
 779                         return null;
 780                 }
 781                 if ( $shard !== null ) {
 782                         // File listing is confined to a single container/shard
 783                         return $this->getFileListInternal( $fullCont, $dir, $params );
 784                 } else {
 785                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 786                         // File listing spans multiple containers/shards
 787                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 788                         return new FileBackendStoreShardFileIterator( $this,
 789                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 790                 }
 791         }
 792
 793         /**
 794          * Do not call this function from places outside FileBackend
 795          *
 796          * @see FileBackendStore::getFileList()
 797          *
 798          * @param $container string Resolved container name
 799          * @param $dir string Resolved path relative to container
 800          * @param $params Array
 801          * @return Traversable|Array|null Returns null on failure
 802          */
 803         abstract public function getFileListInternal( $container, $dir, array $params );
 804
 805         /**
 806          * Get the list of supported operations and their corresponding FileOp classes.
 807          *
 808          * @return Array
 809          */
 810         protected function supportedOperations() {
 811                 return array(
 812                         'store'       => 'StoreFileOp',
 813                         'copy'        => 'CopyFileOp',
 814                         'move'        => 'MoveFileOp',
 815                         'delete'      => 'DeleteFileOp',
 816                         'create'      => 'CreateFileOp',
 817                         'null'        => 'NullFileOp'
 818                 );
 819         }
 820
 821         /**
 822          * Return a list of FileOp objects from a list of operations.
 823          * Do not call this function from places outside FileBackend.
 824          *
 825          * The result must have the same number of items as the input.
 826          * An exception is thrown if an unsupported operation is requested.
 827          *
 828          * @param $ops Array Same format as doOperations()
 829          * @return Array List of FileOp objects
 830          * @throws MWException
 831          */
 832         final public function getOperationsInternal( array $ops ) {
 833                 $supportedOps = $this->supportedOperations();
 834
 835                 $performOps = array(); // array of FileOp objects
 836                 // Build up ordered array of FileOps...
 837                 foreach ( $ops as $operation ) {
 838                         $opName = $operation['op'];
 839                         if ( isset( $supportedOps[$opName] ) ) {
 840                                 $class = $supportedOps[$opName];
 841                                 // Get params for this operation
 842                                 $params = $operation;
 843                                 // Append the FileOp class
 844                                 $performOps[] = new $class( $this, $params );
 845                         } else {
 846                                 throw new MWException( "Operation '$opName' is not supported." );
 847                         }
 848                 }
 849
 850                 return $performOps;
 851         }
 852
 853         /**
 854          * Get a list of storage paths to lock for a list of operations
 855          * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
 856          * each corresponding to a list of storage paths to be locked.
 857          *
 858          * @param $performOps Array List of FileOp objects
 859          * @return Array ('sh' => list of paths, 'ex' => list of paths)
 860          */
 861         final public function getPathsToLockForOpsInternal( array $performOps ) {
 862                 // Build up a list of files to lock...
 863                 $paths = array( 'sh' => array(), 'ex' => array() );
 864                 foreach ( $performOps as $fileOp ) {
 865                         $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
 866                         $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
 867                 }
 868                 // Optimization: if doing an EX lock anyway, don't also set an SH one
 869                 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
 870                 // Get a shared lock on the parent directory of each path changed
 871                 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
 872
 873                 return $paths;
 874         }
 875
 876         /**
 877          * @see FileBackend::doOperationsInternal()
 878          * @return Status
 879          */
 880         protected function doOperationsInternal( array $ops, array $opts ) {
 881                 wfProfileIn( __METHOD__ );
 882                 wfProfileIn( __METHOD__ . '-' . $this->name );
 883                 $status = Status::newGood();
 884
 885                 // Build up a list of FileOps...
 886                 $performOps = $this->getOperationsInternal( $ops );
 887
 888                 // Acquire any locks as needed...
 889                 if ( empty( $opts['nonLocking'] ) ) {
 890                         // Build up a list of files to lock...
 891                         $paths = $this->getPathsToLockForOpsInternal( $performOps );
 892                         // Try to lock those files for the scope of this function...
 893                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 894                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 895                         if ( !$status->isOK() ) {
 896                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 897                                 wfProfileOut( __METHOD__ );
 898                                 return $status; // abort
 899                         }
 900                 }
 901
 902                 // Clear any file cache entries (after locks acquired)
 903                 $this->clearCache();
 904
 905                 // Load from the persistent file and container caches
 906                 $this->primeFileCache( $performOps );
 907                 $this->primeContainerCache( $performOps );
 908
 909                 // Actually attempt the operation batch...
 910                 $subStatus = FileOp::attemptBatch( $performOps, $opts, $this->fileJournal );
 911
 912                 // Merge errors into status fields
 913                 $status->merge( $subStatus );
 914                 $status->success = $subStatus->success; // not done in merge()
 915
 916                 wfProfileOut( __METHOD__ . '-' . $this->name );
 917                 wfProfileOut( __METHOD__ );
 918                 return $status;
 919         }
 920
 921         /**
 922          * @see FileBackend::clearCache()
 923          */
 924         final public function clearCache( array $paths = null ) {
 925                 if ( is_array( $paths ) ) {
 926                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
 927                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
 928                 }
 929                 if ( $paths === null ) {
 930                         $this->cache = array();
 931                         $this->expensiveCache = array();
 932                 } else {
 933                         foreach ( $paths as $path ) {
 934                                 unset( $this->cache[$path] );
 935                                 unset( $this->expensiveCache[$path] );
 936                         }
 937                 }
 938                 $this->doClearCache( $paths );
 939         }
 940
 941         /**
 942          * Clears any additional stat caches for storage paths
 943          *
 944          * @see FileBackend::clearCache()
 945          *
 946          * @param $paths Array Storage paths (optional)
 947          * @return void
 948          */
 949         protected function doClearCache( array $paths = null ) {}
 950
 951         /**
 952          * Is this a key/value store where directories are just virtual?
 953          * Virtual directories exists in so much as files exists that are
 954          * prefixed with the directory path followed by a forward slash.
 955          *
 956          * @return bool
 957          */
 958         abstract protected function directoriesAreVirtual();
 959
 960         /**
 961          * Move a cache entry to the top (such as when accessed)
 962          *
 963          * @param $path string Storage path
 964          * @return void
 965          */
 966         protected function pingCache( $path ) {
 967                 if ( isset( $this->cache[$path] ) ) {
 968                         $tmp = $this->cache[$path];
 969                         unset( $this->cache[$path] );
 970                         $this->cache[$path] = $tmp;
 971                 }
 972         }
 973
 974         /**
 975          * Prune the inexpensive cache if it is too big to add an item
 976          *
 977          * @return void
 978          */
 979         protected function trimCache() {
 980                 if ( count( $this->cache ) >= $this->maxCacheSize ) {
 981                         reset( $this->cache );
 982                         unset( $this->cache[key( $this->cache )] );
 983                 }
 984         }
 985
 986         /**
 987          * Move a cache entry to the top (such as when accessed)
 988          *
 989          * @param $path string Storage path
 990          * @return void
 991          */
 992         protected function pingExpensiveCache( $path ) {
 993                 if ( isset( $this->expensiveCache[$path] ) ) {
 994                         $tmp = $this->expensiveCache[$path];
 995                         unset( $this->expensiveCache[$path] );
 996                         $this->expensiveCache[$path] = $tmp;
 997                 }
 998         }
 999
1000         /**
1001          * Prune the expensive cache if it is too big to add an item
1002          *
1003          * @return void
1004          */
1005         protected function trimExpensiveCache() {
1006                 if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) {
1007                         reset( $this->expensiveCache );
1008                         unset( $this->expensiveCache[key( $this->expensiveCache )] );
1009                 }
1010         }
1011
1012         /**
1013          * Check if a container name is valid.
1014          * This checks for for length and illegal characters.
1015          *
1016          * @param $container string
1017          * @return bool
1018          */
1019         final protected static function isValidContainerName( $container ) {
1020                 // This accounts for Swift and S3 restrictions while leaving room
1021                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1022                 // This disallows directory separators or traversal characters.
1023                 // Note that matching strings URL encode to the same string;
1024                 // in Swift, the length restriction is *after* URL encoding.
1025                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1026         }
1027
1028         /**
1029          * Splits a storage path into an internal container name,
1030          * an internal relative file name, and a container shard suffix.
1031          * Any shard suffix is already appended to the internal container name.
1032          * This also checks that the storage path is valid and within this backend.
1033          *
1034          * If the container is sharded but a suffix could not be determined,
1035          * this means that the path can only refer to a directory and can only
1036          * be scanned by looking in all the container shards.
1037          *
1038          * @param $storagePath string
1039          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1040          */
1041         final protected function resolveStoragePath( $storagePath ) {
1042                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1043                 if ( $backend === $this->name ) { // must be for this backend
1044                         $relPath = self::normalizeContainerPath( $relPath );
1045                         if ( $relPath !== null ) {
1046                                 // Get shard for the normalized path if this container is sharded
1047                                 $cShard = $this->getContainerShard( $container, $relPath );
1048                                 // Validate and sanitize the relative path (backend-specific)
1049                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1050                                 if ( $relPath !== null ) {
1051                                         // Prepend any wiki ID prefix to the container name
1052                                         $container = $this->fullContainerName( $container );
1053                                         if ( self::isValidContainerName( $container ) ) {
1054                                                 // Validate and sanitize the container name (backend-specific)
1055                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1056                                                 if ( $container !== null ) {
1057                                                         return array( $container, $relPath, $cShard );
1058                                                 }
1059                                         }
1060                                 }
1061                         }
1062                 }
1063                 return array( null, null, null );
1064         }
1065
1066         /**
1067          * Like resolveStoragePath() except null values are returned if
1068          * the container is sharded and the shard could not be determined.
1069          *
1070          * @see FileBackendStore::resolveStoragePath()
1071          *
1072          * @param $storagePath string
1073          * @return Array (container, path) or (null, null) if invalid
1074          */
1075         final protected function resolveStoragePathReal( $storagePath ) {
1076                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1077                 if ( $cShard !== null ) {
1078                         return array( $container, $relPath );
1079                 }
1080                 return array( null, null );
1081         }
1082
1083         /**
1084          * Get the container name shard suffix for a given path.
1085          * Any empty suffix means the container is not sharded.
1086          *
1087          * @param $container string Container name
1088          * @param $relStoragePath string Storage path relative to the container
1089          * @return string|null Returns null if shard could not be determined
1090          */
1091         final protected function getContainerShard( $container, $relPath ) {
1092                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1093                 if ( $levels == 1 || $levels == 2 ) {
1094                         // Hash characters are either base 16 or 36
1095                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1096                         // Get a regex that represents the shard portion of paths.
1097                         // The concatenation of the captures gives us the shard.
1098                         if ( $levels === 1 ) { // 16 or 36 shards per container
1099                                 $hashDirRegex = '(' . $char . ')';
1100                         } else { // 256 or 1296 shards per container
1101                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1102                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1103                                 } else { // short hash dir format (e.g. "a/b/c")
1104                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1105                                 }
1106                         }
1107                         // Allow certain directories to be above the hash dirs so as
1108                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1109                         // They must be 2+ chars to avoid any hash directory ambiguity.
1110                         $m = array();
1111                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1112                                 return '.' . implode( '', array_slice( $m, 1 ) );
1113                         }
1114                         return null; // failed to match
1115                 }
1116                 return ''; // no sharding
1117         }
1118
1119         /**
1120          * Check if a storage path maps to a single shard.
1121          * Container dirs like "a", where the container shards on "x/xy",
1122          * can reside on several shards. Such paths are tricky to handle.
1123          *
1124          * @param $storagePath string Storage path
1125          * @return bool
1126          */
1127         final public function isSingleShardPathInternal( $storagePath ) {
1128                 list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
1129                 return ( $shard !== null );
1130         }
1131
1132         /**
1133          * Get the sharding config for a container.
1134          * If greater than 0, then all file storage paths within
1135          * the container are required to be hashed accordingly.
1136          *
1137          * @param $container string
1138          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1139          */
1140         final protected function getContainerHashLevels( $container ) {
1141                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1142                         $config = $this->shardViaHashLevels[$container];
1143                         $hashLevels = (int)$config['levels'];
1144                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1145                                 $hashBase = (int)$config['base'];
1146                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1147                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1148                                 }
1149                         }
1150                 }
1151                 return array( 0, 0, false ); // no sharding
1152         }
1153
1154         /**
1155          * Get a list of full container shard suffixes for a container
1156          *
1157          * @param $container string
1158          * @return Array
1159          */
1160         final protected function getContainerSuffixes( $container ) {
1161                 $shards = array();
1162                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1163                 if ( $digits > 0 ) {
1164                         $numShards = pow( $base, $digits );
1165                         for ( $index = 0; $index < $numShards; $index++ ) {
1166                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1167                         }
1168                 }
1169                 return $shards;
1170         }
1171
1172         /**
1173          * Get the full container name, including the wiki ID prefix
1174          *
1175          * @param $container string
1176          * @return string
1177          */
1178         final protected function fullContainerName( $container ) {
1179                 if ( $this->wikiId != '' ) {
1180                         return "{$this->wikiId}-$container";
1181                 } else {
1182                         return $container;
1183                 }
1184         }
1185
1186         /**
1187          * Resolve a container name, checking if it's allowed by the backend.
1188          * This is intended for internal use, such as encoding illegal chars.
1189          * Subclasses can override this to be more restrictive.
1190          *
1191          * @param $container string
1192          * @return string|null
1193          */
1194         protected function resolveContainerName( $container ) {
1195                 return $container;
1196         }
1197
1198         /**
1199          * Resolve a relative storage path, checking if it's allowed by the backend.
1200          * This is intended for internal use, such as encoding illegal chars or perhaps
1201          * getting absolute paths (e.g. FS based backends). Note that the relative path
1202          * may be the empty string (e.g. the path is simply to the container).
1203          *
1204          * @param $container string Container name
1205          * @param $relStoragePath string Storage path relative to the container
1206          * @return string|null Path or null if not valid
1207          */
1208         protected function resolveContainerPath( $container, $relStoragePath ) {
1209                 return $relStoragePath;
1210         }
1211
1212         /**
1213          * Get the cache key for a container
1214          *
1215          * @param $container Resolved container name
1216          * @return string
1217          */
1218         private function containerCacheKey( $container ) {
1219                 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1220         }
1221
1222         /**
1223          * Set the cached info for a container
1224          *
1225          * @param $container Resolved container name
1226          * @param $val mixed Information to cache
1227          * @return void
1228          */
1229         final protected function setContainerCache( $container, $val ) {
1230                 $this->memCache->set( $this->containerCacheKey( $container ), $val, 14*86400 );
1231         }
1232
1233         /**
1234          * Delete the cached info for a container
1235          *
1236          * @param $container Resolved container name
1237          * @return void
1238          */
1239         final protected function deleteContainerCache( $container ) {
1240                 for ( $attempts=1; $attempts <= 3; $attempts++ ) {
1241                         if ( $this->memCache->delete( $this->containerCacheKey( $container ) ) ) {
1242                                 return; // done!
1243                         }
1244                 }
1245                 trigger_error( "Unable to delete stat cache for container $container." );
1246         }
1247
1248         /**
1249          * Do a batch lookup from cache for container stats for all containers
1250          * used in a list of container names, storage paths, or FileOp objects.
1251          *
1252          * @param $items Array
1253          * @return void
1254          */
1255         final protected function primeContainerCache( array $items ) {
1256                 wfProfileIn( __METHOD__ );
1257                 wfProfileIn( __METHOD__ . '-' . $this->name );
1258                 $paths = array(); // list of storage paths
1259                 $contNames = array(); // (cache key => resolved container name)
1260                 // Get all the paths/containers from the items...
1261                 foreach ( $items as $item ) {
1262                         if ( $item instanceof FileOp ) {
1263                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1264                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1265                         } elseif ( self::isStoragePath( $item ) ) {
1266                                 $paths[] = $item;
1267                         } elseif ( is_string( $item ) ) { // full container name
1268                                 $contNames[$this->containerCacheKey( $item )] = $item;
1269                         }
1270                 }
1271                 // Get all the corresponding cache keys for paths...
1272                 foreach ( $paths as $path ) {
1273                         list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
1274                         if ( $fullCont !== null ) { // valid path for this backend
1275                                 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1276                         }
1277                 }
1278
1279                 $contInfo = array(); // (resolved container name => cache value)
1280                 // Get all cache entries for these container cache keys...
1281                 $values = $this->memCache->getBatch( array_keys( $contNames ) );
1282                 foreach ( $values as $cacheKey => $val ) {
1283                         $contInfo[$contNames[$cacheKey]] = $val;
1284                 }
1285
1286                 // Populate the container process cache for the backend...
1287                 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1288                 wfProfileOut( __METHOD__ . '-' . $this->name );
1289                 wfProfileOut( __METHOD__ );
1290         }
1291
1292         /**
1293          * Fill the backend-specific process cache given an array of
1294          * resolved container names and their corresponding cached info.
1295          * Only containers that actually exist should appear in the map.
1296          *
1297          * @param $containerInfo Array Map of resolved container names to cached info
1298          * @return void
1299          */
1300         protected function doPrimeContainerCache( array $containerInfo ) {}
1301
1302         /**
1303          * Get the cache key for a file path
1304          *
1305          * @param $path Storage path
1306          * @return string
1307          */
1308         private function fileCacheKey( $path ) {
1309                 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1310         }
1311
1312         /**
1313          * Set the cached stat info for a file path
1314          *
1315          * @param $path Storage path
1316          * @param $val mixed Information to cache
1317          * @return void
1318          */
1319         final protected function setFileCache( $path, $val ) {
1320                 $this->memCache->set( $this->fileCacheKey( $path ), $val, 7*86400 );
1321         }
1322
1323         /**
1324          * Delete the cached stat info for a file path
1325          *
1326          * @param $path Storage path
1327          * @return void
1328          */
1329         final protected function deleteFileCache( $path ) {
1330                 for ( $attempts=1; $attempts <= 3; $attempts++ ) {
1331                         if ( $this->memCache->delete( $this->fileCacheKey( $path ) ) ) {
1332                                 return; // done!
1333                         }
1334                 }
1335                 trigger_error( "Unable to delete stat cache for file $path." );
1336         }
1337
1338         /**
1339          * Do a batch lookup from cache for file stats for all paths
1340          * used in a list of storage paths or FileOp objects.
1341          *
1342          * @param $items Array List of storage paths or FileOps
1343          * @return void
1344          */
1345         final protected function primeFileCache( array $items ) {
1346                 wfProfileIn( __METHOD__ );
1347                 wfProfileIn( __METHOD__ . '-' . $this->name );
1348                 $paths = array(); // list of storage paths
1349                 $pathNames = array(); // (cache key => storage path)
1350                 // Get all the paths/containers from the items...
1351                 foreach ( $items as $item ) {
1352                         if ( $item instanceof FileOp ) {
1353                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1354                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1355                         } elseif ( self::isStoragePath( $item ) ) {
1356                                 $paths[] = $item;
1357                         }
1358                 }
1359                 // Get all the corresponding cache keys for paths...
1360                 foreach ( $paths as $path ) {
1361                         list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
1362                         if ( $rel !== null ) { // valid path for this backend
1363                                 $pathNames[$this->fileCacheKey( $path )] = $path;
1364                         }
1365                 }
1366                 // Get all cache entries for these container cache keys...
1367                 $values = $this->memCache->getBatch( array_keys( $pathNames ) );
1368                 foreach ( $values as $cacheKey => $val ) {
1369                         if ( is_array( $val ) ) {
1370                                 $this->trimCache(); // limit memory
1371                                 $this->cache[$pathNames[$cacheKey]]['stat'] = $val;
1372                         }
1373                 }
1374                 wfProfileOut( __METHOD__ . '-' . $this->name );
1375                 wfProfileOut( __METHOD__ );
1376         }
1377 }
1378
1379 /**
1380  * FileBackendStore helper function to handle listings that span container shards.
1381  * Do not use this class from places outside of FileBackendStore.
1382  *
1383  * @ingroup FileBackend
1384  */
1385 abstract class FileBackendStoreShardListIterator implements Iterator {
1386         /** @var FileBackendStore */
1387         protected $backend;
1388         /** @var Array */
1389         protected $params;
1390         /** @var Array */
1391         protected $shardSuffixes;
1392         protected $container; // string; full container name
1393         protected $directory; // string; resolved relative path
1394
1395         /** @var Traversable */
1396         protected $iter;
1397         protected $curShard = 0; // integer
1398         protected $pos = 0; // integer
1399
1400         /** @var Array */
1401         protected $multiShardPaths = array(); // (rel path => 1)
1402
1403         /**
1404          * @param $backend FileBackendStore
1405          * @param $container string Full storage container name
1406          * @param $dir string Storage directory relative to container
1407          * @param $suffixes Array List of container shard suffixes
1408          * @param $params Array
1409          */
1410         public function __construct(
1411                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1412         ) {
1413                 $this->backend = $backend;
1414                 $this->container = $container;
1415                 $this->directory = $dir;
1416                 $this->shardSuffixes = $suffixes;
1417                 $this->params = $params;
1418         }
1419
1420         /**
1421          * @see Iterator::current()
1422          * @return string|bool String or false
1423          */
1424         public function current() {
1425                 if ( is_array( $this->iter ) ) {
1426                         return current( $this->iter );
1427                 } else {
1428                         return $this->iter->current();
1429                 }
1430         }
1431
1432         /**
1433          * @see Iterator::key()
1434          * @return integer
1435          */
1436         public function key() {
1437                 return $this->pos;
1438         }
1439
1440         /**
1441          * @see Iterator::next()
1442          * @return void
1443          */
1444         public function next() {
1445                 ++$this->pos;
1446                 if ( is_array( $this->iter ) ) {
1447                         next( $this->iter );
1448                 } else {
1449                         $this->iter->next();
1450                 }
1451                 // Filter out items that we already listed
1452                 $this->filterViaNext();
1453                 // Find the next non-empty shard if no elements are left
1454                 $this->nextShardIteratorIfNotValid();
1455         }
1456
1457         /**
1458          * @see Iterator::rewind()
1459          * @return void
1460          */
1461         public function rewind() {
1462                 $this->pos = 0;
1463                 $this->curShard = 0;
1464                 $this->setIteratorFromCurrentShard();
1465                 // Filter out items that we already listed
1466                 $this->filterViaNext();
1467                 // Find the next non-empty shard if this one has no elements
1468                 $this->nextShardIteratorIfNotValid();
1469         }
1470
1471         /**
1472          * @see Iterator::valid()
1473          * @return bool
1474          */
1475         public function valid() {
1476                 if ( $this->iter === null ) {
1477                         return false; // some failure?
1478                 } elseif ( is_array( $this->iter ) ) {
1479                         return ( current( $this->iter ) !== false ); // no paths can have this value
1480                 } else {
1481                         return $this->iter->valid();
1482                 }
1483         }
1484
1485         /**
1486          * Filter out duplicate items by advancing to the next ones
1487          */
1488         protected function filterViaNext() {
1489                 while ( $this->iter->valid() ) {
1490                         $rel = $this->iter->current(); // path relative to given directory
1491                         $path = $this->params['dir'] . "/{$rel}"; // full storage path
1492                         if ( !$this->backend->isSingleShardPathInternal( $path ) ) {
1493                                 // Don't keep listing paths that are on multiple shards
1494                                 if ( isset( $this->multiShardPaths[$rel] ) ) {
1495                                         $this->iter->next(); // we already listed this path
1496                                 } else {
1497                                         $this->multiShardPaths[$rel] = 1;
1498                                         break;
1499                                 }
1500                         }
1501                 }
1502         }
1503
1504         /**
1505          * If the list iterator for this container shard is out of items,
1506          * then move on to the next container that has items.
1507          * If there are none, then it advances to the last container.
1508          */
1509         protected function nextShardIteratorIfNotValid() {
1510                 while ( !$this->valid() ) {
1511                         if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1512                                 break; // no more container shards
1513                         }
1514                         $this->setIteratorFromCurrentShard();
1515                 }
1516         }
1517
1518         /**
1519          * Set the list iterator to that of the current container shard
1520          */
1521         protected function setIteratorFromCurrentShard() {
1522                 $suffix = $this->shardSuffixes[$this->curShard];
1523                 $this->iter = $this->listFromShard(
1524                         "{$this->container}{$suffix}", $this->directory, $this->params );
1525         }
1526
1527         /**
1528          * Get the list for a given container shard
1529          *
1530          * @param $container string Resolved container name
1531          * @param $dir string Resolved path relative to container
1532          * @param $params Array
1533          * @return Traversable|Array|null
1534          */
1535         abstract protected function listFromShard( $container, $dir, array $params );
1536 }
1537
1538 /**
1539  * Iterator for listing directories
1540  */
1541 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1542         protected function listFromShard( $container, $dir, array $params ) {
1543                 return $this->backend->getDirectoryListInternal( $container, $dir, $params );
1544         }
1545 }
1546
1547 /**
1548  * Iterator for listing regular files
1549  */
1550 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1551         protected function listFromShard( $container, $dir, array $params ) {
1552                 return $this->backend->getFileListInternal( $container, $dir, $params );
1553         }
1554 }