includes/filebackend/FileBackendStore.php

   1 <?php
   2 /**
   3  * Base class for all backends using particular storage medium.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Base class for all backends using particular storage medium.
  27  *
  28  * This class defines the methods as abstract that subclasses must implement.
  29  * Outside callers should *not* use functions with "Internal" in the name.
  30  *
  31  * The FileBackend operations are implemented using basic functions
  32  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  33  * This class is also responsible for path resolution and sanitization.
  34  *
  35  * @ingroup FileBackend
  36  * @since 1.19
  37  */
  38 abstract class FileBackendStore extends FileBackend {
  39         /** @var BagOStuff */
  40         protected $memCache;
  41         /** @var ProcessCacheLRU */
  42         protected $cheapCache; // Map of paths to small (RAM/disk) cache items
  43         /** @var ProcessCacheLRU */
  44         protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
  45
  46         /** @var Array Map of container names to sharding settings */
  47         protected $shardViaHashLevels = array(); // (container name => config array)
  48
  49         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  50
  51         /**
  52          * @see FileBackend::__construct()
  53          *
  54          * @param $config Array
  55          */
  56         public function __construct( array $config ) {
  57                 parent::__construct( $config );
  58                 $this->memCache       = new EmptyBagOStuff(); // disabled by default
  59                 $this->cheapCache     = new ProcessCacheLRU( 300 );
  60                 $this->expensiveCache = new ProcessCacheLRU( 5 );
  61         }
  62
  63         /**
  64          * Get the maximum allowable file size given backend
  65          * medium restrictions and basic performance constraints.
  66          * Do not call this function from places outside FileBackend and FileOp.
  67          *
  68          * @return integer Bytes
  69          */
  70         final public function maxFileSizeInternal() {
  71                 return $this->maxFileSize;
  72         }
  73
  74         /**
  75          * Check if a file can be created at a given storage path.
  76          * FS backends should check if the parent directory exists and the file is writable.
  77          * Backends using key/value stores should check if the container exists.
  78          *
  79          * @param $storagePath string
  80          * @return bool
  81          */
  82         abstract public function isPathUsableInternal( $storagePath );
  83
  84         /**
  85          * Create a file in the backend with the given contents.
  86          * Do not call this function from places outside FileBackend and FileOp.
  87          *
  88          * $params include:
  89          *   - content       : the raw file contents
  90          *   - dst           : destination storage path
  91          *   - overwrite     : overwrite any file that exists at the destination
  92          *   - disposition   : Content-Disposition header value for the destination
  93          *   - async         : Status will be returned immediately if supported.
  94          *                     If the status is OK, then its value field will be
  95          *                     set to a FileBackendStoreOpHandle object.
  96          *
  97          * @param $params Array
  98          * @return Status
  99          */
 100         final public function createInternal( array $params ) {
 101                 wfProfileIn( __METHOD__ );
 102                 wfProfileIn( __METHOD__ . '-' . $this->name );
 103                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 104                         $status = Status::newFatal( 'backend-fail-maxsize',
 105                                 $params['dst'], $this->maxFileSizeInternal() );
 106                 } else {
 107                         $status = $this->doCreateInternal( $params );
 108                         $this->clearCache( array( $params['dst'] ) );
 109                         if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 110                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 111                         }
 112                 }
 113                 wfProfileOut( __METHOD__ . '-' . $this->name );
 114                 wfProfileOut( __METHOD__ );
 115                 return $status;
 116         }
 117
 118         /**
 119          * @see FileBackendStore::createInternal()
 120          */
 121         abstract protected function doCreateInternal( array $params );
 122
 123         /**
 124          * Store a file into the backend from a file on disk.
 125          * Do not call this function from places outside FileBackend and FileOp.
 126          *
 127          * $params include:
 128          *   - src           : source path on disk
 129          *   - dst           : destination storage path
 130          *   - overwrite     : overwrite any file that exists at the destination
 131          *   - disposition   : Content-Disposition header value for the destination
 132          *   - async         : Status will be returned immediately if supported.
 133          *                     If the status is OK, then its value field will be
 134          *                     set to a FileBackendStoreOpHandle object.
 135          *
 136          * @param $params Array
 137          * @return Status
 138          */
 139         final public function storeInternal( array $params ) {
 140                 wfProfileIn( __METHOD__ );
 141                 wfProfileIn( __METHOD__ . '-' . $this->name );
 142                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 143                         $status = Status::newFatal( 'backend-fail-maxsize',
 144                                 $params['dst'], $this->maxFileSizeInternal() );
 145                 } else {
 146                         $status = $this->doStoreInternal( $params );
 147                         $this->clearCache( array( $params['dst'] ) );
 148                         if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 149                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 150                         }
 151                 }
 152                 wfProfileOut( __METHOD__ . '-' . $this->name );
 153                 wfProfileOut( __METHOD__ );
 154                 return $status;
 155         }
 156
 157         /**
 158          * @see FileBackendStore::storeInternal()
 159          */
 160         abstract protected function doStoreInternal( array $params );
 161
 162         /**
 163          * Copy a file from one storage path to another in the backend.
 164          * Do not call this function from places outside FileBackend and FileOp.
 165          *
 166          * $params include:
 167          *   - src           : source storage path
 168          *   - dst           : destination storage path
 169          *   - overwrite     : overwrite any file that exists at the destination
 170          *   - disposition   : Content-Disposition header value for the destination
 171          *   - async         : Status will be returned immediately if supported.
 172          *                     If the status is OK, then its value field will be
 173          *                     set to a FileBackendStoreOpHandle object.
 174          *
 175          * @param $params Array
 176          * @return Status
 177          */
 178         final public function copyInternal( array $params ) {
 179                 wfProfileIn( __METHOD__ );
 180                 wfProfileIn( __METHOD__ . '-' . $this->name );
 181                 $status = $this->doCopyInternal( $params );
 182                 $this->clearCache( array( $params['dst'] ) );
 183                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 184                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 185                 }
 186                 wfProfileOut( __METHOD__ . '-' . $this->name );
 187                 wfProfileOut( __METHOD__ );
 188                 return $status;
 189         }
 190
 191         /**
 192          * @see FileBackendStore::copyInternal()
 193          */
 194         abstract protected function doCopyInternal( array $params );
 195
 196         /**
 197          * Delete a file at the storage path.
 198          * Do not call this function from places outside FileBackend and FileOp.
 199          *
 200          * $params include:
 201          *   - src                 : source storage path
 202          *   - ignoreMissingSource : do nothing if the source file does not exist
 203          *   - async               : Status will be returned immediately if supported.
 204          *                           If the status is OK, then its value field will be
 205          *                           set to a FileBackendStoreOpHandle object.
 206          *
 207          * @param $params Array
 208          * @return Status
 209          */
 210         final public function deleteInternal( array $params ) {
 211                 wfProfileIn( __METHOD__ );
 212                 wfProfileIn( __METHOD__ . '-' . $this->name );
 213                 $status = $this->doDeleteInternal( $params );
 214                 $this->clearCache( array( $params['src'] ) );
 215                 $this->deleteFileCache( $params['src'] ); // persistent cache
 216                 wfProfileOut( __METHOD__ . '-' . $this->name );
 217                 wfProfileOut( __METHOD__ );
 218                 return $status;
 219         }
 220
 221         /**
 222          * @see FileBackendStore::deleteInternal()
 223          */
 224         abstract protected function doDeleteInternal( array $params );
 225
 226         /**
 227          * Move a file from one storage path to another in the backend.
 228          * Do not call this function from places outside FileBackend and FileOp.
 229          *
 230          * $params include:
 231          *   - src           : source storage path
 232          *   - dst           : destination storage path
 233          *   - overwrite     : overwrite any file that exists at the destination
 234          *   - disposition   : Content-Disposition header value for the destination
 235          *   - async         : Status will be returned immediately if supported.
 236          *                     If the status is OK, then its value field will be
 237          *                     set to a FileBackendStoreOpHandle object.
 238          *
 239          * @param $params Array
 240          * @return Status
 241          */
 242         final public function moveInternal( array $params ) {
 243                 wfProfileIn( __METHOD__ );
 244                 wfProfileIn( __METHOD__ . '-' . $this->name );
 245                 $status = $this->doMoveInternal( $params );
 246                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 247                 $this->deleteFileCache( $params['src'] ); // persistent cache
 248                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 249                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 250                 }
 251                 wfProfileOut( __METHOD__ . '-' . $this->name );
 252                 wfProfileOut( __METHOD__ );
 253                 return $status;
 254         }
 255
 256         /**
 257          * @see FileBackendStore::moveInternal()
 258          * @return Status
 259          */
 260         protected function doMoveInternal( array $params ) {
 261                 unset( $params['async'] ); // two steps, won't work here :)
 262                 // Copy source to dest
 263                 $status = $this->copyInternal( $params );
 264                 if ( $status->isOK() ) {
 265                         // Delete source (only fails due to races or medium going down)
 266                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 267                         $status->setResult( true, $status->value ); // ignore delete() errors
 268                 }
 269                 return $status;
 270         }
 271
 272         /**
 273          * No-op file operation that does nothing.
 274          * Do not call this function from places outside FileBackend and FileOp.
 275          *
 276          * @param $params Array
 277          * @return Status
 278          */
 279         final public function nullInternal( array $params ) {
 280                 return Status::newGood();
 281         }
 282
 283         /**
 284          * @see FileBackend::concatenate()
 285          * @return Status
 286          */
 287         final public function concatenate( array $params ) {
 288                 wfProfileIn( __METHOD__ );
 289                 wfProfileIn( __METHOD__ . '-' . $this->name );
 290                 $status = Status::newGood();
 291
 292                 // Try to lock the source files for the scope of this function
 293                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 294                 if ( $status->isOK() ) {
 295                         // Actually do the concatenation
 296                         $status->merge( $this->doConcatenate( $params ) );
 297                 }
 298
 299                 wfProfileOut( __METHOD__ . '-' . $this->name );
 300                 wfProfileOut( __METHOD__ );
 301                 return $status;
 302         }
 303
 304         /**
 305          * @see FileBackendStore::concatenate()
 306          * @return Status
 307          */
 308         protected function doConcatenate( array $params ) {
 309                 $status = Status::newGood();
 310                 $tmpPath = $params['dst']; // convenience
 311
 312                 // Check that the specified temp file is valid...
 313                 wfSuppressWarnings();
 314                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 315                 wfRestoreWarnings();
 316                 if ( !$ok ) { // not present or not empty
 317                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 318                         return $status;
 319                 }
 320
 321                 // Build up the temp file using the source chunks (in order)...
 322                 $tmpHandle = fopen( $tmpPath, 'ab' );
 323                 if ( $tmpHandle === false ) {
 324                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 325                         return $status;
 326                 }
 327                 foreach ( $params['srcs'] as $virtualSource ) {
 328                         // Get a local FS version of the chunk
 329                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 330                         if ( !$tmpFile ) {
 331                                 $status->fatal( 'backend-fail-read', $virtualSource );
 332                                 return $status;
 333                         }
 334                         // Get a handle to the local FS version
 335                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 336                         if ( $sourceHandle === false ) {
 337                                 fclose( $tmpHandle );
 338                                 $status->fatal( 'backend-fail-read', $virtualSource );
 339                                 return $status;
 340                         }
 341                         // Append chunk to file (pass chunk size to avoid magic quotes)
 342                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 343                                 fclose( $sourceHandle );
 344                                 fclose( $tmpHandle );
 345                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 346                                 return $status;
 347                         }
 348                         fclose( $sourceHandle );
 349                 }
 350                 if ( !fclose( $tmpHandle ) ) {
 351                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 352                         return $status;
 353                 }
 354
 355                 clearstatcache(); // temp file changed
 356
 357                 return $status;
 358         }
 359
 360         /**
 361          * @see FileBackend::doPrepare()
 362          * @return Status
 363          */
 364         final protected function doPrepare( array $params ) {
 365                 wfProfileIn( __METHOD__ );
 366                 wfProfileIn( __METHOD__ . '-' . $this->name );
 367
 368                 $status = Status::newGood();
 369                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 370                 if ( $dir === null ) {
 371                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 372                         wfProfileOut( __METHOD__ . '-' . $this->name );
 373                         wfProfileOut( __METHOD__ );
 374                         return $status; // invalid storage path
 375                 }
 376
 377                 if ( $shard !== null ) { // confined to a single container/shard
 378                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 379                 } else { // directory is on several shards
 380                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 381                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 382                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 383                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 384                         }
 385                 }
 386
 387                 wfProfileOut( __METHOD__ . '-' . $this->name );
 388                 wfProfileOut( __METHOD__ );
 389                 return $status;
 390         }
 391
 392         /**
 393          * @see FileBackendStore::doPrepare()
 394          * @return Status
 395          */
 396         protected function doPrepareInternal( $container, $dir, array $params ) {
 397                 return Status::newGood();
 398         }
 399
 400         /**
 401          * @see FileBackend::doSecure()
 402          * @return Status
 403          */
 404         final protected function doSecure( array $params ) {
 405                 wfProfileIn( __METHOD__ );
 406                 wfProfileIn( __METHOD__ . '-' . $this->name );
 407                 $status = Status::newGood();
 408
 409                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 410                 if ( $dir === null ) {
 411                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 412                         wfProfileOut( __METHOD__ . '-' . $this->name );
 413                         wfProfileOut( __METHOD__ );
 414                         return $status; // invalid storage path
 415                 }
 416
 417                 if ( $shard !== null ) { // confined to a single container/shard
 418                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 419                 } else { // directory is on several shards
 420                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 421                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 422                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 423                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 424                         }
 425                 }
 426
 427                 wfProfileOut( __METHOD__ . '-' . $this->name );
 428                 wfProfileOut( __METHOD__ );
 429                 return $status;
 430         }
 431
 432         /**
 433          * @see FileBackendStore::doSecure()
 434          * @return Status
 435          */
 436         protected function doSecureInternal( $container, $dir, array $params ) {
 437                 return Status::newGood();
 438         }
 439
 440         /**
 441          * @see FileBackend::doPublish()
 442          * @return Status
 443          */
 444         final protected function doPublish( array $params ) {
 445                 wfProfileIn( __METHOD__ );
 446                 wfProfileIn( __METHOD__ . '-' . $this->name );
 447                 $status = Status::newGood();
 448
 449                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 450                 if ( $dir === null ) {
 451                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 452                         wfProfileOut( __METHOD__ . '-' . $this->name );
 453                         wfProfileOut( __METHOD__ );
 454                         return $status; // invalid storage path
 455                 }
 456
 457                 if ( $shard !== null ) { // confined to a single container/shard
 458                         $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 459                 } else { // directory is on several shards
 460                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 461                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 462                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 463                                 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 464                         }
 465                 }
 466
 467                 wfProfileOut( __METHOD__ . '-' . $this->name );
 468                 wfProfileOut( __METHOD__ );
 469                 return $status;
 470         }
 471
 472         /**
 473          * @see FileBackendStore::doPublish()
 474          * @return Status
 475          */
 476         protected function doPublishInternal( $container, $dir, array $params ) {
 477                 return Status::newGood();
 478         }
 479
 480         /**
 481          * @see FileBackend::doClean()
 482          * @return Status
 483          */
 484         final protected function doClean( array $params ) {
 485                 wfProfileIn( __METHOD__ );
 486                 wfProfileIn( __METHOD__ . '-' . $this->name );
 487                 $status = Status::newGood();
 488
 489                 // Recursive: first delete all empty subdirs recursively
 490                 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
 491                         $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
 492                         if ( $subDirsRel !== null ) { // no errors
 493                                 foreach ( $subDirsRel as $subDirRel ) {
 494                                         $subDir = $params['dir'] . "/{$subDirRel}"; // full path
 495                                         $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
 496                                 }
 497                         }
 498                 }
 499
 500                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 501                 if ( $dir === null ) {
 502                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 503                         wfProfileOut( __METHOD__ . '-' . $this->name );
 504                         wfProfileOut( __METHOD__ );
 505                         return $status; // invalid storage path
 506                 }
 507
 508                 // Attempt to lock this directory...
 509                 $filesLockEx = array( $params['dir'] );
 510                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 511                 if ( !$status->isOK() ) {
 512                         wfProfileOut( __METHOD__ . '-' . $this->name );
 513                         wfProfileOut( __METHOD__ );
 514                         return $status; // abort
 515                 }
 516
 517                 if ( $shard !== null ) { // confined to a single container/shard
 518                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 519                         $this->deleteContainerCache( $fullCont ); // purge cache
 520                 } else { // directory is on several shards
 521                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 522                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 523                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 524                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 525                                 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
 526                         }
 527                 }
 528
 529                 wfProfileOut( __METHOD__ . '-' . $this->name );
 530                 wfProfileOut( __METHOD__ );
 531                 return $status;
 532         }
 533
 534         /**
 535          * @see FileBackendStore::doClean()
 536          * @return Status
 537          */
 538         protected function doCleanInternal( $container, $dir, array $params ) {
 539                 return Status::newGood();
 540         }
 541
 542         /**
 543          * @see FileBackend::fileExists()
 544          * @return bool|null
 545          */
 546         final public function fileExists( array $params ) {
 547                 wfProfileIn( __METHOD__ );
 548                 wfProfileIn( __METHOD__ . '-' . $this->name );
 549                 $stat = $this->getFileStat( $params );
 550                 wfProfileOut( __METHOD__ . '-' . $this->name );
 551                 wfProfileOut( __METHOD__ );
 552                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 553         }
 554
 555         /**
 556          * @see FileBackend::getFileTimestamp()
 557          * @return bool
 558          */
 559         final public function getFileTimestamp( array $params ) {
 560                 wfProfileIn( __METHOD__ );
 561                 wfProfileIn( __METHOD__ . '-' . $this->name );
 562                 $stat = $this->getFileStat( $params );
 563                 wfProfileOut( __METHOD__ . '-' . $this->name );
 564                 wfProfileOut( __METHOD__ );
 565                 return $stat ? $stat['mtime'] : false;
 566         }
 567
 568         /**
 569          * @see FileBackend::getFileSize()
 570          * @return bool
 571          */
 572         final public function getFileSize( array $params ) {
 573                 wfProfileIn( __METHOD__ );
 574                 wfProfileIn( __METHOD__ . '-' . $this->name );
 575                 $stat = $this->getFileStat( $params );
 576                 wfProfileOut( __METHOD__ . '-' . $this->name );
 577                 wfProfileOut( __METHOD__ );
 578                 return $stat ? $stat['size'] : false;
 579         }
 580
 581         /**
 582          * @see FileBackend::getFileStat()
 583          * @return bool
 584          */
 585         final public function getFileStat( array $params ) {
 586                 $path = self::normalizeStoragePath( $params['src'] );
 587                 if ( $path === null ) {
 588                         return false; // invalid storage path
 589                 }
 590                 wfProfileIn( __METHOD__ );
 591                 wfProfileIn( __METHOD__ . '-' . $this->name );
 592                 $latest = !empty( $params['latest'] ); // use latest data?
 593                 if ( !$this->cheapCache->has( $path, 'stat' ) ) {
 594                         $this->primeFileCache( array( $path ) ); // check persistent cache
 595                 }
 596                 if ( $this->cheapCache->has( $path, 'stat' ) ) {
 597                         $stat = $this->cheapCache->get( $path, 'stat' );
 598                         // If we want the latest data, check that this cached
 599                         // value was in fact fetched with the latest available data.
 600                         if ( !$latest || $stat['latest'] ) {
 601                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 602                                 wfProfileOut( __METHOD__ );
 603                                 return $stat;
 604                         }
 605                 }
 606                 wfProfileIn( __METHOD__ . '-miss' );
 607                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 608                 $stat = $this->doGetFileStat( $params );
 609                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 610                 wfProfileOut( __METHOD__ . '-miss' );
 611                 if ( is_array( $stat ) ) { // don't cache negatives
 612                         $stat['latest'] = $latest;
 613                         $this->cheapCache->set( $path, 'stat', $stat );
 614                         $this->setFileCache( $path, $stat ); // update persistent cache
 615                         if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
 616                                 $this->cheapCache->set( $path, 'sha1',
 617                                         array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
 618                         }
 619                 } else {
 620                         wfDebug( __METHOD__ . ": File $path does not exist.\n" );
 621                 }
 622                 wfProfileOut( __METHOD__ . '-' . $this->name );
 623                 wfProfileOut( __METHOD__ );
 624                 return $stat;
 625         }
 626
 627         /**
 628          * @see FileBackendStore::getFileStat()
 629          */
 630         abstract protected function doGetFileStat( array $params );
 631
 632         /**
 633          * @see FileBackend::getFileContents()
 634          * @return bool|string
 635          */
 636         public function getFileContents( array $params ) {
 637                 wfProfileIn( __METHOD__ );
 638                 wfProfileIn( __METHOD__ . '-' . $this->name );
 639                 $tmpFile = $this->getLocalReference( $params );
 640                 if ( !$tmpFile ) {
 641                         wfProfileOut( __METHOD__ . '-' . $this->name );
 642                         wfProfileOut( __METHOD__ );
 643                         return false;
 644                 }
 645                 wfSuppressWarnings();
 646                 $data = file_get_contents( $tmpFile->getPath() );
 647                 wfRestoreWarnings();
 648                 wfProfileOut( __METHOD__ . '-' . $this->name );
 649                 wfProfileOut( __METHOD__ );
 650                 return $data;
 651         }
 652
 653         /**
 654          * @see FileBackend::getFileSha1Base36()
 655          * @return bool|string
 656          */
 657         final public function getFileSha1Base36( array $params ) {
 658                 $path = self::normalizeStoragePath( $params['src'] );
 659                 if ( $path === null ) {
 660                         return false; // invalid storage path
 661                 }
 662                 wfProfileIn( __METHOD__ );
 663                 wfProfileIn( __METHOD__ . '-' . $this->name );
 664                 $latest = !empty( $params['latest'] ); // use latest data?
 665                 if ( $this->cheapCache->has( $path, 'sha1' ) ) {
 666                         $stat = $this->cheapCache->get( $path, 'sha1' );
 667                         // If we want the latest data, check that this cached
 668                         // value was in fact fetched with the latest available data.
 669                         if ( !$latest || $stat['latest'] ) {
 670                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 671                                 wfProfileOut( __METHOD__ );
 672                                 return $stat['hash'];
 673                         }
 674                 }
 675                 wfProfileIn( __METHOD__ . '-miss' );
 676                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 677                 $hash = $this->doGetFileSha1Base36( $params );
 678                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 679                 wfProfileOut( __METHOD__ . '-miss' );
 680                 if ( $hash ) { // don't cache negatives
 681                         $this->cheapCache->set( $path, 'sha1',
 682                                 array( 'hash' => $hash, 'latest' => $latest ) );
 683                 }
 684                 wfProfileOut( __METHOD__ . '-' . $this->name );
 685                 wfProfileOut( __METHOD__ );
 686                 return $hash;
 687         }
 688
 689         /**
 690          * @see FileBackendStore::getFileSha1Base36()
 691          * @return bool|string
 692          */
 693         protected function doGetFileSha1Base36( array $params ) {
 694                 $fsFile = $this->getLocalReference( $params );
 695                 if ( !$fsFile ) {
 696                         return false;
 697                 } else {
 698                         return $fsFile->getSha1Base36();
 699                 }
 700         }
 701
 702         /**
 703          * @see FileBackend::getFileProps()
 704          * @return Array
 705          */
 706         final public function getFileProps( array $params ) {
 707                 wfProfileIn( __METHOD__ );
 708                 wfProfileIn( __METHOD__ . '-' . $this->name );
 709                 $fsFile = $this->getLocalReference( $params );
 710                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 711                 wfProfileOut( __METHOD__ . '-' . $this->name );
 712                 wfProfileOut( __METHOD__ );
 713                 return $props;
 714         }
 715
 716         /**
 717          * @see FileBackend::getLocalReference()
 718          * @return TempFSFile|null
 719          */
 720         public function getLocalReference( array $params ) {
 721                 $path = self::normalizeStoragePath( $params['src'] );
 722                 if ( $path === null ) {
 723                         return null; // invalid storage path
 724                 }
 725                 wfProfileIn( __METHOD__ );
 726                 wfProfileIn( __METHOD__ . '-' . $this->name );
 727                 $latest = !empty( $params['latest'] ); // use latest data?
 728                 if ( $this->expensiveCache->has( $path, 'localRef' ) ) {
 729                         $val = $this->expensiveCache->get( $path, 'localRef' );
 730                         // If we want the latest data, check that this cached
 731                         // value was in fact fetched with the latest available data.
 732                         if ( !$latest || $val['latest'] ) {
 733                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 734                                 wfProfileOut( __METHOD__ );
 735                                 return $val['object'];
 736                         }
 737                 }
 738                 $tmpFile = $this->getLocalCopy( $params );
 739                 if ( $tmpFile ) { // don't cache negatives
 740                         $this->expensiveCache->set( $path, 'localRef',
 741                                 array( 'object' => $tmpFile, 'latest' => $latest ) );
 742                 }
 743                 wfProfileOut( __METHOD__ . '-' . $this->name );
 744                 wfProfileOut( __METHOD__ );
 745                 return $tmpFile;
 746         }
 747
 748         /**
 749          * @see FileBackend::streamFile()
 750          * @return Status
 751          */
 752         final public function streamFile( array $params ) {
 753                 wfProfileIn( __METHOD__ );
 754                 wfProfileIn( __METHOD__ . '-' . $this->name );
 755                 $status = Status::newGood();
 756
 757                 $info = $this->getFileStat( $params );
 758                 if ( !$info ) { // let StreamFile handle the 404
 759                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 760                 }
 761
 762                 // Set output buffer and HTTP headers for stream
 763                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 764                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 765                 if ( $res == StreamFile::NOT_MODIFIED ) {
 766                         // do nothing; client cache is up to date
 767                 } elseif ( $res == StreamFile::READY_STREAM ) {
 768                         wfProfileIn( __METHOD__ . '-send' );
 769                         wfProfileIn( __METHOD__ . '-send-' . $this->name );
 770                         $status = $this->doStreamFile( $params );
 771                         wfProfileOut( __METHOD__ . '-send-' . $this->name );
 772                         wfProfileOut( __METHOD__ . '-send' );
 773                 } else {
 774                         $status->fatal( 'backend-fail-stream', $params['src'] );
 775                 }
 776
 777                 wfProfileOut( __METHOD__ . '-' . $this->name );
 778                 wfProfileOut( __METHOD__ );
 779                 return $status;
 780         }
 781
 782         /**
 783          * @see FileBackendStore::streamFile()
 784          * @return Status
 785          */
 786         protected function doStreamFile( array $params ) {
 787                 $status = Status::newGood();
 788
 789                 $fsFile = $this->getLocalReference( $params );
 790                 if ( !$fsFile ) {
 791                         $status->fatal( 'backend-fail-stream', $params['src'] );
 792                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 793                         $status->fatal( 'backend-fail-stream', $params['src'] );
 794                 }
 795
 796                 return $status;
 797         }
 798
 799         /**
 800          * @see FileBackend::directoryExists()
 801          * @return bool|null
 802          */
 803         final public function directoryExists( array $params ) {
 804                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 805                 if ( $dir === null ) {
 806                         return false; // invalid storage path
 807                 }
 808                 if ( $shard !== null ) { // confined to a single container/shard
 809                         return $this->doDirectoryExists( $fullCont, $dir, $params );
 810                 } else { // directory is on several shards
 811                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 812                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 813                         $res = false; // response
 814                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 815                                 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
 816                                 if ( $exists ) {
 817                                         $res = true;
 818                                         break; // found one!
 819                                 } elseif ( $exists === null ) { // error?
 820                                         $res = null; // if we don't find anything, it is indeterminate
 821                                 }
 822                         }
 823                         return $res;
 824                 }
 825         }
 826
 827         /**
 828          * @see FileBackendStore::directoryExists()
 829          *
 830          * @param $container string Resolved container name
 831          * @param $dir string Resolved path relative to container
 832          * @param $params Array
 833          * @return bool|null
 834          */
 835         abstract protected function doDirectoryExists( $container, $dir, array $params );
 836
 837         /**
 838          * @see FileBackend::getDirectoryList()
 839          * @return Traversable|Array|null Returns null on failure
 840          */
 841         final public function getDirectoryList( array $params ) {
 842                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 843                 if ( $dir === null ) { // invalid storage path
 844                         return null;
 845                 }
 846                 if ( $shard !== null ) {
 847                         // File listing is confined to a single container/shard
 848                         return $this->getDirectoryListInternal( $fullCont, $dir, $params );
 849                 } else {
 850                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 851                         // File listing spans multiple containers/shards
 852                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 853                         return new FileBackendStoreShardDirIterator( $this,
 854                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 855                 }
 856         }
 857
 858         /**
 859          * Do not call this function from places outside FileBackend
 860          *
 861          * @see FileBackendStore::getDirectoryList()
 862          *
 863          * @param $container string Resolved container name
 864          * @param $dir string Resolved path relative to container
 865          * @param $params Array
 866          * @return Traversable|Array|null Returns null on failure
 867          */
 868         abstract public function getDirectoryListInternal( $container, $dir, array $params );
 869
 870         /**
 871          * @see FileBackend::getFileList()
 872          * @return Traversable|Array|null Returns null on failure
 873          */
 874         final public function getFileList( array $params ) {
 875                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 876                 if ( $dir === null ) { // invalid storage path
 877                         return null;
 878                 }
 879                 if ( $shard !== null ) {
 880                         // File listing is confined to a single container/shard
 881                         return $this->getFileListInternal( $fullCont, $dir, $params );
 882                 } else {
 883                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 884                         // File listing spans multiple containers/shards
 885                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 886                         return new FileBackendStoreShardFileIterator( $this,
 887                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 888                 }
 889         }
 890
 891         /**
 892          * Do not call this function from places outside FileBackend
 893          *
 894          * @see FileBackendStore::getFileList()
 895          *
 896          * @param $container string Resolved container name
 897          * @param $dir string Resolved path relative to container
 898          * @param $params Array
 899          * @return Traversable|Array|null Returns null on failure
 900          */
 901         abstract public function getFileListInternal( $container, $dir, array $params );
 902
 903         /**
 904          * Return a list of FileOp objects from a list of operations.
 905          * Do not call this function from places outside FileBackend.
 906          *
 907          * The result must have the same number of items as the input.
 908          * An exception is thrown if an unsupported operation is requested.
 909          *
 910          * @param $ops Array Same format as doOperations()
 911          * @return Array List of FileOp objects
 912          * @throws MWException
 913          */
 914         final public function getOperationsInternal( array $ops ) {
 915                 $supportedOps = array(
 916                         'store'       => 'StoreFileOp',
 917                         'copy'        => 'CopyFileOp',
 918                         'move'        => 'MoveFileOp',
 919                         'delete'      => 'DeleteFileOp',
 920                         'create'      => 'CreateFileOp',
 921                         'null'        => 'NullFileOp'
 922                 );
 923
 924                 $performOps = array(); // array of FileOp objects
 925                 // Build up ordered array of FileOps...
 926                 foreach ( $ops as $operation ) {
 927                         $opName = $operation['op'];
 928                         if ( isset( $supportedOps[$opName] ) ) {
 929                                 $class = $supportedOps[$opName];
 930                                 // Get params for this operation
 931                                 $params = $operation;
 932                                 // Append the FileOp class
 933                                 $performOps[] = new $class( $this, $params );
 934                         } else {
 935                                 throw new MWException( "Operation '$opName' is not supported." );
 936                         }
 937                 }
 938
 939                 return $performOps;
 940         }
 941
 942         /**
 943          * Get a list of storage paths to lock for a list of operations
 944          * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
 945          * each corresponding to a list of storage paths to be locked.
 946          *
 947          * @param $performOps Array List of FileOp objects
 948          * @return Array ('sh' => list of paths, 'ex' => list of paths)
 949          */
 950         final public function getPathsToLockForOpsInternal( array $performOps ) {
 951                 // Build up a list of files to lock...
 952                 $paths = array( 'sh' => array(), 'ex' => array() );
 953                 foreach ( $performOps as $fileOp ) {
 954                         $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
 955                         $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
 956                 }
 957                 // Optimization: if doing an EX lock anyway, don't also set an SH one
 958                 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
 959                 // Get a shared lock on the parent directory of each path changed
 960                 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
 961
 962                 return $paths;
 963         }
 964
 965         /**
 966          * @see FileBackend::getScopedLocksForOps()
 967          * @return Array
 968          */
 969         public function getScopedLocksForOps( array $ops, Status $status ) {
 970                 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
 971                 return array(
 972                         $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
 973                         $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
 974                 );
 975         }
 976
 977         /**
 978          * @see FileBackend::doOperationsInternal()
 979          * @return Status
 980          */
 981         final protected function doOperationsInternal( array $ops, array $opts ) {
 982                 wfProfileIn( __METHOD__ );
 983                 wfProfileIn( __METHOD__ . '-' . $this->name );
 984                 $status = Status::newGood();
 985
 986                 // Build up a list of FileOps...
 987                 $performOps = $this->getOperationsInternal( $ops );
 988
 989                 // Acquire any locks as needed...
 990                 if ( empty( $opts['nonLocking'] ) ) {
 991                         // Build up a list of files to lock...
 992                         $paths = $this->getPathsToLockForOpsInternal( $performOps );
 993                         // Try to lock those files for the scope of this function...
 994                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 995                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 996                         if ( !$status->isOK() ) {
 997                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 998                                 wfProfileOut( __METHOD__ );
 999                                 return $status; // abort
1000                         }
1001                 }
1002
1003                 // Clear any file cache entries (after locks acquired)
1004                 if ( empty( $opts['preserveCache'] ) ) {
1005                         $this->clearCache();
1006                 }
1007
1008                 // Load from the persistent file and container caches
1009                 $this->primeFileCache( $performOps );
1010                 $this->primeContainerCache( $performOps );
1011
1012                 // Actually attempt the operation batch...
1013                 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1014
1015                 // Merge errors into status fields
1016                 $status->merge( $subStatus );
1017                 $status->success = $subStatus->success; // not done in merge()
1018
1019                 wfProfileOut( __METHOD__ . '-' . $this->name );
1020                 wfProfileOut( __METHOD__ );
1021                 return $status;
1022         }
1023
1024         /**
1025          * @see FileBackend::doQuickOperationsInternal()
1026          * @return Status
1027          * @throws MWException
1028          */
1029         final protected function doQuickOperationsInternal( array $ops ) {
1030                 wfProfileIn( __METHOD__ );
1031                 wfProfileIn( __METHOD__ . '-' . $this->name );
1032                 $status = Status::newGood();
1033
1034                 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
1035                 $async = ( $this->parallelize === 'implicit' );
1036                 $maxConcurrency = $this->concurrency; // throttle
1037
1038                 $statuses = array(); // array of (index => Status)
1039                 $fileOpHandles = array(); // list of (index => handle) arrays
1040                 $curFileOpHandles = array(); // current handle batch
1041                 // Perform the sync-only ops and build up op handles for the async ops...
1042                 foreach ( $ops as $index => $params ) {
1043                         if ( !in_array( $params['op'], $supportedOps ) ) {
1044                                 wfProfileOut( __METHOD__ . '-' . $this->name );
1045                                 wfProfileOut( __METHOD__ );
1046                                 throw new MWException( "Operation '{$params['op']}' is not supported." );
1047                         }
1048                         $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1049                         $subStatus = $this->$method( array( 'async' => $async ) + $params );
1050                         if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1051                                 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1052                                         $fileOpHandles[] = $curFileOpHandles; // push this batch
1053                                         $curFileOpHandles = array();
1054                                 }
1055                                 $curFileOpHandles[$index] = $subStatus->value; // keep index
1056                         } else { // error or completed
1057                                 $statuses[$index] = $subStatus; // keep index
1058                         }
1059                 }
1060                 if ( count( $curFileOpHandles ) ) {
1061                         $fileOpHandles[] = $curFileOpHandles; // last batch
1062                 }
1063                 // Do all the async ops that can be done concurrently...
1064                 foreach ( $fileOpHandles as $fileHandleBatch ) {
1065                         $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1066                 }
1067                 // Marshall and merge all the responses...
1068                 foreach ( $statuses as $index => $subStatus ) {
1069                         $status->merge( $subStatus );
1070                         if ( $subStatus->isOK() ) {
1071                                 $status->success[$index] = true;
1072                                 ++$status->successCount;
1073                         } else {
1074                                 $status->success[$index] = false;
1075                                 ++$status->failCount;
1076                         }
1077                 }
1078
1079                 wfProfileOut( __METHOD__ . '-' . $this->name );
1080                 wfProfileOut( __METHOD__ );
1081                 return $status;
1082         }
1083
1084         /**
1085          * Execute a list of FileBackendStoreOpHandle handles in parallel.
1086          * The resulting Status object fields will correspond
1087          * to the order in which the handles where given.
1088          *
1089          * @param $handles Array List of FileBackendStoreOpHandle objects
1090          * @return Array Map of Status objects
1091          * @throws MWException
1092          */
1093         final public function executeOpHandlesInternal( array $fileOpHandles ) {
1094                 wfProfileIn( __METHOD__ );
1095                 wfProfileIn( __METHOD__ . '-' . $this->name );
1096                 foreach ( $fileOpHandles as $fileOpHandle ) {
1097                         if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1098                                 throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
1099                         } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1100                                 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
1101                         }
1102                 }
1103                 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1104                 foreach ( $fileOpHandles as $fileOpHandle ) {
1105                         $fileOpHandle->closeResources();
1106                 }
1107                 wfProfileOut( __METHOD__ . '-' . $this->name );
1108                 wfProfileOut( __METHOD__ );
1109                 return $res;
1110         }
1111
1112         /**
1113          * @see FileBackendStore::executeOpHandlesInternal()
1114          * @return Array List of corresponding Status objects
1115          */
1116         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1117                 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
1118                         throw new MWException( "This backend supports no asynchronous operations." );
1119                 }
1120                 return array();
1121         }
1122
1123         /**
1124          * @see FileBackend::preloadCache()
1125          */
1126         final public function preloadCache( array $paths ) {
1127                 $fullConts = array(); // full container names
1128                 foreach ( $paths as $path ) {
1129                         list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
1130                         $fullConts[] = $fullCont;
1131                 }
1132                 // Load from the persistent file and container caches
1133                 $this->primeContainerCache( $fullConts );
1134                 $this->primeFileCache( $paths );
1135         }
1136
1137         /**
1138          * @see FileBackend::clearCache()
1139          */
1140         final public function clearCache( array $paths = null ) {
1141                 if ( is_array( $paths ) ) {
1142                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1143                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
1144                 }
1145                 if ( $paths === null ) {
1146                         $this->cheapCache->clear();
1147                         $this->expensiveCache->clear();
1148                 } else {
1149                         foreach ( $paths as $path ) {
1150                                 $this->cheapCache->clear( $path );
1151                                 $this->expensiveCache->clear( $path );
1152                         }
1153                 }
1154                 $this->doClearCache( $paths );
1155         }
1156
1157         /**
1158          * Clears any additional stat caches for storage paths
1159          *
1160          * @see FileBackend::clearCache()
1161          *
1162          * @param $paths Array Storage paths (optional)
1163          * @return void
1164          */
1165         protected function doClearCache( array $paths = null ) {}
1166
1167         /**
1168          * Is this a key/value store where directories are just virtual?
1169          * Virtual directories exists in so much as files exists that are
1170          * prefixed with the directory path followed by a forward slash.
1171          *
1172          * @return bool
1173          */
1174         abstract protected function directoriesAreVirtual();
1175
1176         /**
1177          * Check if a container name is valid.
1178          * This checks for for length and illegal characters.
1179          *
1180          * @param $container string
1181          * @return bool
1182          */
1183         final protected static function isValidContainerName( $container ) {
1184                 // This accounts for Swift and S3 restrictions while leaving room
1185                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1186                 // This disallows directory separators or traversal characters.
1187                 // Note that matching strings URL encode to the same string;
1188                 // in Swift, the length restriction is *after* URL encoding.
1189                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1190         }
1191
1192         /**
1193          * Splits a storage path into an internal container name,
1194          * an internal relative file name, and a container shard suffix.
1195          * Any shard suffix is already appended to the internal container name.
1196          * This also checks that the storage path is valid and within this backend.
1197          *
1198          * If the container is sharded but a suffix could not be determined,
1199          * this means that the path can only refer to a directory and can only
1200          * be scanned by looking in all the container shards.
1201          *
1202          * @param $storagePath string
1203          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1204          */
1205         final protected function resolveStoragePath( $storagePath ) {
1206                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1207                 if ( $backend === $this->name ) { // must be for this backend
1208                         $relPath = self::normalizeContainerPath( $relPath );
1209                         if ( $relPath !== null ) {
1210                                 // Get shard for the normalized path if this container is sharded
1211                                 $cShard = $this->getContainerShard( $container, $relPath );
1212                                 // Validate and sanitize the relative path (backend-specific)
1213                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1214                                 if ( $relPath !== null ) {
1215                                         // Prepend any wiki ID prefix to the container name
1216                                         $container = $this->fullContainerName( $container );
1217                                         if ( self::isValidContainerName( $container ) ) {
1218                                                 // Validate and sanitize the container name (backend-specific)
1219                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1220                                                 if ( $container !== null ) {
1221                                                         return array( $container, $relPath, $cShard );
1222                                                 }
1223                                         }
1224                                 }
1225                         }
1226                 }
1227                 return array( null, null, null );
1228         }
1229
1230         /**
1231          * Like resolveStoragePath() except null values are returned if
1232          * the container is sharded and the shard could not be determined.
1233          *
1234          * @see FileBackendStore::resolveStoragePath()
1235          *
1236          * @param $storagePath string
1237          * @return Array (container, path) or (null, null) if invalid
1238          */
1239         final protected function resolveStoragePathReal( $storagePath ) {
1240                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1241                 if ( $cShard !== null ) {
1242                         return array( $container, $relPath );
1243                 }
1244                 return array( null, null );
1245         }
1246
1247         /**
1248          * Get the container name shard suffix for a given path.
1249          * Any empty suffix means the container is not sharded.
1250          *
1251          * @param $container string Container name
1252          * @param $relPath string Storage path relative to the container
1253          * @return string|null Returns null if shard could not be determined
1254          */
1255         final protected function getContainerShard( $container, $relPath ) {
1256                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1257                 if ( $levels == 1 || $levels == 2 ) {
1258                         // Hash characters are either base 16 or 36
1259                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1260                         // Get a regex that represents the shard portion of paths.
1261                         // The concatenation of the captures gives us the shard.
1262                         if ( $levels === 1 ) { // 16 or 36 shards per container
1263                                 $hashDirRegex = '(' . $char . ')';
1264                         } else { // 256 or 1296 shards per container
1265                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1266                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1267                                 } else { // short hash dir format (e.g. "a/b/c")
1268                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1269                                 }
1270                         }
1271                         // Allow certain directories to be above the hash dirs so as
1272                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1273                         // They must be 2+ chars to avoid any hash directory ambiguity.
1274                         $m = array();
1275                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1276                                 return '.' . implode( '', array_slice( $m, 1 ) );
1277                         }
1278                         return null; // failed to match
1279                 }
1280                 return ''; // no sharding
1281         }
1282
1283         /**
1284          * Check if a storage path maps to a single shard.
1285          * Container dirs like "a", where the container shards on "x/xy",
1286          * can reside on several shards. Such paths are tricky to handle.
1287          *
1288          * @param $storagePath string Storage path
1289          * @return bool
1290          */
1291         final public function isSingleShardPathInternal( $storagePath ) {
1292                 list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
1293                 return ( $shard !== null );
1294         }
1295
1296         /**
1297          * Get the sharding config for a container.
1298          * If greater than 0, then all file storage paths within
1299          * the container are required to be hashed accordingly.
1300          *
1301          * @param $container string
1302          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1303          */
1304         final protected function getContainerHashLevels( $container ) {
1305                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1306                         $config = $this->shardViaHashLevels[$container];
1307                         $hashLevels = (int)$config['levels'];
1308                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1309                                 $hashBase = (int)$config['base'];
1310                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1311                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1312                                 }
1313                         }
1314                 }
1315                 return array( 0, 0, false ); // no sharding
1316         }
1317
1318         /**
1319          * Get a list of full container shard suffixes for a container
1320          *
1321          * @param $container string
1322          * @return Array
1323          */
1324         final protected function getContainerSuffixes( $container ) {
1325                 $shards = array();
1326                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1327                 if ( $digits > 0 ) {
1328                         $numShards = pow( $base, $digits );
1329                         for ( $index = 0; $index < $numShards; $index++ ) {
1330                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1331                         }
1332                 }
1333                 return $shards;
1334         }
1335
1336         /**
1337          * Get the full container name, including the wiki ID prefix
1338          *
1339          * @param $container string
1340          * @return string
1341          */
1342         final protected function fullContainerName( $container ) {
1343                 if ( $this->wikiId != '' ) {
1344                         return "{$this->wikiId}-$container";
1345                 } else {
1346                         return $container;
1347                 }
1348         }
1349
1350         /**
1351          * Resolve a container name, checking if it's allowed by the backend.
1352          * This is intended for internal use, such as encoding illegal chars.
1353          * Subclasses can override this to be more restrictive.
1354          *
1355          * @param $container string
1356          * @return string|null
1357          */
1358         protected function resolveContainerName( $container ) {
1359                 return $container;
1360         }
1361
1362         /**
1363          * Resolve a relative storage path, checking if it's allowed by the backend.
1364          * This is intended for internal use, such as encoding illegal chars or perhaps
1365          * getting absolute paths (e.g. FS based backends). Note that the relative path
1366          * may be the empty string (e.g. the path is simply to the container).
1367          *
1368          * @param $container string Container name
1369          * @param $relStoragePath string Storage path relative to the container
1370          * @return string|null Path or null if not valid
1371          */
1372         protected function resolveContainerPath( $container, $relStoragePath ) {
1373                 return $relStoragePath;
1374         }
1375
1376         /**
1377          * Get the cache key for a container
1378          *
1379          * @param $container string Resolved container name
1380          * @return string
1381          */
1382         private function containerCacheKey( $container ) {
1383                 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1384         }
1385
1386         /**
1387          * Set the cached info for a container
1388          *
1389          * @param $container string Resolved container name
1390          * @param $val mixed Information to cache
1391          */
1392         final protected function setContainerCache( $container, $val ) {
1393                 $this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 );
1394         }
1395
1396         /**
1397          * Delete the cached info for a container.
1398          * The cache key is salted for a while to prevent race conditions.
1399          *
1400          * @param $container string Resolved container name
1401          */
1402         final protected function deleteContainerCache( $container ) {
1403                 if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
1404                         trigger_error( "Unable to delete stat cache for container $container." );
1405                 }
1406         }
1407
1408         /**
1409          * Do a batch lookup from cache for container stats for all containers
1410          * used in a list of container names, storage paths, or FileOp objects.
1411          *
1412          * @param $items Array
1413          * @return void
1414          */
1415         final protected function primeContainerCache( array $items ) {
1416                 wfProfileIn( __METHOD__ );
1417                 wfProfileIn( __METHOD__ . '-' . $this->name );
1418
1419                 $paths = array(); // list of storage paths
1420                 $contNames = array(); // (cache key => resolved container name)
1421                 // Get all the paths/containers from the items...
1422                 foreach ( $items as $item ) {
1423                         if ( $item instanceof FileOp ) {
1424                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1425                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1426                         } elseif ( self::isStoragePath( $item ) ) {
1427                                 $paths[] = $item;
1428                         } elseif ( is_string( $item ) ) { // full container name
1429                                 $contNames[$this->containerCacheKey( $item )] = $item;
1430                         }
1431                 }
1432                 // Get all the corresponding cache keys for paths...
1433                 foreach ( $paths as $path ) {
1434                         list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
1435                         if ( $fullCont !== null ) { // valid path for this backend
1436                                 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1437                         }
1438                 }
1439
1440                 $contInfo = array(); // (resolved container name => cache value)
1441                 // Get all cache entries for these container cache keys...
1442                 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1443                 foreach ( $values as $cacheKey => $val ) {
1444                         $contInfo[$contNames[$cacheKey]] = $val;
1445                 }
1446
1447                 // Populate the container process cache for the backend...
1448                 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1449
1450                 wfProfileOut( __METHOD__ . '-' . $this->name );
1451                 wfProfileOut( __METHOD__ );
1452         }
1453
1454         /**
1455          * Fill the backend-specific process cache given an array of
1456          * resolved container names and their corresponding cached info.
1457          * Only containers that actually exist should appear in the map.
1458          *
1459          * @param $containerInfo Array Map of resolved container names to cached info
1460          * @return void
1461          */
1462         protected function doPrimeContainerCache( array $containerInfo ) {}
1463
1464         /**
1465          * Get the cache key for a file path
1466          *
1467          * @param $path string Storage path
1468          * @return string
1469          */
1470         private function fileCacheKey( $path ) {
1471                 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1472         }
1473
1474         /**
1475          * Set the cached stat info for a file path.
1476          * Negatives (404s) are not cached. By not caching negatives, we can skip cache
1477          * salting for the case when a file is created at a path were there was none before.
1478          *
1479          * @param $path string Storage path
1480          * @param $val mixed Information to cache
1481          */
1482         final protected function setFileCache( $path, $val ) {
1483                 $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
1484         }
1485
1486         /**
1487          * Delete the cached stat info for a file path.
1488          * The cache key is salted for a while to prevent race conditions.
1489          *
1490          * @param $path string Storage path
1491          */
1492         final protected function deleteFileCache( $path ) {
1493                 if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
1494                         trigger_error( "Unable to delete stat cache for file $path." );
1495                 }
1496         }
1497
1498         /**
1499          * Do a batch lookup from cache for file stats for all paths
1500          * used in a list of storage paths or FileOp objects.
1501          *
1502          * @param $items Array List of storage paths or FileOps
1503          * @return void
1504          */
1505         final protected function primeFileCache( array $items ) {
1506                 wfProfileIn( __METHOD__ );
1507                 wfProfileIn( __METHOD__ . '-' . $this->name );
1508
1509                 $paths = array(); // list of storage paths
1510                 $pathNames = array(); // (cache key => storage path)
1511                 // Get all the paths/containers from the items...
1512                 foreach ( $items as $item ) {
1513                         if ( $item instanceof FileOp ) {
1514                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1515                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1516                         } elseif ( self::isStoragePath( $item ) ) {
1517                                 $paths[] = $item;
1518                         }
1519                 }
1520                 // Get all the corresponding cache keys for paths...
1521                 foreach ( $paths as $path ) {
1522                         list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
1523                         if ( $rel !== null ) { // valid path for this backend
1524                                 $pathNames[$this->fileCacheKey( $path )] = $path;
1525                         }
1526                 }
1527                 // Get all cache entries for these container cache keys...
1528                 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1529                 foreach ( $values as $cacheKey => $val ) {
1530                         if ( is_array( $val ) ) {
1531                                 $path = $pathNames[$cacheKey];
1532                                 $this->cheapCache->set( $path, 'stat', $val );
1533                                 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1534                                         $this->cheapCache->set( $path, 'sha1',
1535                                                 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
1536                                 }
1537                         }
1538                 }
1539
1540                 wfProfileOut( __METHOD__ . '-' . $this->name );
1541                 wfProfileOut( __METHOD__ );
1542         }
1543 }
1544
1545 /**
1546  * FileBackendStore helper class for performing asynchronous file operations.
1547  *
1548  * For example, calling FileBackendStore::createInternal() with the "async"
1549  * param flag may result in a Status that contains this object as a value.
1550  * This class is largely backend-specific and is mostly just "magic" to be
1551  * passed to FileBackendStore::executeOpHandlesInternal().
1552  */
1553 abstract class FileBackendStoreOpHandle {
1554         /** @var Array */
1555         public $params = array(); // params to caller functions
1556         /** @var FileBackendStore */
1557         public $backend;
1558         /** @var Array */
1559         public $resourcesToClose = array();
1560
1561         public $call; // string; name that identifies the function called
1562
1563         /**
1564          * Close all open file handles
1565          *
1566          * @return void
1567          */
1568         public function closeResources() {
1569                 array_map( 'fclose', $this->resourcesToClose );
1570         }
1571 }
1572
1573 /**
1574  * FileBackendStore helper function to handle listings that span container shards.
1575  * Do not use this class from places outside of FileBackendStore.
1576  *
1577  * @ingroup FileBackend
1578  */
1579 abstract class FileBackendStoreShardListIterator implements Iterator {
1580         /** @var FileBackendStore */
1581         protected $backend;
1582         /** @var Array */
1583         protected $params;
1584         /** @var Array */
1585         protected $shardSuffixes;
1586         protected $container; // string; full container name
1587         protected $directory; // string; resolved relative path
1588
1589         /** @var Traversable */
1590         protected $iter;
1591         protected $curShard = 0; // integer
1592         protected $pos = 0; // integer
1593
1594         /** @var Array */
1595         protected $multiShardPaths = array(); // (rel path => 1)
1596
1597         /**
1598          * @param $backend FileBackendStore
1599          * @param $container string Full storage container name
1600          * @param $dir string Storage directory relative to container
1601          * @param $suffixes Array List of container shard suffixes
1602          * @param $params Array
1603          */
1604         public function __construct(
1605                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1606         ) {
1607                 $this->backend = $backend;
1608                 $this->container = $container;
1609                 $this->directory = $dir;
1610                 $this->shardSuffixes = $suffixes;
1611                 $this->params = $params;
1612         }
1613
1614         /**
1615          * @see Iterator::key()
1616          * @return integer
1617          */
1618         public function key() {
1619                 return $this->pos;
1620         }
1621
1622         /**
1623          * @see Iterator::valid()
1624          * @return bool
1625          */
1626         public function valid() {
1627                 if ( $this->iter instanceof Iterator ) {
1628                         return $this->iter->valid();
1629                 } elseif ( is_array( $this->iter ) ) {
1630                         return ( current( $this->iter ) !== false ); // no paths can have this value
1631                 }
1632                 return false; // some failure?
1633         }
1634
1635         /**
1636          * @see Iterator::current()
1637          * @return string|bool String or false
1638          */
1639         public function current() {
1640                 return ( $this->iter instanceof Iterator )
1641                         ? $this->iter->current()
1642                         : current( $this->iter );
1643         }
1644
1645         /**
1646          * @see Iterator::next()
1647          * @return void
1648          */
1649         public function next() {
1650                 ++$this->pos;
1651                 ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
1652                 do {
1653                         $continue = false; // keep scanning shards?
1654                         $this->filterViaNext(); // filter out duplicates
1655                         // Find the next non-empty shard if no elements are left
1656                         if ( !$this->valid() ) {
1657                                 $this->nextShardIteratorIfNotValid();
1658                                 $continue = $this->valid(); // re-filter unless we ran out of shards
1659                         }
1660                 } while ( $continue );
1661         }
1662
1663         /**
1664          * @see Iterator::rewind()
1665          * @return void
1666          */
1667         public function rewind() {
1668                 $this->pos = 0;
1669                 $this->curShard = 0;
1670                 $this->setIteratorFromCurrentShard();
1671                 do {
1672                         $continue = false; // keep scanning shards?
1673                         $this->filterViaNext(); // filter out duplicates
1674                         // Find the next non-empty shard if no elements are left
1675                         if ( !$this->valid() ) {
1676                                 $this->nextShardIteratorIfNotValid();
1677                                 $continue = $this->valid(); // re-filter unless we ran out of shards
1678                         }
1679                 } while ( $continue );
1680         }
1681
1682         /**
1683          * Filter out duplicate items by advancing to the next ones
1684          */
1685         protected function filterViaNext() {
1686                 while ( $this->valid() ) {
1687                         $rel = $this->iter->current(); // path relative to given directory
1688                         $path = $this->params['dir'] . "/{$rel}"; // full storage path
1689                         if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1690                                 break; // path is only on one shard; no issue with duplicates
1691                         } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1692                                 // Don't keep listing paths that are on multiple shards
1693                                 ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
1694                         } else {
1695                                 $this->multiShardPaths[$rel] = 1;
1696                                 break;
1697                         }
1698                 }
1699         }
1700
1701         /**
1702          * If the list iterator for this container shard is out of items,
1703          * then move on to the next container that has items.
1704          * If there are none, then it advances to the last container.
1705          */
1706         protected function nextShardIteratorIfNotValid() {
1707                 while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) {
1708                         $this->setIteratorFromCurrentShard();
1709                 }
1710         }
1711
1712         /**
1713          * Set the list iterator to that of the current container shard
1714          */
1715         protected function setIteratorFromCurrentShard() {
1716                 $this->iter = $this->listFromShard(
1717                         $this->container . $this->shardSuffixes[$this->curShard],
1718                         $this->directory, $this->params );
1719                 // Start loading results so that current() works
1720                 if ( $this->iter ) {
1721                         ( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter );
1722                 }
1723         }
1724
1725         /**
1726          * Get the list for a given container shard
1727          *
1728          * @param $container string Resolved container name
1729          * @param $dir string Resolved path relative to container
1730          * @param $params Array
1731          * @return Traversable|Array|null
1732          */
1733         abstract protected function listFromShard( $container, $dir, array $params );
1734 }
1735
1736 /**
1737  * Iterator for listing directories
1738  */
1739 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1740         /**
1741          * @see FileBackendStoreShardListIterator::listFromShard()
1742          * @return Array|null|Traversable
1743          */
1744         protected function listFromShard( $container, $dir, array $params ) {
1745                 return $this->backend->getDirectoryListInternal( $container, $dir, $params );
1746         }
1747 }
1748
1749 /**
1750  * Iterator for listing regular files
1751  */
1752 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1753         /**
1754          * @see FileBackendStoreShardListIterator::listFromShard()
1755          * @return Array|null|Traversable
1756          */
1757         protected function listFromShard( $container, $dir, array $params ) {
1758                 return $this->backend->getFileListInternal( $container, $dir, $params );
1759         }
1760 }