includes/filebackend/FileBackendStore.php

   1 <?php
   2 /**
   3  * Base class for all backends using particular storage medium.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Base class for all backends using particular storage medium.
  27  *
  28  * This class defines the methods as abstract that subclasses must implement.
  29  * Outside callers should *not* use functions with "Internal" in the name.
  30  *
  31  * The FileBackend operations are implemented using basic functions
  32  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  33  * This class is also responsible for path resolution and sanitization.
  34  *
  35  * @ingroup FileBackend
  36  * @since 1.19
  37  */
  38 abstract class FileBackendStore extends FileBackend {
  39         /** @var BagOStuff */
  40         protected $memCache;
  41         /** @var ProcessCacheLRU */
  42         protected $cheapCache; // Map of paths to small (RAM/disk) cache items
  43         /** @var ProcessCacheLRU */
  44         protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
  45
  46         /** @var Array Map of container names to sharding settings */
  47         protected $shardViaHashLevels = array(); // (container name => config array)
  48
  49         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  50
  51         /**
  52          * @see FileBackend::__construct()
  53          *
  54          * @param $config Array
  55          */
  56         public function __construct( array $config ) {
  57                 parent::__construct( $config );
  58                 $this->memCache       = new EmptyBagOStuff(); // disabled by default
  59                 $this->cheapCache     = new ProcessCacheLRU( 300 );
  60                 $this->expensiveCache = new ProcessCacheLRU( 5 );
  61         }
  62
  63         /**
  64          * Get the maximum allowable file size given backend
  65          * medium restrictions and basic performance constraints.
  66          * Do not call this function from places outside FileBackend and FileOp.
  67          *
  68          * @return integer Bytes
  69          */
  70         final public function maxFileSizeInternal() {
  71                 return $this->maxFileSize;
  72         }
  73
  74         /**
  75          * Check if a file can be created at a given storage path.
  76          * FS backends should check if the parent directory exists and the file is writable.
  77          * Backends using key/value stores should check if the container exists.
  78          *
  79          * @param $storagePath string
  80          * @return bool
  81          */
  82         abstract public function isPathUsableInternal( $storagePath );
  83
  84         /**
  85          * Create a file in the backend with the given contents.
  86          * Do not call this function from places outside FileBackend and FileOp.
  87          *
  88          * $params include:
  89          *   - content       : the raw file contents
  90          *   - dst           : destination storage path
  91          *   - overwrite     : overwrite any file that exists at the destination
  92          *   - async         : Status will be returned immediately if supported.
  93          *                     If the status is OK, then its value field will be
  94          *                     set to a FileBackendStoreOpHandle object.
  95          *
  96          * @param $params Array
  97          * @return Status
  98          */
  99         final public function createInternal( array $params ) {
 100                 wfProfileIn( __METHOD__ );
 101                 wfProfileIn( __METHOD__ . '-' . $this->name );
 102                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 103                         $status = Status::newFatal( 'backend-fail-maxsize',
 104                                 $params['dst'], $this->maxFileSizeInternal() );
 105                 } else {
 106                         $status = $this->doCreateInternal( $params );
 107                         $this->clearCache( array( $params['dst'] ) );
 108                         if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 109                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 110                         }
 111                 }
 112                 wfProfileOut( __METHOD__ . '-' . $this->name );
 113                 wfProfileOut( __METHOD__ );
 114                 return $status;
 115         }
 116
 117         /**
 118          * @see FileBackendStore::createInternal()
 119          */
 120         abstract protected function doCreateInternal( array $params );
 121
 122         /**
 123          * Store a file into the backend from a file on disk.
 124          * Do not call this function from places outside FileBackend and FileOp.
 125          *
 126          * $params include:
 127          *   - src           : source path on disk
 128          *   - dst           : destination storage path
 129          *   - overwrite     : overwrite any file that exists at the destination
 130          *   - async         : Status will be returned immediately if supported.
 131          *                     If the status is OK, then its value field will be
 132          *                     set to a FileBackendStoreOpHandle object.
 133          *
 134          * @param $params Array
 135          * @return Status
 136          */
 137         final public function storeInternal( array $params ) {
 138                 wfProfileIn( __METHOD__ );
 139                 wfProfileIn( __METHOD__ . '-' . $this->name );
 140                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 141                         $status = Status::newFatal( 'backend-fail-maxsize',
 142                                 $params['dst'], $this->maxFileSizeInternal() );
 143                 } else {
 144                         $status = $this->doStoreInternal( $params );
 145                         $this->clearCache( array( $params['dst'] ) );
 146                         if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 147                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 148                         }
 149                 }
 150                 wfProfileOut( __METHOD__ . '-' . $this->name );
 151                 wfProfileOut( __METHOD__ );
 152                 return $status;
 153         }
 154
 155         /**
 156          * @see FileBackendStore::storeInternal()
 157          */
 158         abstract protected function doStoreInternal( array $params );
 159
 160         /**
 161          * Copy a file from one storage path to another in the backend.
 162          * Do not call this function from places outside FileBackend and FileOp.
 163          *
 164          * $params include:
 165          *   - src           : source storage path
 166          *   - dst           : destination storage path
 167          *   - overwrite     : overwrite any file that exists at the destination
 168          *   - async         : Status will be returned immediately if supported.
 169          *                     If the status is OK, then its value field will be
 170          *                     set to a FileBackendStoreOpHandle object.
 171          *
 172          * @param $params Array
 173          * @return Status
 174          */
 175         final public function copyInternal( array $params ) {
 176                 wfProfileIn( __METHOD__ );
 177                 wfProfileIn( __METHOD__ . '-' . $this->name );
 178                 $status = $this->doCopyInternal( $params );
 179                 $this->clearCache( array( $params['dst'] ) );
 180                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 181                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 182                 }
 183                 wfProfileOut( __METHOD__ . '-' . $this->name );
 184                 wfProfileOut( __METHOD__ );
 185                 return $status;
 186         }
 187
 188         /**
 189          * @see FileBackendStore::copyInternal()
 190          */
 191         abstract protected function doCopyInternal( array $params );
 192
 193         /**
 194          * Delete a file at the storage path.
 195          * Do not call this function from places outside FileBackend and FileOp.
 196          *
 197          * $params include:
 198          *   - src                 : source storage path
 199          *   - ignoreMissingSource : do nothing if the source file does not exist
 200          *   - async               : Status will be returned immediately if supported.
 201          *                           If the status is OK, then its value field will be
 202          *                           set to a FileBackendStoreOpHandle object.
 203          *
 204          * @param $params Array
 205          * @return Status
 206          */
 207         final public function deleteInternal( array $params ) {
 208                 wfProfileIn( __METHOD__ );
 209                 wfProfileIn( __METHOD__ . '-' . $this->name );
 210                 $status = $this->doDeleteInternal( $params );
 211                 $this->clearCache( array( $params['src'] ) );
 212                 $this->deleteFileCache( $params['src'] ); // persistent cache
 213                 wfProfileOut( __METHOD__ . '-' . $this->name );
 214                 wfProfileOut( __METHOD__ );
 215                 return $status;
 216         }
 217
 218         /**
 219          * @see FileBackendStore::deleteInternal()
 220          */
 221         abstract protected function doDeleteInternal( array $params );
 222
 223         /**
 224          * Move a file from one storage path to another in the backend.
 225          * Do not call this function from places outside FileBackend and FileOp.
 226          *
 227          * $params include:
 228          *   - src           : source storage path
 229          *   - dst           : destination storage path
 230          *   - overwrite     : overwrite any file that exists at the destination
 231          *   - async         : Status will be returned immediately if supported.
 232          *                     If the status is OK, then its value field will be
 233          *                     set to a FileBackendStoreOpHandle object.
 234          *
 235          * @param $params Array
 236          * @return Status
 237          */
 238         final public function moveInternal( array $params ) {
 239                 wfProfileIn( __METHOD__ );
 240                 wfProfileIn( __METHOD__ . '-' . $this->name );
 241                 $status = $this->doMoveInternal( $params );
 242                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 243                 $this->deleteFileCache( $params['src'] ); // persistent cache
 244                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 245                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 246                 }
 247                 wfProfileOut( __METHOD__ . '-' . $this->name );
 248                 wfProfileOut( __METHOD__ );
 249                 return $status;
 250         }
 251
 252         /**
 253          * @see FileBackendStore::moveInternal()
 254          * @return Status
 255          */
 256         protected function doMoveInternal( array $params ) {
 257                 unset( $params['async'] ); // two steps, won't work here :)
 258                 // Copy source to dest
 259                 $status = $this->copyInternal( $params );
 260                 if ( $status->isOK() ) {
 261                         // Delete source (only fails due to races or medium going down)
 262                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 263                         $status->setResult( true, $status->value ); // ignore delete() errors
 264                 }
 265                 return $status;
 266         }
 267
 268         /**
 269          * No-op file operation that does nothing.
 270          * Do not call this function from places outside FileBackend and FileOp.
 271          *
 272          * @param $params Array
 273          * @return Status
 274          */
 275         final public function nullInternal( array $params ) {
 276                 return Status::newGood();
 277         }
 278
 279         /**
 280          * @see FileBackend::concatenate()
 281          * @return Status
 282          */
 283         final public function concatenate( array $params ) {
 284                 wfProfileIn( __METHOD__ );
 285                 wfProfileIn( __METHOD__ . '-' . $this->name );
 286                 $status = Status::newGood();
 287
 288                 // Try to lock the source files for the scope of this function
 289                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 290                 if ( $status->isOK() ) {
 291                         // Actually do the concatenation
 292                         $status->merge( $this->doConcatenate( $params ) );
 293                 }
 294
 295                 wfProfileOut( __METHOD__ . '-' . $this->name );
 296                 wfProfileOut( __METHOD__ );
 297                 return $status;
 298         }
 299
 300         /**
 301          * @see FileBackendStore::concatenate()
 302          * @return Status
 303          */
 304         protected function doConcatenate( array $params ) {
 305                 $status = Status::newGood();
 306                 $tmpPath = $params['dst']; // convenience
 307
 308                 // Check that the specified temp file is valid...
 309                 wfSuppressWarnings();
 310                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 311                 wfRestoreWarnings();
 312                 if ( !$ok ) { // not present or not empty
 313                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 314                         return $status;
 315                 }
 316
 317                 // Build up the temp file using the source chunks (in order)...
 318                 $tmpHandle = fopen( $tmpPath, 'ab' );
 319                 if ( $tmpHandle === false ) {
 320                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 321                         return $status;
 322                 }
 323                 foreach ( $params['srcs'] as $virtualSource ) {
 324                         // Get a local FS version of the chunk
 325                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 326                         if ( !$tmpFile ) {
 327                                 $status->fatal( 'backend-fail-read', $virtualSource );
 328                                 return $status;
 329                         }
 330                         // Get a handle to the local FS version
 331                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 332                         if ( $sourceHandle === false ) {
 333                                 fclose( $tmpHandle );
 334                                 $status->fatal( 'backend-fail-read', $virtualSource );
 335                                 return $status;
 336                         }
 337                         // Append chunk to file (pass chunk size to avoid magic quotes)
 338                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 339                                 fclose( $sourceHandle );
 340                                 fclose( $tmpHandle );
 341                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 342                                 return $status;
 343                         }
 344                         fclose( $sourceHandle );
 345                 }
 346                 if ( !fclose( $tmpHandle ) ) {
 347                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 348                         return $status;
 349                 }
 350
 351                 clearstatcache(); // temp file changed
 352
 353                 return $status;
 354         }
 355
 356         /**
 357          * @see FileBackend::doPrepare()
 358          * @return Status
 359          */
 360         final protected function doPrepare( array $params ) {
 361                 wfProfileIn( __METHOD__ );
 362                 wfProfileIn( __METHOD__ . '-' . $this->name );
 363
 364                 $status = Status::newGood();
 365                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 366                 if ( $dir === null ) {
 367                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 368                         wfProfileOut( __METHOD__ . '-' . $this->name );
 369                         wfProfileOut( __METHOD__ );
 370                         return $status; // invalid storage path
 371                 }
 372
 373                 if ( $shard !== null ) { // confined to a single container/shard
 374                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 375                 } else { // directory is on several shards
 376                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 377                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 378                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 379                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 380                         }
 381                 }
 382
 383                 wfProfileOut( __METHOD__ . '-' . $this->name );
 384                 wfProfileOut( __METHOD__ );
 385                 return $status;
 386         }
 387
 388         /**
 389          * @see FileBackendStore::doPrepare()
 390          * @return Status
 391          */
 392         protected function doPrepareInternal( $container, $dir, array $params ) {
 393                 return Status::newGood();
 394         }
 395
 396         /**
 397          * @see FileBackend::doSecure()
 398          * @return Status
 399          */
 400         final protected function doSecure( array $params ) {
 401                 wfProfileIn( __METHOD__ );
 402                 wfProfileIn( __METHOD__ . '-' . $this->name );
 403                 $status = Status::newGood();
 404
 405                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 406                 if ( $dir === null ) {
 407                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 408                         wfProfileOut( __METHOD__ . '-' . $this->name );
 409                         wfProfileOut( __METHOD__ );
 410                         return $status; // invalid storage path
 411                 }
 412
 413                 if ( $shard !== null ) { // confined to a single container/shard
 414                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 415                 } else { // directory is on several shards
 416                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 417                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 418                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 419                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 420                         }
 421                 }
 422
 423                 wfProfileOut( __METHOD__ . '-' . $this->name );
 424                 wfProfileOut( __METHOD__ );
 425                 return $status;
 426         }
 427
 428         /**
 429          * @see FileBackendStore::doSecure()
 430          * @return Status
 431          */
 432         protected function doSecureInternal( $container, $dir, array $params ) {
 433                 return Status::newGood();
 434         }
 435
 436         /**
 437          * @see FileBackend::doPublish()
 438          * @return Status
 439          */
 440         final protected function doPublish( array $params ) {
 441                 wfProfileIn( __METHOD__ );
 442                 wfProfileIn( __METHOD__ . '-' . $this->name );
 443                 $status = Status::newGood();
 444
 445                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 446                 if ( $dir === null ) {
 447                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 448                         wfProfileOut( __METHOD__ . '-' . $this->name );
 449                         wfProfileOut( __METHOD__ );
 450                         return $status; // invalid storage path
 451                 }
 452
 453                 if ( $shard !== null ) { // confined to a single container/shard
 454                         $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 455                 } else { // directory is on several shards
 456                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 457                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 458                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 459                                 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 460                         }
 461                 }
 462
 463                 wfProfileOut( __METHOD__ . '-' . $this->name );
 464                 wfProfileOut( __METHOD__ );
 465                 return $status;
 466         }
 467
 468         /**
 469          * @see FileBackendStore::doPublish()
 470          * @return Status
 471          */
 472         protected function doPublishInternal( $container, $dir, array $params ) {
 473                 return Status::newGood();
 474         }
 475
 476         /**
 477          * @see FileBackend::doClean()
 478          * @return Status
 479          */
 480         final protected function doClean( array $params ) {
 481                 wfProfileIn( __METHOD__ );
 482                 wfProfileIn( __METHOD__ . '-' . $this->name );
 483                 $status = Status::newGood();
 484
 485                 // Recursive: first delete all empty subdirs recursively
 486                 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
 487                         $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
 488                         if ( $subDirsRel !== null ) { // no errors
 489                                 foreach ( $subDirsRel as $subDirRel ) {
 490                                         $subDir = $params['dir'] . "/{$subDirRel}"; // full path
 491                                         $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
 492                                 }
 493                         }
 494                 }
 495
 496                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 497                 if ( $dir === null ) {
 498                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 499                         wfProfileOut( __METHOD__ . '-' . $this->name );
 500                         wfProfileOut( __METHOD__ );
 501                         return $status; // invalid storage path
 502                 }
 503
 504                 // Attempt to lock this directory...
 505                 $filesLockEx = array( $params['dir'] );
 506                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 507                 if ( !$status->isOK() ) {
 508                         wfProfileOut( __METHOD__ . '-' . $this->name );
 509                         wfProfileOut( __METHOD__ );
 510                         return $status; // abort
 511                 }
 512
 513                 if ( $shard !== null ) { // confined to a single container/shard
 514                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 515                         $this->deleteContainerCache( $fullCont ); // purge cache
 516                 } else { // directory is on several shards
 517                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 518                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 519                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 520                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 521                                 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
 522                         }
 523                 }
 524
 525                 wfProfileOut( __METHOD__ . '-' . $this->name );
 526                 wfProfileOut( __METHOD__ );
 527                 return $status;
 528         }
 529
 530         /**
 531          * @see FileBackendStore::doClean()
 532          * @return Status
 533          */
 534         protected function doCleanInternal( $container, $dir, array $params ) {
 535                 return Status::newGood();
 536         }
 537
 538         /**
 539          * @see FileBackend::fileExists()
 540          * @return bool|null
 541          */
 542         final public function fileExists( array $params ) {
 543                 wfProfileIn( __METHOD__ );
 544                 wfProfileIn( __METHOD__ . '-' . $this->name );
 545                 $stat = $this->getFileStat( $params );
 546                 wfProfileOut( __METHOD__ . '-' . $this->name );
 547                 wfProfileOut( __METHOD__ );
 548                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 549         }
 550
 551         /**
 552          * @see FileBackend::getFileTimestamp()
 553          * @return bool
 554          */
 555         final public function getFileTimestamp( array $params ) {
 556                 wfProfileIn( __METHOD__ );
 557                 wfProfileIn( __METHOD__ . '-' . $this->name );
 558                 $stat = $this->getFileStat( $params );
 559                 wfProfileOut( __METHOD__ . '-' . $this->name );
 560                 wfProfileOut( __METHOD__ );
 561                 return $stat ? $stat['mtime'] : false;
 562         }
 563
 564         /**
 565          * @see FileBackend::getFileSize()
 566          * @return bool
 567          */
 568         final public function getFileSize( array $params ) {
 569                 wfProfileIn( __METHOD__ );
 570                 wfProfileIn( __METHOD__ . '-' . $this->name );
 571                 $stat = $this->getFileStat( $params );
 572                 wfProfileOut( __METHOD__ . '-' . $this->name );
 573                 wfProfileOut( __METHOD__ );
 574                 return $stat ? $stat['size'] : false;
 575         }
 576
 577         /**
 578          * @see FileBackend::getFileStat()
 579          * @return bool
 580          */
 581         final public function getFileStat( array $params ) {
 582                 $path = self::normalizeStoragePath( $params['src'] );
 583                 if ( $path === null ) {
 584                         return false; // invalid storage path
 585                 }
 586                 wfProfileIn( __METHOD__ );
 587                 wfProfileIn( __METHOD__ . '-' . $this->name );
 588                 $latest = !empty( $params['latest'] ); // use latest data?
 589                 if ( !$this->cheapCache->has( $path, 'stat' ) ) {
 590                         $this->primeFileCache( array( $path ) ); // check persistent cache
 591                 }
 592                 if ( $this->cheapCache->has( $path, 'stat' ) ) {
 593                         $stat = $this->cheapCache->get( $path, 'stat' );
 594                         // If we want the latest data, check that this cached
 595                         // value was in fact fetched with the latest available data.
 596                         if ( !$latest || $stat['latest'] ) {
 597                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 598                                 wfProfileOut( __METHOD__ );
 599                                 return $stat;
 600                         }
 601                 }
 602                 wfProfileIn( __METHOD__ . '-miss' );
 603                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 604                 $stat = $this->doGetFileStat( $params );
 605                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 606                 wfProfileOut( __METHOD__ . '-miss' );
 607                 if ( is_array( $stat ) ) { // don't cache negatives
 608                         $stat['latest'] = $latest;
 609                         $this->cheapCache->set( $path, 'stat', $stat );
 610                         $this->setFileCache( $path, $stat ); // update persistent cache
 611                         if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
 612                                 $this->cheapCache->set( $path, 'sha1',
 613                                         array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
 614                         }
 615                 } else {
 616                         wfDebug( __METHOD__ . ": File $path does not exist.\n" );
 617                 }
 618                 wfProfileOut( __METHOD__ . '-' . $this->name );
 619                 wfProfileOut( __METHOD__ );
 620                 return $stat;
 621         }
 622
 623         /**
 624          * @see FileBackendStore::getFileStat()
 625          */
 626         abstract protected function doGetFileStat( array $params );
 627
 628         /**
 629          * @see FileBackend::getFileContents()
 630          * @return bool|string
 631          */
 632         public function getFileContents( array $params ) {
 633                 wfProfileIn( __METHOD__ );
 634                 wfProfileIn( __METHOD__ . '-' . $this->name );
 635                 $tmpFile = $this->getLocalReference( $params );
 636                 if ( !$tmpFile ) {
 637                         wfProfileOut( __METHOD__ . '-' . $this->name );
 638                         wfProfileOut( __METHOD__ );
 639                         return false;
 640                 }
 641                 wfSuppressWarnings();
 642                 $data = file_get_contents( $tmpFile->getPath() );
 643                 wfRestoreWarnings();
 644                 wfProfileOut( __METHOD__ . '-' . $this->name );
 645                 wfProfileOut( __METHOD__ );
 646                 return $data;
 647         }
 648
 649         /**
 650          * @see FileBackend::getFileSha1Base36()
 651          * @return bool|string
 652          */
 653         final public function getFileSha1Base36( array $params ) {
 654                 $path = self::normalizeStoragePath( $params['src'] );
 655                 if ( $path === null ) {
 656                         return false; // invalid storage path
 657                 }
 658                 wfProfileIn( __METHOD__ );
 659                 wfProfileIn( __METHOD__ . '-' . $this->name );
 660                 $latest = !empty( $params['latest'] ); // use latest data?
 661                 if ( $this->cheapCache->has( $path, 'sha1' ) ) {
 662                         $stat = $this->cheapCache->get( $path, 'sha1' );
 663                         // If we want the latest data, check that this cached
 664                         // value was in fact fetched with the latest available data.
 665                         if ( !$latest || $stat['latest'] ) {
 666                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 667                                 wfProfileOut( __METHOD__ );
 668                                 return $stat['hash'];
 669                         }
 670                 }
 671                 wfProfileIn( __METHOD__ . '-miss' );
 672                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 673                 $hash = $this->doGetFileSha1Base36( $params );
 674                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 675                 wfProfileOut( __METHOD__ . '-miss' );
 676                 if ( $hash ) { // don't cache negatives
 677                         $this->cheapCache->set( $path, 'sha1',
 678                                 array( 'hash' => $hash, 'latest' => $latest ) );
 679                 }
 680                 wfProfileOut( __METHOD__ . '-' . $this->name );
 681                 wfProfileOut( __METHOD__ );
 682                 return $hash;
 683         }
 684
 685         /**
 686          * @see FileBackendStore::getFileSha1Base36()
 687          * @return bool|string
 688          */
 689         protected function doGetFileSha1Base36( array $params ) {
 690                 $fsFile = $this->getLocalReference( $params );
 691                 if ( !$fsFile ) {
 692                         return false;
 693                 } else {
 694                         return $fsFile->getSha1Base36();
 695                 }
 696         }
 697
 698         /**
 699          * @see FileBackend::getFileProps()
 700          * @return Array
 701          */
 702         final public function getFileProps( array $params ) {
 703                 wfProfileIn( __METHOD__ );
 704                 wfProfileIn( __METHOD__ . '-' . $this->name );
 705                 $fsFile = $this->getLocalReference( $params );
 706                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 707                 wfProfileOut( __METHOD__ . '-' . $this->name );
 708                 wfProfileOut( __METHOD__ );
 709                 return $props;
 710         }
 711
 712         /**
 713          * @see FileBackend::getLocalReference()
 714          * @return TempFSFile|null
 715          */
 716         public function getLocalReference( array $params ) {
 717                 $path = self::normalizeStoragePath( $params['src'] );
 718                 if ( $path === null ) {
 719                         return null; // invalid storage path
 720                 }
 721                 wfProfileIn( __METHOD__ );
 722                 wfProfileIn( __METHOD__ . '-' . $this->name );
 723                 $latest = !empty( $params['latest'] ); // use latest data?
 724                 if ( $this->expensiveCache->has( $path, 'localRef' ) ) {
 725                         $val = $this->expensiveCache->get( $path, 'localRef' );
 726                         // If we want the latest data, check that this cached
 727                         // value was in fact fetched with the latest available data.
 728                         if ( !$latest || $val['latest'] ) {
 729                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 730                                 wfProfileOut( __METHOD__ );
 731                                 return $val['object'];
 732                         }
 733                 }
 734                 $tmpFile = $this->getLocalCopy( $params );
 735                 if ( $tmpFile ) { // don't cache negatives
 736                         $this->expensiveCache->set( $path, 'localRef',
 737                                 array( 'object' => $tmpFile, 'latest' => $latest ) );
 738                 }
 739                 wfProfileOut( __METHOD__ . '-' . $this->name );
 740                 wfProfileOut( __METHOD__ );
 741                 return $tmpFile;
 742         }
 743
 744         /**
 745          * @see FileBackend::streamFile()
 746          * @return Status
 747          */
 748         final public function streamFile( array $params ) {
 749                 wfProfileIn( __METHOD__ );
 750                 wfProfileIn( __METHOD__ . '-' . $this->name );
 751                 $status = Status::newGood();
 752
 753                 $info = $this->getFileStat( $params );
 754                 if ( !$info ) { // let StreamFile handle the 404
 755                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 756                 }
 757
 758                 // Set output buffer and HTTP headers for stream
 759                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 760                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 761                 if ( $res == StreamFile::NOT_MODIFIED ) {
 762                         // do nothing; client cache is up to date
 763                 } elseif ( $res == StreamFile::READY_STREAM ) {
 764                         wfProfileIn( __METHOD__ . '-send' );
 765                         wfProfileIn( __METHOD__ . '-send-' . $this->name );
 766                         $status = $this->doStreamFile( $params );
 767                         wfProfileOut( __METHOD__ . '-send-' . $this->name );
 768                         wfProfileOut( __METHOD__ . '-send' );
 769                 } else {
 770                         $status->fatal( 'backend-fail-stream', $params['src'] );
 771                 }
 772
 773                 wfProfileOut( __METHOD__ . '-' . $this->name );
 774                 wfProfileOut( __METHOD__ );
 775                 return $status;
 776         }
 777
 778         /**
 779          * @see FileBackendStore::streamFile()
 780          * @return Status
 781          */
 782         protected function doStreamFile( array $params ) {
 783                 $status = Status::newGood();
 784
 785                 $fsFile = $this->getLocalReference( $params );
 786                 if ( !$fsFile ) {
 787                         $status->fatal( 'backend-fail-stream', $params['src'] );
 788                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 789                         $status->fatal( 'backend-fail-stream', $params['src'] );
 790                 }
 791
 792                 return $status;
 793         }
 794
 795         /**
 796          * @see FileBackend::directoryExists()
 797          * @return bool|null
 798          */
 799         final public function directoryExists( array $params ) {
 800                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 801                 if ( $dir === null ) {
 802                         return false; // invalid storage path
 803                 }
 804                 if ( $shard !== null ) { // confined to a single container/shard
 805                         return $this->doDirectoryExists( $fullCont, $dir, $params );
 806                 } else { // directory is on several shards
 807                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 808                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 809                         $res = false; // response
 810                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 811                                 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
 812                                 if ( $exists ) {
 813                                         $res = true;
 814                                         break; // found one!
 815                                 } elseif ( $exists === null ) { // error?
 816                                         $res = null; // if we don't find anything, it is indeterminate
 817                                 }
 818                         }
 819                         return $res;
 820                 }
 821         }
 822
 823         /**
 824          * @see FileBackendStore::directoryExists()
 825          *
 826          * @param $container string Resolved container name
 827          * @param $dir string Resolved path relative to container
 828          * @param $params Array
 829          * @return bool|null
 830          */
 831         abstract protected function doDirectoryExists( $container, $dir, array $params );
 832
 833         /**
 834          * @see FileBackend::getDirectoryList()
 835          * @return Traversable|Array|null Returns null on failure
 836          */
 837         final public function getDirectoryList( array $params ) {
 838                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 839                 if ( $dir === null ) { // invalid storage path
 840                         return null;
 841                 }
 842                 if ( $shard !== null ) {
 843                         // File listing is confined to a single container/shard
 844                         return $this->getDirectoryListInternal( $fullCont, $dir, $params );
 845                 } else {
 846                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 847                         // File listing spans multiple containers/shards
 848                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 849                         return new FileBackendStoreShardDirIterator( $this,
 850                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 851                 }
 852         }
 853
 854         /**
 855          * Do not call this function from places outside FileBackend
 856          *
 857          * @see FileBackendStore::getDirectoryList()
 858          *
 859          * @param $container string Resolved container name
 860          * @param $dir string Resolved path relative to container
 861          * @param $params Array
 862          * @return Traversable|Array|null Returns null on failure
 863          */
 864         abstract public function getDirectoryListInternal( $container, $dir, array $params );
 865
 866         /**
 867          * @see FileBackend::getFileList()
 868          * @return Traversable|Array|null Returns null on failure
 869          */
 870         final public function getFileList( array $params ) {
 871                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 872                 if ( $dir === null ) { // invalid storage path
 873                         return null;
 874                 }
 875                 if ( $shard !== null ) {
 876                         // File listing is confined to a single container/shard
 877                         return $this->getFileListInternal( $fullCont, $dir, $params );
 878                 } else {
 879                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 880                         // File listing spans multiple containers/shards
 881                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 882                         return new FileBackendStoreShardFileIterator( $this,
 883                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 884                 }
 885         }
 886
 887         /**
 888          * Do not call this function from places outside FileBackend
 889          *
 890          * @see FileBackendStore::getFileList()
 891          *
 892          * @param $container string Resolved container name
 893          * @param $dir string Resolved path relative to container
 894          * @param $params Array
 895          * @return Traversable|Array|null Returns null on failure
 896          */
 897         abstract public function getFileListInternal( $container, $dir, array $params );
 898
 899         /**
 900          * Return a list of FileOp objects from a list of operations.
 901          * Do not call this function from places outside FileBackend.
 902          *
 903          * The result must have the same number of items as the input.
 904          * An exception is thrown if an unsupported operation is requested.
 905          *
 906          * @param $ops Array Same format as doOperations()
 907          * @return Array List of FileOp objects
 908          * @throws MWException
 909          */
 910         final public function getOperationsInternal( array $ops ) {
 911                 $supportedOps = array(
 912                         'store'       => 'StoreFileOp',
 913                         'copy'        => 'CopyFileOp',
 914                         'move'        => 'MoveFileOp',
 915                         'delete'      => 'DeleteFileOp',
 916                         'create'      => 'CreateFileOp',
 917                         'null'        => 'NullFileOp'
 918                 );
 919
 920                 $performOps = array(); // array of FileOp objects
 921                 // Build up ordered array of FileOps...
 922                 foreach ( $ops as $operation ) {
 923                         $opName = $operation['op'];
 924                         if ( isset( $supportedOps[$opName] ) ) {
 925                                 $class = $supportedOps[$opName];
 926                                 // Get params for this operation
 927                                 $params = $operation;
 928                                 // Append the FileOp class
 929                                 $performOps[] = new $class( $this, $params );
 930                         } else {
 931                                 throw new MWException( "Operation '$opName' is not supported." );
 932                         }
 933                 }
 934
 935                 return $performOps;
 936         }
 937
 938         /**
 939          * Get a list of storage paths to lock for a list of operations
 940          * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
 941          * each corresponding to a list of storage paths to be locked.
 942          *
 943          * @param $performOps Array List of FileOp objects
 944          * @return Array ('sh' => list of paths, 'ex' => list of paths)
 945          */
 946         final public function getPathsToLockForOpsInternal( array $performOps ) {
 947                 // Build up a list of files to lock...
 948                 $paths = array( 'sh' => array(), 'ex' => array() );
 949                 foreach ( $performOps as $fileOp ) {
 950                         $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
 951                         $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
 952                 }
 953                 // Optimization: if doing an EX lock anyway, don't also set an SH one
 954                 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
 955                 // Get a shared lock on the parent directory of each path changed
 956                 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
 957
 958                 return $paths;
 959         }
 960
 961         /**
 962          * @see FileBackend::getScopedLocksForOps()
 963          * @return Array
 964          */
 965         public function getScopedLocksForOps( array $ops, Status $status ) {
 966                 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
 967                 return array(
 968                         $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
 969                         $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
 970                 );
 971         }
 972
 973         /**
 974          * @see FileBackend::doOperationsInternal()
 975          * @return Status
 976          */
 977         final protected function doOperationsInternal( array $ops, array $opts ) {
 978                 wfProfileIn( __METHOD__ );
 979                 wfProfileIn( __METHOD__ . '-' . $this->name );
 980                 $status = Status::newGood();
 981
 982                 // Build up a list of FileOps...
 983                 $performOps = $this->getOperationsInternal( $ops );
 984
 985                 // Acquire any locks as needed...
 986                 if ( empty( $opts['nonLocking'] ) ) {
 987                         // Build up a list of files to lock...
 988                         $paths = $this->getPathsToLockForOpsInternal( $performOps );
 989                         // Try to lock those files for the scope of this function...
 990                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 991                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 992                         if ( !$status->isOK() ) {
 993                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 994                                 wfProfileOut( __METHOD__ );
 995                                 return $status; // abort
 996                         }
 997                 }
 998
 999                 // Clear any file cache entries (after locks acquired)
1000                 if ( empty( $opts['preserveCache'] ) ) {
1001                         $this->clearCache();
1002                 }
1003
1004                 // Load from the persistent file and container caches
1005                 $this->primeFileCache( $performOps );
1006                 $this->primeContainerCache( $performOps );
1007
1008                 // Actually attempt the operation batch...
1009                 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1010
1011                 // Merge errors into status fields
1012                 $status->merge( $subStatus );
1013                 $status->success = $subStatus->success; // not done in merge()
1014
1015                 wfProfileOut( __METHOD__ . '-' . $this->name );
1016                 wfProfileOut( __METHOD__ );
1017                 return $status;
1018         }
1019
1020         /**
1021          * @see FileBackend::doQuickOperationsInternal()
1022          * @return Status
1023          * @throws MWException
1024          */
1025         final protected function doQuickOperationsInternal( array $ops ) {
1026                 wfProfileIn( __METHOD__ );
1027                 wfProfileIn( __METHOD__ . '-' . $this->name );
1028                 $status = Status::newGood();
1029
1030                 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
1031                 $async = ( $this->parallelize === 'implicit' );
1032                 $maxConcurrency = $this->concurrency; // throttle
1033
1034                 $statuses = array(); // array of (index => Status)
1035                 $fileOpHandles = array(); // list of (index => handle) arrays
1036                 $curFileOpHandles = array(); // current handle batch
1037                 // Perform the sync-only ops and build up op handles for the async ops...
1038                 foreach ( $ops as $index => $params ) {
1039                         if ( !in_array( $params['op'], $supportedOps ) ) {
1040                                 wfProfileOut( __METHOD__ . '-' . $this->name );
1041                                 wfProfileOut( __METHOD__ );
1042                                 throw new MWException( "Operation '{$params['op']}' is not supported." );
1043                         }
1044                         $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1045                         $subStatus = $this->$method( array( 'async' => $async ) + $params );
1046                         if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1047                                 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1048                                         $fileOpHandles[] = $curFileOpHandles; // push this batch
1049                                         $curFileOpHandles = array();
1050                                 }
1051                                 $curFileOpHandles[$index] = $subStatus->value; // keep index
1052                         } else { // error or completed
1053                                 $statuses[$index] = $subStatus; // keep index
1054                         }
1055                 }
1056                 if ( count( $curFileOpHandles ) ) {
1057                         $fileOpHandles[] = $curFileOpHandles; // last batch
1058                 }
1059                 // Do all the async ops that can be done concurrently...
1060                 foreach ( $fileOpHandles as $fileHandleBatch ) {
1061                         $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1062                 }
1063                 // Marshall and merge all the responses...
1064                 foreach ( $statuses as $index => $subStatus ) {
1065                         $status->merge( $subStatus );
1066                         if ( $subStatus->isOK() ) {
1067                                 $status->success[$index] = true;
1068                                 ++$status->successCount;
1069                         } else {
1070                                 $status->success[$index] = false;
1071                                 ++$status->failCount;
1072                         }
1073                 }
1074
1075                 wfProfileOut( __METHOD__ . '-' . $this->name );
1076                 wfProfileOut( __METHOD__ );
1077                 return $status;
1078         }
1079
1080         /**
1081          * Execute a list of FileBackendStoreOpHandle handles in parallel.
1082          * The resulting Status object fields will correspond
1083          * to the order in which the handles where given.
1084          *
1085          * @param $handles Array List of FileBackendStoreOpHandle objects
1086          * @return Array Map of Status objects
1087          * @throws MWException
1088          */
1089         final public function executeOpHandlesInternal( array $fileOpHandles ) {
1090                 wfProfileIn( __METHOD__ );
1091                 wfProfileIn( __METHOD__ . '-' . $this->name );
1092                 foreach ( $fileOpHandles as $fileOpHandle ) {
1093                         if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1094                                 throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
1095                         } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1096                                 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
1097                         }
1098                 }
1099                 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1100                 foreach ( $fileOpHandles as $fileOpHandle ) {
1101                         $fileOpHandle->closeResources();
1102                 }
1103                 wfProfileOut( __METHOD__ . '-' . $this->name );
1104                 wfProfileOut( __METHOD__ );
1105                 return $res;
1106         }
1107
1108         /**
1109          * @see FileBackendStore::executeOpHandlesInternal()
1110          * @return Array List of corresponding Status objects
1111          */
1112         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1113                 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
1114                         throw new MWException( "This backend supports no asynchronous operations." );
1115                 }
1116                 return array();
1117         }
1118
1119         /**
1120          * @see FileBackend::preloadCache()
1121          */
1122         final public function preloadCache( array $paths ) {
1123                 $fullConts = array(); // full container names
1124                 foreach ( $paths as $path ) {
1125                         list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
1126                         $fullConts[] = $fullCont;
1127                 }
1128                 // Load from the persistent file and container caches
1129                 $this->primeContainerCache( $fullConts );
1130                 $this->primeFileCache( $paths );
1131         }
1132
1133         /**
1134          * @see FileBackend::clearCache()
1135          */
1136         final public function clearCache( array $paths = null ) {
1137                 if ( is_array( $paths ) ) {
1138                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1139                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
1140                 }
1141                 if ( $paths === null ) {
1142                         $this->cheapCache->clear();
1143                         $this->expensiveCache->clear();
1144                 } else {
1145                         foreach ( $paths as $path ) {
1146                                 $this->cheapCache->clear( $path );
1147                                 $this->expensiveCache->clear( $path );
1148                         }
1149                 }
1150                 $this->doClearCache( $paths );
1151         }
1152
1153         /**
1154          * Clears any additional stat caches for storage paths
1155          *
1156          * @see FileBackend::clearCache()
1157          *
1158          * @param $paths Array Storage paths (optional)
1159          * @return void
1160          */
1161         protected function doClearCache( array $paths = null ) {}
1162
1163         /**
1164          * Is this a key/value store where directories are just virtual?
1165          * Virtual directories exists in so much as files exists that are
1166          * prefixed with the directory path followed by a forward slash.
1167          *
1168          * @return bool
1169          */
1170         abstract protected function directoriesAreVirtual();
1171
1172         /**
1173          * Check if a container name is valid.
1174          * This checks for for length and illegal characters.
1175          *
1176          * @param $container string
1177          * @return bool
1178          */
1179         final protected static function isValidContainerName( $container ) {
1180                 // This accounts for Swift and S3 restrictions while leaving room
1181                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1182                 // This disallows directory separators or traversal characters.
1183                 // Note that matching strings URL encode to the same string;
1184                 // in Swift, the length restriction is *after* URL encoding.
1185                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1186         }
1187
1188         /**
1189          * Splits a storage path into an internal container name,
1190          * an internal relative file name, and a container shard suffix.
1191          * Any shard suffix is already appended to the internal container name.
1192          * This also checks that the storage path is valid and within this backend.
1193          *
1194          * If the container is sharded but a suffix could not be determined,
1195          * this means that the path can only refer to a directory and can only
1196          * be scanned by looking in all the container shards.
1197          *
1198          * @param $storagePath string
1199          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1200          */
1201         final protected function resolveStoragePath( $storagePath ) {
1202                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1203                 if ( $backend === $this->name ) { // must be for this backend
1204                         $relPath = self::normalizeContainerPath( $relPath );
1205                         if ( $relPath !== null ) {
1206                                 // Get shard for the normalized path if this container is sharded
1207                                 $cShard = $this->getContainerShard( $container, $relPath );
1208                                 // Validate and sanitize the relative path (backend-specific)
1209                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1210                                 if ( $relPath !== null ) {
1211                                         // Prepend any wiki ID prefix to the container name
1212                                         $container = $this->fullContainerName( $container );
1213                                         if ( self::isValidContainerName( $container ) ) {
1214                                                 // Validate and sanitize the container name (backend-specific)
1215                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1216                                                 if ( $container !== null ) {
1217                                                         return array( $container, $relPath, $cShard );
1218                                                 }
1219                                         }
1220                                 }
1221                         }
1222                 }
1223                 return array( null, null, null );
1224         }
1225
1226         /**
1227          * Like resolveStoragePath() except null values are returned if
1228          * the container is sharded and the shard could not be determined.
1229          *
1230          * @see FileBackendStore::resolveStoragePath()
1231          *
1232          * @param $storagePath string
1233          * @return Array (container, path) or (null, null) if invalid
1234          */
1235         final protected function resolveStoragePathReal( $storagePath ) {
1236                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1237                 if ( $cShard !== null ) {
1238                         return array( $container, $relPath );
1239                 }
1240                 return array( null, null );
1241         }
1242
1243         /**
1244          * Get the container name shard suffix for a given path.
1245          * Any empty suffix means the container is not sharded.
1246          *
1247          * @param $container string Container name
1248          * @param $relPath string Storage path relative to the container
1249          * @return string|null Returns null if shard could not be determined
1250          */
1251         final protected function getContainerShard( $container, $relPath ) {
1252                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1253                 if ( $levels == 1 || $levels == 2 ) {
1254                         // Hash characters are either base 16 or 36
1255                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1256                         // Get a regex that represents the shard portion of paths.
1257                         // The concatenation of the captures gives us the shard.
1258                         if ( $levels === 1 ) { // 16 or 36 shards per container
1259                                 $hashDirRegex = '(' . $char . ')';
1260                         } else { // 256 or 1296 shards per container
1261                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1262                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1263                                 } else { // short hash dir format (e.g. "a/b/c")
1264                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1265                                 }
1266                         }
1267                         // Allow certain directories to be above the hash dirs so as
1268                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1269                         // They must be 2+ chars to avoid any hash directory ambiguity.
1270                         $m = array();
1271                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1272                                 return '.' . implode( '', array_slice( $m, 1 ) );
1273                         }
1274                         return null; // failed to match
1275                 }
1276                 return ''; // no sharding
1277         }
1278
1279         /**
1280          * Check if a storage path maps to a single shard.
1281          * Container dirs like "a", where the container shards on "x/xy",
1282          * can reside on several shards. Such paths are tricky to handle.
1283          *
1284          * @param $storagePath string Storage path
1285          * @return bool
1286          */
1287         final public function isSingleShardPathInternal( $storagePath ) {
1288                 list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
1289                 return ( $shard !== null );
1290         }
1291
1292         /**
1293          * Get the sharding config for a container.
1294          * If greater than 0, then all file storage paths within
1295          * the container are required to be hashed accordingly.
1296          *
1297          * @param $container string
1298          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1299          */
1300         final protected function getContainerHashLevels( $container ) {
1301                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1302                         $config = $this->shardViaHashLevels[$container];
1303                         $hashLevels = (int)$config['levels'];
1304                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1305                                 $hashBase = (int)$config['base'];
1306                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1307                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1308                                 }
1309                         }
1310                 }
1311                 return array( 0, 0, false ); // no sharding
1312         }
1313
1314         /**
1315          * Get a list of full container shard suffixes for a container
1316          *
1317          * @param $container string
1318          * @return Array
1319          */
1320         final protected function getContainerSuffixes( $container ) {
1321                 $shards = array();
1322                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1323                 if ( $digits > 0 ) {
1324                         $numShards = pow( $base, $digits );
1325                         for ( $index = 0; $index < $numShards; $index++ ) {
1326                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1327                         }
1328                 }
1329                 return $shards;
1330         }
1331
1332         /**
1333          * Get the full container name, including the wiki ID prefix
1334          *
1335          * @param $container string
1336          * @return string
1337          */
1338         final protected function fullContainerName( $container ) {
1339                 if ( $this->wikiId != '' ) {
1340                         return "{$this->wikiId}-$container";
1341                 } else {
1342                         return $container;
1343                 }
1344         }
1345
1346         /**
1347          * Resolve a container name, checking if it's allowed by the backend.
1348          * This is intended for internal use, such as encoding illegal chars.
1349          * Subclasses can override this to be more restrictive.
1350          *
1351          * @param $container string
1352          * @return string|null
1353          */
1354         protected function resolveContainerName( $container ) {
1355                 return $container;
1356         }
1357
1358         /**
1359          * Resolve a relative storage path, checking if it's allowed by the backend.
1360          * This is intended for internal use, such as encoding illegal chars or perhaps
1361          * getting absolute paths (e.g. FS based backends). Note that the relative path
1362          * may be the empty string (e.g. the path is simply to the container).
1363          *
1364          * @param $container string Container name
1365          * @param $relStoragePath string Storage path relative to the container
1366          * @return string|null Path or null if not valid
1367          */
1368         protected function resolveContainerPath( $container, $relStoragePath ) {
1369                 return $relStoragePath;
1370         }
1371
1372         /**
1373          * Get the cache key for a container
1374          *
1375          * @param $container string Resolved container name
1376          * @return string
1377          */
1378         private function containerCacheKey( $container ) {
1379                 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1380         }
1381
1382         /**
1383          * Set the cached info for a container
1384          *
1385          * @param $container string Resolved container name
1386          * @param $val mixed Information to cache
1387          */
1388         final protected function setContainerCache( $container, $val ) {
1389                 $this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 );
1390         }
1391
1392         /**
1393          * Delete the cached info for a container.
1394          * The cache key is salted for a while to prevent race conditions.
1395          *
1396          * @param $container string Resolved container name
1397          */
1398         final protected function deleteContainerCache( $container ) {
1399                 if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
1400                         trigger_error( "Unable to delete stat cache for container $container." );
1401                 }
1402         }
1403
1404         /**
1405          * Do a batch lookup from cache for container stats for all containers
1406          * used in a list of container names, storage paths, or FileOp objects.
1407          *
1408          * @param $items Array
1409          * @return void
1410          */
1411         final protected function primeContainerCache( array $items ) {
1412                 wfProfileIn( __METHOD__ );
1413                 wfProfileIn( __METHOD__ . '-' . $this->name );
1414
1415                 $paths = array(); // list of storage paths
1416                 $contNames = array(); // (cache key => resolved container name)
1417                 // Get all the paths/containers from the items...
1418                 foreach ( $items as $item ) {
1419                         if ( $item instanceof FileOp ) {
1420                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1421                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1422                         } elseif ( self::isStoragePath( $item ) ) {
1423                                 $paths[] = $item;
1424                         } elseif ( is_string( $item ) ) { // full container name
1425                                 $contNames[$this->containerCacheKey( $item )] = $item;
1426                         }
1427                 }
1428                 // Get all the corresponding cache keys for paths...
1429                 foreach ( $paths as $path ) {
1430                         list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
1431                         if ( $fullCont !== null ) { // valid path for this backend
1432                                 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1433                         }
1434                 }
1435
1436                 $contInfo = array(); // (resolved container name => cache value)
1437                 // Get all cache entries for these container cache keys...
1438                 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1439                 foreach ( $values as $cacheKey => $val ) {
1440                         $contInfo[$contNames[$cacheKey]] = $val;
1441                 }
1442
1443                 // Populate the container process cache for the backend...
1444                 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1445
1446                 wfProfileOut( __METHOD__ . '-' . $this->name );
1447                 wfProfileOut( __METHOD__ );
1448         }
1449
1450         /**
1451          * Fill the backend-specific process cache given an array of
1452          * resolved container names and their corresponding cached info.
1453          * Only containers that actually exist should appear in the map.
1454          *
1455          * @param $containerInfo Array Map of resolved container names to cached info
1456          * @return void
1457          */
1458         protected function doPrimeContainerCache( array $containerInfo ) {}
1459
1460         /**
1461          * Get the cache key for a file path
1462          *
1463          * @param $path string Storage path
1464          * @return string
1465          */
1466         private function fileCacheKey( $path ) {
1467                 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1468         }
1469
1470         /**
1471          * Set the cached stat info for a file path.
1472          * Negatives (404s) are not cached. By not caching negatives, we can skip cache
1473          * salting for the case when a file is created at a path were there was none before.
1474          *
1475          * @param $path string Storage path
1476          * @param $val mixed Information to cache
1477          */
1478         final protected function setFileCache( $path, $val ) {
1479                 $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
1480         }
1481
1482         /**
1483          * Delete the cached stat info for a file path.
1484          * The cache key is salted for a while to prevent race conditions.
1485          *
1486          * @param $path string Storage path
1487          */
1488         final protected function deleteFileCache( $path ) {
1489                 if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
1490                         trigger_error( "Unable to delete stat cache for file $path." );
1491                 }
1492         }
1493
1494         /**
1495          * Do a batch lookup from cache for file stats for all paths
1496          * used in a list of storage paths or FileOp objects.
1497          *
1498          * @param $items Array List of storage paths or FileOps
1499          * @return void
1500          */
1501         final protected function primeFileCache( array $items ) {
1502                 wfProfileIn( __METHOD__ );
1503                 wfProfileIn( __METHOD__ . '-' . $this->name );
1504
1505                 $paths = array(); // list of storage paths
1506                 $pathNames = array(); // (cache key => storage path)
1507                 // Get all the paths/containers from the items...
1508                 foreach ( $items as $item ) {
1509                         if ( $item instanceof FileOp ) {
1510                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1511                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1512                         } elseif ( self::isStoragePath( $item ) ) {
1513                                 $paths[] = $item;
1514                         }
1515                 }
1516                 // Get all the corresponding cache keys for paths...
1517                 foreach ( $paths as $path ) {
1518                         list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
1519                         if ( $rel !== null ) { // valid path for this backend
1520                                 $pathNames[$this->fileCacheKey( $path )] = $path;
1521                         }
1522                 }
1523                 // Get all cache entries for these container cache keys...
1524                 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1525                 foreach ( $values as $cacheKey => $val ) {
1526                         if ( is_array( $val ) ) {
1527                                 $path = $pathNames[$cacheKey];
1528                                 $this->cheapCache->set( $path, 'stat', $val );
1529                                 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1530                                         $this->cheapCache->set( $path, 'sha1',
1531                                                 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
1532                                 }
1533                         }
1534                 }
1535
1536                 wfProfileOut( __METHOD__ . '-' . $this->name );
1537                 wfProfileOut( __METHOD__ );
1538         }
1539 }
1540
1541 /**
1542  * FileBackendStore helper class for performing asynchronous file operations.
1543  *
1544  * For example, calling FileBackendStore::createInternal() with the "async"
1545  * param flag may result in a Status that contains this object as a value.
1546  * This class is largely backend-specific and is mostly just "magic" to be
1547  * passed to FileBackendStore::executeOpHandlesInternal().
1548  */
1549 abstract class FileBackendStoreOpHandle {
1550         /** @var Array */
1551         public $params = array(); // params to caller functions
1552         /** @var FileBackendStore */
1553         public $backend;
1554         /** @var Array */
1555         public $resourcesToClose = array();
1556
1557         public $call; // string; name that identifies the function called
1558
1559         /**
1560          * Close all open file handles
1561          *
1562          * @return void
1563          */
1564         public function closeResources() {
1565                 array_map( 'fclose', $this->resourcesToClose );
1566         }
1567 }
1568
1569 /**
1570  * FileBackendStore helper function to handle listings that span container shards.
1571  * Do not use this class from places outside of FileBackendStore.
1572  *
1573  * @ingroup FileBackend
1574  */
1575 abstract class FileBackendStoreShardListIterator implements Iterator {
1576         /** @var FileBackendStore */
1577         protected $backend;
1578         /** @var Array */
1579         protected $params;
1580         /** @var Array */
1581         protected $shardSuffixes;
1582         protected $container; // string; full container name
1583         protected $directory; // string; resolved relative path
1584
1585         /** @var Traversable */
1586         protected $iter;
1587         protected $curShard = 0; // integer
1588         protected $pos = 0; // integer
1589
1590         /** @var Array */
1591         protected $multiShardPaths = array(); // (rel path => 1)
1592
1593         /**
1594          * @param $backend FileBackendStore
1595          * @param $container string Full storage container name
1596          * @param $dir string Storage directory relative to container
1597          * @param $suffixes Array List of container shard suffixes
1598          * @param $params Array
1599          */
1600         public function __construct(
1601                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1602         ) {
1603                 $this->backend = $backend;
1604                 $this->container = $container;
1605                 $this->directory = $dir;
1606                 $this->shardSuffixes = $suffixes;
1607                 $this->params = $params;
1608         }
1609
1610         /**
1611          * @see Iterator::key()
1612          * @return integer
1613          */
1614         public function key() {
1615                 return $this->pos;
1616         }
1617
1618         /**
1619          * @see Iterator::valid()
1620          * @return bool
1621          */
1622         public function valid() {
1623                 if ( $this->iter instanceof Iterator ) {
1624                         return $this->iter->valid();
1625                 } elseif ( is_array( $this->iter ) ) {
1626                         return ( current( $this->iter ) !== false ); // no paths can have this value
1627                 }
1628                 return false; // some failure?
1629         }
1630
1631         /**
1632          * @see Iterator::current()
1633          * @return string|bool String or false
1634          */
1635         public function current() {
1636                 return ( $this->iter instanceof Iterator )
1637                         ? $this->iter->current()
1638                         : current( $this->iter );
1639         }
1640
1641         /**
1642          * @see Iterator::next()
1643          * @return void
1644          */
1645         public function next() {
1646                 ++$this->pos;
1647                 ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
1648                 do {
1649                         $continue = false; // keep scanning shards?
1650                         $this->filterViaNext(); // filter out duplicates
1651                         // Find the next non-empty shard if no elements are left
1652                         if ( !$this->valid() ) {
1653                                 $this->nextShardIteratorIfNotValid();
1654                                 $continue = $this->valid(); // re-filter unless we ran out of shards
1655                         }
1656                 } while ( $continue );
1657         }
1658
1659         /**
1660          * @see Iterator::rewind()
1661          * @return void
1662          */
1663         public function rewind() {
1664                 $this->pos = 0;
1665                 $this->curShard = 0;
1666                 $this->setIteratorFromCurrentShard();
1667                 do {
1668                         $continue = false; // keep scanning shards?
1669                         $this->filterViaNext(); // filter out duplicates
1670                         // Find the next non-empty shard if no elements are left
1671                         if ( !$this->valid() ) {
1672                                 $this->nextShardIteratorIfNotValid();
1673                                 $continue = $this->valid(); // re-filter unless we ran out of shards
1674                         }
1675                 } while ( $continue );
1676         }
1677
1678         /**
1679          * Filter out duplicate items by advancing to the next ones
1680          */
1681         protected function filterViaNext() {
1682                 while ( $this->valid() ) {
1683                         $rel = $this->iter->current(); // path relative to given directory
1684                         $path = $this->params['dir'] . "/{$rel}"; // full storage path
1685                         if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1686                                 break; // path is only on one shard; no issue with duplicates
1687                         } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1688                                 // Don't keep listing paths that are on multiple shards
1689                                 ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter );
1690                         } else {
1691                                 $this->multiShardPaths[$rel] = 1;
1692                                 break;
1693                         }
1694                 }
1695         }
1696
1697         /**
1698          * If the list iterator for this container shard is out of items,
1699          * then move on to the next container that has items.
1700          * If there are none, then it advances to the last container.
1701          */
1702         protected function nextShardIteratorIfNotValid() {
1703                 while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) {
1704                         $this->setIteratorFromCurrentShard();
1705                 }
1706         }
1707
1708         /**
1709          * Set the list iterator to that of the current container shard
1710          */
1711         protected function setIteratorFromCurrentShard() {
1712                 $this->iter = $this->listFromShard(
1713                         $this->container . $this->shardSuffixes[$this->curShard],
1714                         $this->directory, $this->params );
1715                 // Start loading results so that current() works
1716                 if ( $this->iter ) {
1717                         ( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter );
1718                 }
1719         }
1720
1721         /**
1722          * Get the list for a given container shard
1723          *
1724          * @param $container string Resolved container name
1725          * @param $dir string Resolved path relative to container
1726          * @param $params Array
1727          * @return Traversable|Array|null
1728          */
1729         abstract protected function listFromShard( $container, $dir, array $params );
1730 }
1731
1732 /**
1733  * Iterator for listing directories
1734  */
1735 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1736         /**
1737          * @see FileBackendStoreShardListIterator::listFromShard()
1738          * @return Array|null|Traversable
1739          */
1740         protected function listFromShard( $container, $dir, array $params ) {
1741                 return $this->backend->getDirectoryListInternal( $container, $dir, $params );
1742         }
1743 }
1744
1745 /**
1746  * Iterator for listing regular files
1747  */
1748 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1749         /**
1750          * @see FileBackendStoreShardListIterator::listFromShard()
1751          * @return Array|null|Traversable
1752          */
1753         protected function listFromShard( $container, $dir, array $params ) {
1754                 return $this->backend->getFileListInternal( $container, $dir, $params );
1755         }
1756 }