includes/filebackend/FileBackendStore.php

   1 <?php
   2 /**
   3  * Base class for all backends using particular storage medium.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Base class for all backends using particular storage medium.
  27  *
  28  * This class defines the methods as abstract that subclasses must implement.
  29  * Outside callers should *not* use functions with "Internal" in the name.
  30  *
  31  * The FileBackend operations are implemented using basic functions
  32  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  33  * This class is also responsible for path resolution and sanitization.
  34  *
  35  * @ingroup FileBackend
  36  * @since 1.19
  37  */
  38 abstract class FileBackendStore extends FileBackend {
  39         /** @var BagOStuff */
  40         protected $memCache;
  41         /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */
  42         protected $cheapCache;
  43         /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */
  44         protected $expensiveCache;
  45
  46         /** @var Array Map of container names to sharding config */
  47         protected $shardViaHashLevels = array();
  48
  49         /** @var callback Method to get the MIME type of files */
  50         protected $mimeCallback;
  51
  52         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  53
  54         const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
  55         const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
  56         const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
  57
  58         /**
  59          * @see FileBackend::__construct()
  60          * Additional $config params include:
  61          *   - mimeCallback : Callback that takes (storage path, content, file system path) and
  62          *                    returns the MIME type of the file or 'unknown/unknown'. The file
  63          *                    system path parameter should be used if the content one is null.
  64          *
  65          * @param array $config
  66          */
  67         public function __construct( array $config ) {
  68                 parent::__construct( $config );
  69                 $this->mimeCallback = isset( $config['mimeCallback'] )
  70                         ? $config['mimeCallback']
  71                         : function( $storagePath, $content, $fsPath ) {
  72                                 // @TODO: handle the case of extension-less files using the contents
  73                                 return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown';
  74                         };
  75                 $this->memCache = new EmptyBagOStuff(); // disabled by default
  76                 $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
  77                 $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );
  78         }
  79
  80         /**
  81          * Get the maximum allowable file size given backend
  82          * medium restrictions and basic performance constraints.
  83          * Do not call this function from places outside FileBackend and FileOp.
  84          *
  85          * @return integer Bytes
  86          */
  87         final public function maxFileSizeInternal() {
  88                 return $this->maxFileSize;
  89         }
  90
  91         /**
  92          * Check if a file can be created or changed at a given storage path.
  93          * FS backends should check if the parent directory exists, files can be
  94          * written under it, and that any file already there is writable.
  95          * Backends using key/value stores should check if the container exists.
  96          *
  97          * @param string $storagePath
  98          * @return bool
  99          */
 100         abstract public function isPathUsableInternal( $storagePath );
 101
 102         /**
 103          * Create a file in the backend with the given contents.
 104          * This will overwrite any file that exists at the destination.
 105          * Do not call this function from places outside FileBackend and FileOp.
 106          *
 107          * $params include:
 108          *   - content     : the raw file contents
 109          *   - dst         : destination storage path
 110          *   - headers     : HTTP header name/value map
 111          *   - async       : Status will be returned immediately if supported.
 112          *                   If the status is OK, then its value field will be
 113          *                   set to a FileBackendStoreOpHandle object.
 114          *   - dstExists   : Whether a file exists at the destination (optimization).
 115          *                   Callers can use "false" if no existing file is being changed.
 116          *
 117          * @param array $params
 118          * @return Status
 119          */
 120         final public function createInternal( array $params ) {
 121                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 122                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 123                         $status = Status::newFatal( 'backend-fail-maxsize',
 124                                 $params['dst'], $this->maxFileSizeInternal() );
 125                 } else {
 126                         $status = $this->doCreateInternal( $params );
 127                         $this->clearCache( array( $params['dst'] ) );
 128                         if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 129                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 130                         }
 131                 }
 132                 return $status;
 133         }
 134
 135         /**
 136          * @see FileBackendStore::createInternal()
 137          * @return Status
 138          */
 139         abstract protected function doCreateInternal( array $params );
 140
 141         /**
 142          * Store a file into the backend from a file on disk.
 143          * This will overwrite any file that exists at the destination.
 144          * Do not call this function from places outside FileBackend and FileOp.
 145          *
 146          * $params include:
 147          *   - src         : source path on disk
 148          *   - dst         : destination storage path
 149          *   - headers     : HTTP header name/value map
 150          *   - async       : Status will be returned immediately if supported.
 151          *                   If the status is OK, then its value field will be
 152          *                   set to a FileBackendStoreOpHandle object.
 153          *   - dstExists   : Whether a file exists at the destination (optimization).
 154          *                   Callers can use "false" if no existing file is being changed.
 155          *
 156          * @param array $params
 157          * @return Status
 158          */
 159         final public function storeInternal( array $params ) {
 160                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 161                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 162                         $status = Status::newFatal( 'backend-fail-maxsize',
 163                                 $params['dst'], $this->maxFileSizeInternal() );
 164                 } else {
 165                         $status = $this->doStoreInternal( $params );
 166                         $this->clearCache( array( $params['dst'] ) );
 167                         if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 168                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 169                         }
 170                 }
 171                 return $status;
 172         }
 173
 174         /**
 175          * @see FileBackendStore::storeInternal()
 176          * @return Status
 177          */
 178         abstract protected function doStoreInternal( array $params );
 179
 180         /**
 181          * Copy a file from one storage path to another in the backend.
 182          * This will overwrite any file that exists at the destination.
 183          * Do not call this function from places outside FileBackend and FileOp.
 184          *
 185          * $params include:
 186          *   - src                 : source storage path
 187          *   - dst                 : destination storage path
 188          *   - ignoreMissingSource : do nothing if the source file does not exist
 189          *   - headers             : HTTP header name/value map
 190          *   - async               : Status will be returned immediately if supported.
 191          *                           If the status is OK, then its value field will be
 192          *                           set to a FileBackendStoreOpHandle object.
 193          *   - dstExists           : Whether a file exists at the destination (optimization).
 194          *                           Callers can use "false" if no existing file is being changed.
 195          *
 196          * @param array $params
 197          * @return Status
 198          */
 199         final public function copyInternal( array $params ) {
 200                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 201                 $status = $this->doCopyInternal( $params );
 202                 $this->clearCache( array( $params['dst'] ) );
 203                 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 204                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 205                 }
 206                 return $status;
 207         }
 208
 209         /**
 210          * @see FileBackendStore::copyInternal()
 211          * @return Status
 212          */
 213         abstract protected function doCopyInternal( array $params );
 214
 215         /**
 216          * Delete a file at the storage path.
 217          * Do not call this function from places outside FileBackend and FileOp.
 218          *
 219          * $params include:
 220          *   - src                 : source storage path
 221          *   - ignoreMissingSource : do nothing if the source file does not exist
 222          *   - async               : Status will be returned immediately if supported.
 223          *                           If the status is OK, then its value field will be
 224          *                           set to a FileBackendStoreOpHandle object.
 225          *
 226          * @param array $params
 227          * @return Status
 228          */
 229         final public function deleteInternal( array $params ) {
 230                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 231                 $status = $this->doDeleteInternal( $params );
 232                 $this->clearCache( array( $params['src'] ) );
 233                 $this->deleteFileCache( $params['src'] ); // persistent cache
 234                 return $status;
 235         }
 236
 237         /**
 238          * @see FileBackendStore::deleteInternal()
 239          * @return Status
 240          */
 241         abstract protected function doDeleteInternal( array $params );
 242
 243         /**
 244          * Move a file from one storage path to another in the backend.
 245          * This will overwrite any file that exists at the destination.
 246          * Do not call this function from places outside FileBackend and FileOp.
 247          *
 248          * $params include:
 249          *   - src                 : source storage path
 250          *   - dst                 : destination storage path
 251          *   - ignoreMissingSource : do nothing if the source file does not exist
 252          *   - headers             : HTTP header name/value map
 253          *   - async               : Status will be returned immediately if supported.
 254          *                           If the status is OK, then its value field will be
 255          *                           set to a FileBackendStoreOpHandle object.
 256          *   - dstExists           : Whether a file exists at the destination (optimization).
 257          *                           Callers can use "false" if no existing file is being changed.
 258          *
 259          * @param array $params
 260          * @return Status
 261          */
 262         final public function moveInternal( array $params ) {
 263                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 264                 $status = $this->doMoveInternal( $params );
 265                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 266                 $this->deleteFileCache( $params['src'] ); // persistent cache
 267                 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 268                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 269                 }
 270                 return $status;
 271         }
 272
 273         /**
 274          * @see FileBackendStore::moveInternal()
 275          * @return Status
 276          */
 277         protected function doMoveInternal( array $params ) {
 278                 unset( $params['async'] ); // two steps, won't work here :)
 279                 $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
 280                 $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
 281                 // Copy source to dest
 282                 $status = $this->copyInternal( $params );
 283                 if ( $nsrc !== $ndst && $status->isOK() ) {
 284                         // Delete source (only fails due to races or network problems)
 285                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 286                         $status->setResult( true, $status->value ); // ignore delete() errors
 287                 }
 288                 return $status;
 289         }
 290
 291         /**
 292          * Alter metadata for a file at the storage path.
 293          * Do not call this function from places outside FileBackend and FileOp.
 294          *
 295          * $params include:
 296          *   - src           : source storage path
 297          *   - headers       : HTTP header name/value map
 298          *   - async         : Status will be returned immediately if supported.
 299          *                     If the status is OK, then its value field will be
 300          *                     set to a FileBackendStoreOpHandle object.
 301          *
 302          * @param array $params
 303          * @return Status
 304          */
 305         final public function describeInternal( array $params ) {
 306                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 307                 if ( count( $params['headers'] ) ) {
 308                         $status = $this->doDescribeInternal( $params );
 309                         $this->clearCache( array( $params['src'] ) );
 310                         $this->deleteFileCache( $params['src'] ); // persistent cache
 311                 } else {
 312                         $status = Status::newGood(); // nothing to do
 313                 }
 314                 return $status;
 315         }
 316
 317         /**
 318          * @see FileBackendStore::describeInternal()
 319          * @return Status
 320          */
 321         protected function doDescribeInternal( array $params ) {
 322                 return Status::newGood();
 323         }
 324
 325         /**
 326          * No-op file operation that does nothing.
 327          * Do not call this function from places outside FileBackend and FileOp.
 328          *
 329          * @param array $params
 330          * @return Status
 331          */
 332         final public function nullInternal( array $params ) {
 333                 return Status::newGood();
 334         }
 335
 336         final public function concatenate( array $params ) {
 337                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 338                 $status = Status::newGood();
 339
 340                 // Try to lock the source files for the scope of this function
 341                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 342                 if ( $status->isOK() ) {
 343                         // Actually do the file concatenation...
 344                         $start_time = microtime( true );
 345                         $status->merge( $this->doConcatenate( $params ) );
 346                         $sec = microtime( true ) - $start_time;
 347                         if ( !$status->isOK() ) {
 348                                 wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
 349                                         count( $params['srcs'] ) . " file(s) [$sec sec]" );
 350                         }
 351                 }
 352
 353                 return $status;
 354         }
 355
 356         /**
 357          * @see FileBackendStore::concatenate()
 358          * @return Status
 359          */
 360         protected function doConcatenate( array $params ) {
 361                 $status = Status::newGood();
 362                 $tmpPath = $params['dst']; // convenience
 363                 unset( $params['latest'] ); // sanity
 364
 365                 // Check that the specified temp file is valid...
 366                 wfSuppressWarnings();
 367                 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
 368                 wfRestoreWarnings();
 369                 if ( !$ok ) { // not present or not empty
 370                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 371                         return $status;
 372                 }
 373
 374                 // Get local FS versions of the chunks needed for the concatenation...
 375                 $fsFiles = $this->getLocalReferenceMulti( $params );
 376                 foreach ( $fsFiles as $path => &$fsFile ) {
 377                         if ( !$fsFile ) { // chunk failed to download?
 378                                 $fsFile = $this->getLocalReference( array( 'src' => $path ) );
 379                                 if ( !$fsFile ) { // retry failed?
 380                                         $status->fatal( 'backend-fail-read', $path );
 381                                         return $status;
 382                                 }
 383                         }
 384                 }
 385                 unset( $fsFile ); // unset reference so we can reuse $fsFile
 386
 387                 // Get a handle for the destination temp file
 388                 $tmpHandle = fopen( $tmpPath, 'ab' );
 389                 if ( $tmpHandle === false ) {
 390                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 391                         return $status;
 392                 }
 393
 394                 // Build up the temp file using the source chunks (in order)...
 395                 foreach ( $fsFiles as $virtualSource => $fsFile ) {
 396                         // Get a handle to the local FS version
 397                         $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
 398                         if ( $sourceHandle === false ) {
 399                                 fclose( $tmpHandle );
 400                                 $status->fatal( 'backend-fail-read', $virtualSource );
 401                                 return $status;
 402                         }
 403                         // Append chunk to file (pass chunk size to avoid magic quotes)
 404                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 405                                 fclose( $sourceHandle );
 406                                 fclose( $tmpHandle );
 407                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 408                                 return $status;
 409                         }
 410                         fclose( $sourceHandle );
 411                 }
 412                 if ( !fclose( $tmpHandle ) ) {
 413                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 414                         return $status;
 415                 }
 416
 417                 clearstatcache(); // temp file changed
 418
 419                 return $status;
 420         }
 421
 422         final protected function doPrepare( array $params ) {
 423                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 424                 $status = Status::newGood();
 425
 426                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 427                 if ( $dir === null ) {
 428                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 429                         return $status; // invalid storage path
 430                 }
 431
 432                 if ( $shard !== null ) { // confined to a single container/shard
 433                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 434                 } else { // directory is on several shards
 435                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 436                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 437                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 438                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 439                         }
 440                 }
 441
 442                 return $status;
 443         }
 444
 445         /**
 446          * @see FileBackendStore::doPrepare()
 447          * @return Status
 448          */
 449         protected function doPrepareInternal( $container, $dir, array $params ) {
 450                 return Status::newGood();
 451         }
 452
 453         final protected function doSecure( array $params ) {
 454                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 455                 $status = Status::newGood();
 456
 457                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 458                 if ( $dir === null ) {
 459                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 460                         return $status; // invalid storage path
 461                 }
 462
 463                 if ( $shard !== null ) { // confined to a single container/shard
 464                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 465                 } else { // directory is on several shards
 466                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 467                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 468                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 469                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 470                         }
 471                 }
 472
 473                 return $status;
 474         }
 475
 476         /**
 477          * @see FileBackendStore::doSecure()
 478          * @return Status
 479          */
 480         protected function doSecureInternal( $container, $dir, array $params ) {
 481                 return Status::newGood();
 482         }
 483
 484         final protected function doPublish( array $params ) {
 485                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 486                 $status = Status::newGood();
 487
 488                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 489                 if ( $dir === null ) {
 490                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 491                         return $status; // invalid storage path
 492                 }
 493
 494                 if ( $shard !== null ) { // confined to a single container/shard
 495                         $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 496                 } else { // directory is on several shards
 497                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 498                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 499                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 500                                 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 501                         }
 502                 }
 503
 504                 return $status;
 505         }
 506
 507         /**
 508          * @see FileBackendStore::doPublish()
 509          * @return Status
 510          */
 511         protected function doPublishInternal( $container, $dir, array $params ) {
 512                 return Status::newGood();
 513         }
 514
 515         final protected function doClean( array $params ) {
 516                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 517                 $status = Status::newGood();
 518
 519                 // Recursive: first delete all empty subdirs recursively
 520                 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
 521                         $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
 522                         if ( $subDirsRel !== null ) { // no errors
 523                                 foreach ( $subDirsRel as $subDirRel ) {
 524                                         $subDir = $params['dir'] . "/{$subDirRel}"; // full path
 525                                         $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
 526                                 }
 527                                 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
 528                         }
 529                 }
 530
 531                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 532                 if ( $dir === null ) {
 533                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 534                         return $status; // invalid storage path
 535                 }
 536
 537                 // Attempt to lock this directory...
 538                 $filesLockEx = array( $params['dir'] );
 539                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 540                 if ( !$status->isOK() ) {
 541                         return $status; // abort
 542                 }
 543
 544                 if ( $shard !== null ) { // confined to a single container/shard
 545                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 546                         $this->deleteContainerCache( $fullCont ); // purge cache
 547                 } else { // directory is on several shards
 548                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 549                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 550                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 551                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 552                                 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
 553                         }
 554                 }
 555
 556                 return $status;
 557         }
 558
 559         /**
 560          * @see FileBackendStore::doClean()
 561          * @return Status
 562          */
 563         protected function doCleanInternal( $container, $dir, array $params ) {
 564                 return Status::newGood();
 565         }
 566
 567         final public function fileExists( array $params ) {
 568                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 569                 $stat = $this->getFileStat( $params );
 570                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 571         }
 572
 573         final public function getFileTimestamp( array $params ) {
 574                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 575                 $stat = $this->getFileStat( $params );
 576                 return $stat ? $stat['mtime'] : false;
 577         }
 578
 579         final public function getFileSize( array $params ) {
 580                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 581                 $stat = $this->getFileStat( $params );
 582                 return $stat ? $stat['size'] : false;
 583         }
 584
 585         final public function getFileStat( array $params ) {
 586                 $path = self::normalizeStoragePath( $params['src'] );
 587                 if ( $path === null ) {
 588                         return false; // invalid storage path
 589                 }
 590                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 591                 $latest = !empty( $params['latest'] ); // use latest data?
 592                 if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
 593                         $this->primeFileCache( array( $path ) ); // check persistent cache
 594                 }
 595                 if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
 596                         $stat = $this->cheapCache->get( $path, 'stat' );
 597                         // If we want the latest data, check that this cached
 598                         // value was in fact fetched with the latest available data.
 599                         if ( is_array( $stat ) ) {
 600                                 if ( !$latest || $stat['latest'] ) {
 601                                         return $stat;
 602                                 }
 603                         } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) {
 604                                 if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
 605                                         return false;
 606                                 }
 607                         }
 608                 }
 609                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 610                 $stat = $this->doGetFileStat( $params );
 611                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 612                 if ( is_array( $stat ) ) { // file exists
 613                         $stat['latest'] = $latest;
 614                         $this->cheapCache->set( $path, 'stat', $stat );
 615                         $this->setFileCache( $path, $stat ); // update persistent cache
 616                         if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
 617                                 $this->cheapCache->set( $path, 'sha1',
 618                                         array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
 619                         }
 620                 } elseif ( $stat === false ) { // file does not exist
 621                         $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
 622                         $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too
 623                                 array( 'hash' => false, 'latest' => $latest ) );
 624                         wfDebug( __METHOD__ . ": File $path does not exist.\n" );
 625                 } else { // an error occurred
 626                         wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
 627                 }
 628                 return $stat;
 629         }
 630
 631         /**
 632          * @see FileBackendStore::getFileStat()
 633          */
 634         abstract protected function doGetFileStat( array $params );
 635
 636         public function getFileContentsMulti( array $params ) {
 637                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 638
 639                 $params = $this->setConcurrencyFlags( $params );
 640                 $contents = $this->doGetFileContentsMulti( $params );
 641
 642                 return $contents;
 643         }
 644
 645         /**
 646          * @see FileBackendStore::getFileContentsMulti()
 647          * @return Array
 648          */
 649         protected function doGetFileContentsMulti( array $params ) {
 650                 $contents = array();
 651                 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
 652                         wfSuppressWarnings();
 653                         $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
 654                         wfRestoreWarnings();
 655                 }
 656                 return $contents;
 657         }
 658
 659         final public function getFileSha1Base36( array $params ) {
 660                 $path = self::normalizeStoragePath( $params['src'] );
 661                 if ( $path === null ) {
 662                         return false; // invalid storage path
 663                 }
 664                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 665                 $latest = !empty( $params['latest'] ); // use latest data?
 666                 if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
 667                         $stat = $this->cheapCache->get( $path, 'sha1' );
 668                         // If we want the latest data, check that this cached
 669                         // value was in fact fetched with the latest available data.
 670                         if ( !$latest || $stat['latest'] ) {
 671                                 return $stat['hash'];
 672                         }
 673                 }
 674                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 675                 $hash = $this->doGetFileSha1Base36( $params );
 676                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 677                 $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
 678                 return $hash;
 679         }
 680
 681         /**
 682          * @see FileBackendStore::getFileSha1Base36()
 683          * @return bool|string
 684          */
 685         protected function doGetFileSha1Base36( array $params ) {
 686                 $fsFile = $this->getLocalReference( $params );
 687                 if ( !$fsFile ) {
 688                         return false;
 689                 } else {
 690                         return $fsFile->getSha1Base36();
 691                 }
 692         }
 693
 694         final public function getFileProps( array $params ) {
 695                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 696                 $fsFile = $this->getLocalReference( $params );
 697                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 698                 return $props;
 699         }
 700
 701         final public function getLocalReferenceMulti( array $params ) {
 702                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 703
 704                 $params = $this->setConcurrencyFlags( $params );
 705
 706                 $fsFiles = array(); // (path => FSFile)
 707                 $latest = !empty( $params['latest'] ); // use latest data?
 708                 // Reuse any files already in process cache...
 709                 foreach ( $params['srcs'] as $src ) {
 710                         $path = self::normalizeStoragePath( $src );
 711                         if ( $path === null ) {
 712                                 $fsFiles[$src] = null; // invalid storage path
 713                         } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
 714                                 $val = $this->expensiveCache->get( $path, 'localRef' );
 715                                 // If we want the latest data, check that this cached
 716                                 // value was in fact fetched with the latest available data.
 717                                 if ( !$latest || $val['latest'] ) {
 718                                         $fsFiles[$src] = $val['object'];
 719                                 }
 720                         }
 721                 }
 722                 // Fetch local references of any remaning files...
 723                 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
 724                 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
 725                         $fsFiles[$path] = $fsFile;
 726                         if ( $fsFile ) { // update the process cache...
 727                                 $this->expensiveCache->set( $path, 'localRef',
 728                                         array( 'object' => $fsFile, 'latest' => $latest ) );
 729                         }
 730                 }
 731
 732                 return $fsFiles;
 733         }
 734
 735         /**
 736          * @see FileBackendStore::getLocalReferenceMulti()
 737          * @return Array
 738          */
 739         protected function doGetLocalReferenceMulti( array $params ) {
 740                 return $this->doGetLocalCopyMulti( $params );
 741         }
 742
 743         final public function getLocalCopyMulti( array $params ) {
 744                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 745
 746                 $params = $this->setConcurrencyFlags( $params );
 747                 $tmpFiles = $this->doGetLocalCopyMulti( $params );
 748
 749                 return $tmpFiles;
 750         }
 751
 752         /**
 753          * @see FileBackendStore::getLocalCopyMulti()
 754          * @return Array
 755          */
 756         abstract protected function doGetLocalCopyMulti( array $params );
 757
 758         /**
 759          * @see FileBackend::getFileHttpUrl()
 760          * @return string|null
 761          */
 762         public function getFileHttpUrl( array $params ) {
 763                 return null; // not supported
 764         }
 765
 766         final public function streamFile( array $params ) {
 767                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 768                 $status = Status::newGood();
 769
 770                 $info = $this->getFileStat( $params );
 771                 if ( !$info ) { // let StreamFile handle the 404
 772                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 773                 }
 774
 775                 // Set output buffer and HTTP headers for stream
 776                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 777                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 778                 if ( $res == StreamFile::NOT_MODIFIED ) {
 779                         // do nothing; client cache is up to date
 780                 } elseif ( $res == StreamFile::READY_STREAM ) {
 781                         wfProfileIn( __METHOD__ . '-send-' . $this->name );
 782                         $status = $this->doStreamFile( $params );
 783                         wfProfileOut( __METHOD__ . '-send-' . $this->name );
 784                         if ( !$status->isOK() ) {
 785                                 // Per bug 41113, nasty things can happen if bad cache entries get
 786                                 // stuck in cache. It's also possible that this error can come up
 787                                 // with simple race conditions. Clear out the stat cache to be safe.
 788                                 $this->clearCache( array( $params['src'] ) );
 789                                 $this->deleteFileCache( $params['src'] );
 790                                 trigger_error( "Bad stat cache or race condition for file {$params['src']}." );
 791                         }
 792                 } else {
 793                         $status->fatal( 'backend-fail-stream', $params['src'] );
 794                 }
 795
 796                 return $status;
 797         }
 798
 799         /**
 800          * @see FileBackendStore::streamFile()
 801          * @return Status
 802          */
 803         protected function doStreamFile( array $params ) {
 804                 $status = Status::newGood();
 805
 806                 $fsFile = $this->getLocalReference( $params );
 807                 if ( !$fsFile ) {
 808                         $status->fatal( 'backend-fail-stream', $params['src'] );
 809                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 810                         $status->fatal( 'backend-fail-stream', $params['src'] );
 811                 }
 812
 813                 return $status;
 814         }
 815
 816         final public function directoryExists( array $params ) {
 817                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 818                 if ( $dir === null ) {
 819                         return false; // invalid storage path
 820                 }
 821                 if ( $shard !== null ) { // confined to a single container/shard
 822                         return $this->doDirectoryExists( $fullCont, $dir, $params );
 823                 } else { // directory is on several shards
 824                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 825                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 826                         $res = false; // response
 827                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 828                                 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
 829                                 if ( $exists ) {
 830                                         $res = true;
 831                                         break; // found one!
 832                                 } elseif ( $exists === null ) { // error?
 833                                         $res = null; // if we don't find anything, it is indeterminate
 834                                 }
 835                         }
 836                         return $res;
 837                 }
 838         }
 839
 840         /**
 841          * @see FileBackendStore::directoryExists()
 842          *
 843          * @param string $container Resolved container name
 844          * @param string $dir Resolved path relative to container
 845          * @param array $params
 846          * @return bool|null
 847          */
 848         abstract protected function doDirectoryExists( $container, $dir, array $params );
 849
 850         final public function getDirectoryList( array $params ) {
 851                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 852                 if ( $dir === null ) { // invalid storage path
 853                         return null;
 854                 }
 855                 if ( $shard !== null ) {
 856                         // File listing is confined to a single container/shard
 857                         return $this->getDirectoryListInternal( $fullCont, $dir, $params );
 858                 } else {
 859                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 860                         // File listing spans multiple containers/shards
 861                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 862                         return new FileBackendStoreShardDirIterator( $this,
 863                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 864                 }
 865         }
 866
 867         /**
 868          * Do not call this function from places outside FileBackend
 869          *
 870          * @see FileBackendStore::getDirectoryList()
 871          *
 872          * @param string $container Resolved container name
 873          * @param string $dir Resolved path relative to container
 874          * @param array $params
 875          * @return Traversable|Array|null Returns null on failure
 876          */
 877         abstract public function getDirectoryListInternal( $container, $dir, array $params );
 878
 879         final public function getFileList( array $params ) {
 880                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 881                 if ( $dir === null ) { // invalid storage path
 882                         return null;
 883                 }
 884                 if ( $shard !== null ) {
 885                         // File listing is confined to a single container/shard
 886                         return $this->getFileListInternal( $fullCont, $dir, $params );
 887                 } else {
 888                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 889                         // File listing spans multiple containers/shards
 890                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 891                         return new FileBackendStoreShardFileIterator( $this,
 892                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 893                 }
 894         }
 895
 896         /**
 897          * Do not call this function from places outside FileBackend
 898          *
 899          * @see FileBackendStore::getFileList()
 900          *
 901          * @param string $container Resolved container name
 902          * @param string $dir Resolved path relative to container
 903          * @param array $params
 904          * @return Traversable|Array|null Returns null on failure
 905          */
 906         abstract public function getFileListInternal( $container, $dir, array $params );
 907
 908         /**
 909          * Return a list of FileOp objects from a list of operations.
 910          * Do not call this function from places outside FileBackend.
 911          *
 912          * The result must have the same number of items as the input.
 913          * An exception is thrown if an unsupported operation is requested.
 914          *
 915          * @param array $ops Same format as doOperations()
 916          * @return Array List of FileOp objects
 917          * @throws MWException
 918          */
 919         final public function getOperationsInternal( array $ops ) {
 920                 $supportedOps = array(
 921                         'store' => 'StoreFileOp',
 922                         'copy' => 'CopyFileOp',
 923                         'move' => 'MoveFileOp',
 924                         'delete' => 'DeleteFileOp',
 925                         'create' => 'CreateFileOp',
 926                         'describe' => 'DescribeFileOp',
 927                         'null' => 'NullFileOp'
 928                 );
 929
 930                 $performOps = array(); // array of FileOp objects
 931                 // Build up ordered array of FileOps...
 932                 foreach ( $ops as $operation ) {
 933                         $opName = $operation['op'];
 934                         if ( isset( $supportedOps[$opName] ) ) {
 935                                 $class = $supportedOps[$opName];
 936                                 // Get params for this operation
 937                                 $params = $operation;
 938                                 // Append the FileOp class
 939                                 $performOps[] = new $class( $this, $params );
 940                         } else {
 941                                 throw new MWException( "Operation '$opName' is not supported." );
 942                         }
 943                 }
 944
 945                 return $performOps;
 946         }
 947
 948         /**
 949          * Get a list of storage paths to lock for a list of operations
 950          * Returns an array with LockManager::LOCK_UW (shared locks) and
 951          * LockManager::LOCK_EX (exclusive locks) keys, each corresponding
 952          * to a list of storage paths to be locked. All returned paths are
 953          * normalized.
 954          *
 955          * @param array $performOps List of FileOp objects
 956          * @return Array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list)
 957          */
 958         final public function getPathsToLockForOpsInternal( array $performOps ) {
 959                 // Build up a list of files to lock...
 960                 $paths = array( 'sh' => array(), 'ex' => array() );
 961                 foreach ( $performOps as $fileOp ) {
 962                         $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
 963                         $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
 964                 }
 965                 // Optimization: if doing an EX lock anyway, don't also set an SH one
 966                 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
 967                 // Get a shared lock on the parent directory of each path changed
 968                 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
 969
 970                 return array(
 971                         LockManager::LOCK_UW => $paths['sh'],
 972                         LockManager::LOCK_EX => $paths['ex']
 973                 );
 974         }
 975
 976         public function getScopedLocksForOps( array $ops, Status $status ) {
 977                 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
 978                 return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) );
 979         }
 980
 981         final protected function doOperationsInternal( array $ops, array $opts ) {
 982                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 983                 $status = Status::newGood();
 984
 985                 // Fix up custom header name/value pairs...
 986                 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
 987
 988                 // Build up a list of FileOps...
 989                 $performOps = $this->getOperationsInternal( $ops );
 990
 991                 // Acquire any locks as needed...
 992                 if ( empty( $opts['nonLocking'] ) ) {
 993                         // Build up a list of files to lock...
 994                         $paths = $this->getPathsToLockForOpsInternal( $performOps );
 995                         // Try to lock those files for the scope of this function...
 996                         $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status );
 997                         if ( !$status->isOK() ) {
 998                                 return $status; // abort
 999                         }
1000                 }
1001
1002                 // Clear any file cache entries (after locks acquired)
1003                 if ( empty( $opts['preserveCache'] ) ) {
1004                         $this->clearCache();
1005                 }
1006
1007                 // Load from the persistent file and container caches
1008                 $this->primeFileCache( $performOps );
1009                 $this->primeContainerCache( $performOps );
1010
1011                 // Actually attempt the operation batch...
1012                 $opts = $this->setConcurrencyFlags( $opts );
1013                 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1014
1015                 // Merge errors into status fields
1016                 $status->merge( $subStatus );
1017                 $status->success = $subStatus->success; // not done in merge()
1018
1019                 return $status;
1020         }
1021
1022         final protected function doQuickOperationsInternal( array $ops ) {
1023                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1024                 $status = Status::newGood();
1025
1026                 // Fix up custom header name/value pairs...
1027                 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
1028
1029                 // Clear any file cache entries
1030                 $this->clearCache();
1031
1032                 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' );
1033                 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1034                 $maxConcurrency = $this->concurrency; // throttle
1035
1036                 $statuses = array(); // array of (index => Status)
1037                 $fileOpHandles = array(); // list of (index => handle) arrays
1038                 $curFileOpHandles = array(); // current handle batch
1039                 // Perform the sync-only ops and build up op handles for the async ops...
1040                 foreach ( $ops as $index => $params ) {
1041                         if ( !in_array( $params['op'], $supportedOps ) ) {
1042                                 throw new MWException( "Operation '{$params['op']}' is not supported." );
1043                         }
1044                         $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1045                         $subStatus = $this->$method( array( 'async' => $async ) + $params );
1046                         if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1047                                 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1048                                         $fileOpHandles[] = $curFileOpHandles; // push this batch
1049                                         $curFileOpHandles = array();
1050                                 }
1051                                 $curFileOpHandles[$index] = $subStatus->value; // keep index
1052                         } else { // error or completed
1053                                 $statuses[$index] = $subStatus; // keep index
1054                         }
1055                 }
1056                 if ( count( $curFileOpHandles ) ) {
1057                         $fileOpHandles[] = $curFileOpHandles; // last batch
1058                 }
1059                 // Do all the async ops that can be done concurrently...
1060                 foreach ( $fileOpHandles as $fileHandleBatch ) {
1061                         $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1062                 }
1063                 // Marshall and merge all the responses...
1064                 foreach ( $statuses as $index => $subStatus ) {
1065                         $status->merge( $subStatus );
1066                         if ( $subStatus->isOK() ) {
1067                                 $status->success[$index] = true;
1068                                 ++$status->successCount;
1069                         } else {
1070                                 $status->success[$index] = false;
1071                                 ++$status->failCount;
1072                         }
1073                 }
1074
1075                 return $status;
1076         }
1077
1078         /**
1079          * Execute a list of FileBackendStoreOpHandle handles in parallel.
1080          * The resulting Status object fields will correspond
1081          * to the order in which the handles where given.
1082          *
1083          * @param array $handles List of FileBackendStoreOpHandle objects
1084          * @return Array Map of Status objects
1085          * @throws MWException
1086          */
1087         final public function executeOpHandlesInternal( array $fileOpHandles ) {
1088                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1089                 foreach ( $fileOpHandles as $fileOpHandle ) {
1090                         if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1091                                 throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
1092                         } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1093                                 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
1094                         }
1095                 }
1096                 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1097                 foreach ( $fileOpHandles as $fileOpHandle ) {
1098                         $fileOpHandle->closeResources();
1099                 }
1100                 return $res;
1101         }
1102
1103         /**
1104          * @see FileBackendStore::executeOpHandlesInternal()
1105          * @param array $fileOpHandles
1106          * @throws MWException
1107          * @return Array List of corresponding Status objects
1108          */
1109         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1110                 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
1111                         throw new MWException( "This backend supports no asynchronous operations." );
1112                 }
1113                 return array();
1114         }
1115
1116         /**
1117          * Strip long HTTP headers from a file operation.
1118          * Most headers are just numbers, but some are allowed to be long.
1119          * This function is useful for cleaning up headers and avoiding backend
1120          * specific errors, especially in the middle of batch file operations.
1121          *
1122          * @param array $op Same format as doOperation()
1123          * @return Array
1124          */
1125         protected function stripInvalidHeadersFromOp( array $op ) {
1126                 static $longs = array( 'Content-Disposition' );
1127                 if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1128                         foreach ( $op['headers'] as $name => $value ) {
1129                                 $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1130                                 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1131                                         trigger_error( "Header '$name: $value' is too long." );
1132                                         unset( $op['headers'][$name] );
1133                                 } elseif ( !strlen( $value ) ) {
1134                                         $op['headers'][$name] = ''; // null/false => ""
1135                                 }
1136                         }
1137                 }
1138                 return $op;
1139         }
1140
1141         final public function preloadCache( array $paths ) {
1142                 $fullConts = array(); // full container names
1143                 foreach ( $paths as $path ) {
1144                         list( $fullCont, , ) = $this->resolveStoragePath( $path );
1145                         $fullConts[] = $fullCont;
1146                 }
1147                 // Load from the persistent file and container caches
1148                 $this->primeContainerCache( $fullConts );
1149                 $this->primeFileCache( $paths );
1150         }
1151
1152         final public function clearCache( array $paths = null ) {
1153                 if ( is_array( $paths ) ) {
1154                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1155                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
1156                 }
1157                 if ( $paths === null ) {
1158                         $this->cheapCache->clear();
1159                         $this->expensiveCache->clear();
1160                 } else {
1161                         foreach ( $paths as $path ) {
1162                                 $this->cheapCache->clear( $path );
1163                                 $this->expensiveCache->clear( $path );
1164                         }
1165                 }
1166                 $this->doClearCache( $paths );
1167         }
1168
1169         /**
1170          * Clears any additional stat caches for storage paths
1171          *
1172          * @see FileBackend::clearCache()
1173          *
1174          * @param array $paths Storage paths (optional)
1175          * @return void
1176          */
1177         protected function doClearCache( array $paths = null ) {}
1178
1179         /**
1180          * Is this a key/value store where directories are just virtual?
1181          * Virtual directories exists in so much as files exists that are
1182          * prefixed with the directory path followed by a forward slash.
1183          *
1184          * @return bool
1185          */
1186         abstract protected function directoriesAreVirtual();
1187
1188         /**
1189          * Check if a container name is valid.
1190          * This checks for for length and illegal characters.
1191          *
1192          * @param string $container
1193          * @return bool
1194          */
1195         final protected static function isValidContainerName( $container ) {
1196                 // This accounts for Swift and S3 restrictions while leaving room
1197                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1198                 // This disallows directory separators or traversal characters.
1199                 // Note that matching strings URL encode to the same string;
1200                 // in Swift, the length restriction is *after* URL encoding.
1201                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1202         }
1203
1204         /**
1205          * Splits a storage path into an internal container name,
1206          * an internal relative file name, and a container shard suffix.
1207          * Any shard suffix is already appended to the internal container name.
1208          * This also checks that the storage path is valid and within this backend.
1209          *
1210          * If the container is sharded but a suffix could not be determined,
1211          * this means that the path can only refer to a directory and can only
1212          * be scanned by looking in all the container shards.
1213          *
1214          * @param string $storagePath
1215          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1216          */
1217         final protected function resolveStoragePath( $storagePath ) {
1218                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1219                 if ( $backend === $this->name ) { // must be for this backend
1220                         $relPath = self::normalizeContainerPath( $relPath );
1221                         if ( $relPath !== null ) {
1222                                 // Get shard for the normalized path if this container is sharded
1223                                 $cShard = $this->getContainerShard( $container, $relPath );
1224                                 // Validate and sanitize the relative path (backend-specific)
1225                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1226                                 if ( $relPath !== null ) {
1227                                         // Prepend any wiki ID prefix to the container name
1228                                         $container = $this->fullContainerName( $container );
1229                                         if ( self::isValidContainerName( $container ) ) {
1230                                                 // Validate and sanitize the container name (backend-specific)
1231                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1232                                                 if ( $container !== null ) {
1233                                                         return array( $container, $relPath, $cShard );
1234                                                 }
1235                                         }
1236                                 }
1237                         }
1238                 }
1239                 return array( null, null, null );
1240         }
1241
1242         /**
1243          * Like resolveStoragePath() except null values are returned if
1244          * the container is sharded and the shard could not be determined
1245          * or if the path ends with '/'. The later case is illegal for FS
1246          * backends and can confuse listings for object store backends.
1247          *
1248          * This function is used when resolving paths that must be valid
1249          * locations for files. Directory and listing functions should
1250          * generally just use resolveStoragePath() instead.
1251          *
1252          * @see FileBackendStore::resolveStoragePath()
1253          *
1254          * @param string $storagePath
1255          * @return Array (container, path) or (null, null) if invalid
1256          */
1257         final protected function resolveStoragePathReal( $storagePath ) {
1258                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1259                 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1260                         return array( $container, $relPath );
1261                 }
1262                 return array( null, null );
1263         }
1264
1265         /**
1266          * Get the container name shard suffix for a given path.
1267          * Any empty suffix means the container is not sharded.
1268          *
1269          * @param string $container Container name
1270          * @param string $relPath Storage path relative to the container
1271          * @return string|null Returns null if shard could not be determined
1272          */
1273         final protected function getContainerShard( $container, $relPath ) {
1274                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1275                 if ( $levels == 1 || $levels == 2 ) {
1276                         // Hash characters are either base 16 or 36
1277                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1278                         // Get a regex that represents the shard portion of paths.
1279                         // The concatenation of the captures gives us the shard.
1280                         if ( $levels === 1 ) { // 16 or 36 shards per container
1281                                 $hashDirRegex = '(' . $char . ')';
1282                         } else { // 256 or 1296 shards per container
1283                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1284                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1285                                 } else { // short hash dir format (e.g. "a/b/c")
1286                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1287                                 }
1288                         }
1289                         // Allow certain directories to be above the hash dirs so as
1290                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1291                         // They must be 2+ chars to avoid any hash directory ambiguity.
1292                         $m = array();
1293                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1294                                 return '.' . implode( '', array_slice( $m, 1 ) );
1295                         }
1296                         return null; // failed to match
1297                 }
1298                 return ''; // no sharding
1299         }
1300
1301         /**
1302          * Check if a storage path maps to a single shard.
1303          * Container dirs like "a", where the container shards on "x/xy",
1304          * can reside on several shards. Such paths are tricky to handle.
1305          *
1306          * @param string $storagePath Storage path
1307          * @return bool
1308          */
1309         final public function isSingleShardPathInternal( $storagePath ) {
1310                 list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1311                 return ( $shard !== null );
1312         }
1313
1314         /**
1315          * Get the sharding config for a container.
1316          * If greater than 0, then all file storage paths within
1317          * the container are required to be hashed accordingly.
1318          *
1319          * @param string $container
1320          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1321          */
1322         final protected function getContainerHashLevels( $container ) {
1323                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1324                         $config = $this->shardViaHashLevels[$container];
1325                         $hashLevels = (int)$config['levels'];
1326                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1327                                 $hashBase = (int)$config['base'];
1328                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1329                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1330                                 }
1331                         }
1332                 }
1333                 return array( 0, 0, false ); // no sharding
1334         }
1335
1336         /**
1337          * Get a list of full container shard suffixes for a container
1338          *
1339          * @param string $container
1340          * @return Array
1341          */
1342         final protected function getContainerSuffixes( $container ) {
1343                 $shards = array();
1344                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1345                 if ( $digits > 0 ) {
1346                         $numShards = pow( $base, $digits );
1347                         for ( $index = 0; $index < $numShards; $index++ ) {
1348                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1349                         }
1350                 }
1351                 return $shards;
1352         }
1353
1354         /**
1355          * Get the full container name, including the wiki ID prefix
1356          *
1357          * @param string $container
1358          * @return string
1359          */
1360         final protected function fullContainerName( $container ) {
1361                 if ( $this->wikiId != '' ) {
1362                         return "{$this->wikiId}-$container";
1363                 } else {
1364                         return $container;
1365                 }
1366         }
1367
1368         /**
1369          * Resolve a container name, checking if it's allowed by the backend.
1370          * This is intended for internal use, such as encoding illegal chars.
1371          * Subclasses can override this to be more restrictive.
1372          *
1373          * @param string $container
1374          * @return string|null
1375          */
1376         protected function resolveContainerName( $container ) {
1377                 return $container;
1378         }
1379
1380         /**
1381          * Resolve a relative storage path, checking if it's allowed by the backend.
1382          * This is intended for internal use, such as encoding illegal chars or perhaps
1383          * getting absolute paths (e.g. FS based backends). Note that the relative path
1384          * may be the empty string (e.g. the path is simply to the container).
1385          *
1386          * @param string $container Container name
1387          * @param string $relStoragePath Storage path relative to the container
1388          * @return string|null Path or null if not valid
1389          */
1390         protected function resolveContainerPath( $container, $relStoragePath ) {
1391                 return $relStoragePath;
1392         }
1393
1394         /**
1395          * Get the cache key for a container
1396          *
1397          * @param string $container Resolved container name
1398          * @return string
1399          */
1400         private function containerCacheKey( $container ) {
1401                 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1402         }
1403
1404         /**
1405          * Set the cached info for a container
1406          *
1407          * @param string $container Resolved container name
1408          * @param array $val Information to cache
1409          * @return void
1410          */
1411         final protected function setContainerCache( $container, array $val ) {
1412                 $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1413         }
1414
1415         /**
1416          * Delete the cached info for a container.
1417          * The cache key is salted for a while to prevent race conditions.
1418          *
1419          * @param string $container Resolved container name
1420          * @return void
1421          */
1422         final protected function deleteContainerCache( $container ) {
1423                 if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
1424                         trigger_error( "Unable to delete stat cache for container $container." );
1425                 }
1426         }
1427
1428         /**
1429          * Do a batch lookup from cache for container stats for all containers
1430          * used in a list of container names, storage paths, or FileOp objects.
1431          * This loads the persistent cache values into the process cache.
1432          *
1433          * @param Array $items
1434          * @return void
1435          */
1436         final protected function primeContainerCache( array $items ) {
1437                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1438
1439                 $paths = array(); // list of storage paths
1440                 $contNames = array(); // (cache key => resolved container name)
1441                 // Get all the paths/containers from the items...
1442                 foreach ( $items as $item ) {
1443                         if ( $item instanceof FileOp ) {
1444                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1445                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1446                         } elseif ( self::isStoragePath( $item ) ) {
1447                                 $paths[] = $item;
1448                         } elseif ( is_string( $item ) ) { // full container name
1449                                 $contNames[$this->containerCacheKey( $item )] = $item;
1450                         }
1451                 }
1452                 // Get all the corresponding cache keys for paths...
1453                 foreach ( $paths as $path ) {
1454                         list( $fullCont, , ) = $this->resolveStoragePath( $path );
1455                         if ( $fullCont !== null ) { // valid path for this backend
1456                                 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1457                         }
1458                 }
1459
1460                 $contInfo = array(); // (resolved container name => cache value)
1461                 // Get all cache entries for these container cache keys...
1462                 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1463                 foreach ( $values as $cacheKey => $val ) {
1464                         $contInfo[$contNames[$cacheKey]] = $val;
1465                 }
1466
1467                 // Populate the container process cache for the backend...
1468                 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1469         }
1470
1471         /**
1472          * Fill the backend-specific process cache given an array of
1473          * resolved container names and their corresponding cached info.
1474          * Only containers that actually exist should appear in the map.
1475          *
1476          * @param array $containerInfo Map of resolved container names to cached info
1477          * @return void
1478          */
1479         protected function doPrimeContainerCache( array $containerInfo ) {}
1480
1481         /**
1482          * Get the cache key for a file path
1483          *
1484          * @param string $path Normalized storage path
1485          * @return string
1486          */
1487         private function fileCacheKey( $path ) {
1488                 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1489         }
1490
1491         /**
1492          * Set the cached stat info for a file path.
1493          * Negatives (404s) are not cached. By not caching negatives, we can skip cache
1494          * salting for the case when a file is created at a path were there was none before.
1495          *
1496          * @param string $path Storage path
1497          * @param array $val Stat information to cache
1498          * @return void
1499          */
1500         final protected function setFileCache( $path, array $val ) {
1501                 $path = FileBackend::normalizeStoragePath( $path );
1502                 if ( $path === null ) {
1503                         return; // invalid storage path
1504                 }
1505                 $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
1506                 $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
1507                 $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
1508         }
1509
1510         /**
1511          * Delete the cached stat info for a file path.
1512          * The cache key is salted for a while to prevent race conditions.
1513          * Since negatives (404s) are not cached, this does not need to be called when
1514          * a file is created at a path were there was none before.
1515          *
1516          * @param string $path Storage path
1517          * @return void
1518          */
1519         final protected function deleteFileCache( $path ) {
1520                 $path = FileBackend::normalizeStoragePath( $path );
1521                 if ( $path === null ) {
1522                         return; // invalid storage path
1523                 }
1524                 if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
1525                         trigger_error( "Unable to delete stat cache for file $path." );
1526                 }
1527         }
1528
1529         /**
1530          * Do a batch lookup from cache for file stats for all paths
1531          * used in a list of storage paths or FileOp objects.
1532          * This loads the persistent cache values into the process cache.
1533          *
1534          * @param array $items List of storage paths or FileOps
1535          * @return void
1536          */
1537         final protected function primeFileCache( array $items ) {
1538                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1539
1540                 $paths = array(); // list of storage paths
1541                 $pathNames = array(); // (cache key => storage path)
1542                 // Get all the paths/containers from the items...
1543                 foreach ( $items as $item ) {
1544                         if ( $item instanceof FileOp ) {
1545                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1546                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1547                         } elseif ( self::isStoragePath( $item ) ) {
1548                                 $paths[] = FileBackend::normalizeStoragePath( $item );
1549                         }
1550                 }
1551                 // Get rid of any paths that failed normalization...
1552                 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1553                 // Get all the corresponding cache keys for paths...
1554                 foreach ( $paths as $path ) {
1555                         list( , $rel, ) = $this->resolveStoragePath( $path );
1556                         if ( $rel !== null ) { // valid path for this backend
1557                                 $pathNames[$this->fileCacheKey( $path )] = $path;
1558                         }
1559                 }
1560                 // Get all cache entries for these container cache keys...
1561                 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1562                 foreach ( $values as $cacheKey => $val ) {
1563                         if ( is_array( $val ) ) {
1564                                 $path = $pathNames[$cacheKey];
1565                                 $this->cheapCache->set( $path, 'stat', $val );
1566                                 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1567                                         $this->cheapCache->set( $path, 'sha1',
1568                                                 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
1569                                 }
1570                         }
1571                 }
1572         }
1573
1574         /**
1575          * Set the 'concurrency' option from a list of operation options
1576          *
1577          * @param array $opts Map of operation options
1578          * @return Array
1579          */
1580         final protected function setConcurrencyFlags( array $opts ) {
1581                 $opts['concurrency'] = 1; // off
1582                 if ( $this->parallelize === 'implicit' ) {
1583                         if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1584                                 $opts['concurrency'] = $this->concurrency;
1585                         }
1586                 } elseif ( $this->parallelize === 'explicit' ) {
1587                         if ( !empty( $opts['parallelize'] ) ) {
1588                                 $opts['concurrency'] = $this->concurrency;
1589                         }
1590                 }
1591                 return $opts;
1592         }
1593
1594         /**
1595          * Get the content type to use in HEAD/GET requests for a file
1596          *
1597          * @param string $storagePath
1598          * @param string|null $content File data
1599          * @param string|null $fsPath File system path
1600          * @return MIME type
1601          */
1602         protected function getContentType( $storagePath, $content, $fsPath ) {
1603                 return call_user_func_array( $this->mimeCallback, func_get_args() );
1604         }
1605 }
1606
1607 /**
1608  * FileBackendStore helper class for performing asynchronous file operations.
1609  *
1610  * For example, calling FileBackendStore::createInternal() with the "async"
1611  * param flag may result in a Status that contains this object as a value.
1612  * This class is largely backend-specific and is mostly just "magic" to be
1613  * passed to FileBackendStore::executeOpHandlesInternal().
1614  */
1615 abstract class FileBackendStoreOpHandle {
1616         /** @var Array */
1617         public $params = array(); // params to caller functions
1618         /** @var FileBackendStore */
1619         public $backend;
1620         /** @var Array */
1621         public $resourcesToClose = array();
1622
1623         public $call; // string; name that identifies the function called
1624
1625         /**
1626          * Close all open file handles
1627          *
1628          * @return void
1629          */
1630         public function closeResources() {
1631                 array_map( 'fclose', $this->resourcesToClose );
1632         }
1633 }
1634
1635 /**
1636  * FileBackendStore helper function to handle listings that span container shards.
1637  * Do not use this class from places outside of FileBackendStore.
1638  *
1639  * @ingroup FileBackend
1640  */
1641 abstract class FileBackendStoreShardListIterator extends FilterIterator {
1642         /** @var FileBackendStore */
1643         protected $backend;
1644         /** @var Array */
1645         protected $params;
1646
1647         protected $container; // string; full container name
1648         protected $directory; // string; resolved relative path
1649
1650         /** @var Array */
1651         protected $multiShardPaths = array(); // (rel path => 1)
1652
1653         /**
1654          * @param FileBackendStore $backend
1655          * @param string $container Full storage container name
1656          * @param string $dir Storage directory relative to container
1657          * @param array $suffixes List of container shard suffixes
1658          * @param array $params
1659          */
1660         public function __construct(
1661                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1662         ) {
1663                 $this->backend = $backend;
1664                 $this->container = $container;
1665                 $this->directory = $dir;
1666                 $this->params = $params;
1667
1668                 $iter = new AppendIterator();
1669                 foreach ( $suffixes as $suffix ) {
1670                         $iter->append( $this->listFromShard( $this->container . $suffix ) );
1671                 }
1672
1673                 parent::__construct( $iter );
1674         }
1675
1676         public function accept() {
1677                 $rel = $this->getInnerIterator()->current(); // path relative to given directory
1678                 $path = $this->params['dir'] . "/{$rel}"; // full storage path
1679                 if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1680                         return true; // path is only on one shard; no issue with duplicates
1681                 } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1682                         // Don't keep listing paths that are on multiple shards
1683                         return false;
1684                 } else {
1685                         $this->multiShardPaths[$rel] = 1;
1686                         return true;
1687                 }
1688         }
1689
1690         public function rewind() {
1691                 parent::rewind();
1692                 $this->multiShardPaths = array();
1693         }
1694
1695         /**
1696          * Get the list for a given container shard
1697          *
1698          * @param string $container Resolved container name
1699          * @return Iterator
1700          */
1701         abstract protected function listFromShard( $container );
1702 }
1703
1704 /**
1705  * Iterator for listing directories
1706  */
1707 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1708         protected function listFromShard( $container ) {
1709                 $list = $this->backend->getDirectoryListInternal(
1710                         $container, $this->directory, $this->params );
1711                 if ( $list === null ) {
1712                         return new ArrayIterator( array() );
1713                 } else {
1714                         return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1715                 }
1716         }
1717 }
1718
1719 /**
1720  * Iterator for listing regular files
1721  */
1722 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1723         protected function listFromShard( $container ) {
1724                 $list = $this->backend->getFileListInternal(
1725                         $container, $this->directory, $this->params );
1726                 if ( $list === null ) {
1727                         return new ArrayIterator( array() );
1728                 } else {
1729                         return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1730                 }
1731         }
1732 }