includes/filerepo/LocalRepo.php

   1 <?php
   2 /**
   3  * Local repository that stores files in the local filesystem and registers them
   4  * in the wiki's own database.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19  * http://www.gnu.org/copyleft/gpl.html
  20  *
  21  * @file
  22  * @ingroup FileRepo
  23  */
  24
  25 /**
  26  * A repository that stores files in the local filesystem and registers them
  27  * in the wiki's own database. This is the most commonly used repository class.
  28  *
  29  * @ingroup FileRepo
  30  */
  31 class LocalRepo extends FileRepo {
  32         /** @var callable */
  33         protected $fileFactory = [ 'LocalFile', 'newFromTitle' ];
  34         /** @var callable */
  35         protected $fileFactoryKey = [ 'LocalFile', 'newFromKey' ];
  36         /** @var callable */
  37         protected $fileFromRowFactory = [ 'LocalFile', 'newFromRow' ];
  38         /** @var callable */
  39         protected $oldFileFromRowFactory = [ 'OldLocalFile', 'newFromRow' ];
  40         /** @var callable */
  41         protected $oldFileFactory = [ 'OldLocalFile', 'newFromTitle' ];
  42         /** @var callable */
  43         protected $oldFileFactoryKey = [ 'OldLocalFile', 'newFromKey' ];
  44
  45         function __construct( array $info = null ) {
  46                 parent::__construct( $info );
  47
  48                 $this->hasSha1Storage = isset( $info['storageLayout'] )
  49                         && $info['storageLayout'] === 'sha1';
  50
  51                 if ( $this->hasSha1Storage() ) {
  52                         $this->backend = new FileBackendDBRepoWrapper( [
  53                                 'backend'         => $this->backend,
  54                                 'repoName'        => $this->name,
  55                                 'dbHandleFactory' => $this->getDBFactory()
  56                         ] );
  57                 }
  58         }
  59
  60         /**
  61          * @throws MWException
  62          * @param stdClass $row
  63          * @return LocalFile
  64          */
  65         function newFileFromRow( $row ) {
  66                 if ( isset( $row->img_name ) ) {
  67                         return call_user_func( $this->fileFromRowFactory, $row, $this );
  68                 } elseif ( isset( $row->oi_name ) ) {
  69                         return call_user_func( $this->oldFileFromRowFactory, $row, $this );
  70                 } else {
  71                         throw new MWException( __METHOD__ . ': invalid row' );
  72                 }
  73         }
  74
  75         /**
  76          * @param Title $title
  77          * @param string $archiveName
  78          * @return OldLocalFile
  79          */
  80         function newFromArchiveName( $title, $archiveName ) {
  81                 return OldLocalFile::newFromArchiveName( $title, $this, $archiveName );
  82         }
  83
  84         /**
  85          * Delete files in the deleted directory if they are not referenced in the
  86          * filearchive table. This needs to be done in the repo because it needs to
  87          * interleave database locks with file operations, which is potentially a
  88          * remote operation.
  89          *
  90          * @param array $storageKeys
  91          *
  92          * @return Status
  93          */
  94         function cleanupDeletedBatch( array $storageKeys ) {
  95                 if ( $this->hasSha1Storage() ) {
  96                         wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
  97                         return Status::newGood();
  98                 }
  99
 100                 $backend = $this->backend; // convenience
 101                 $root = $this->getZonePath( 'deleted' );
 102                 $dbw = $this->getMasterDB();
 103                 $status = $this->newGood();
 104                 $storageKeys = array_unique( $storageKeys );
 105                 foreach ( $storageKeys as $key ) {
 106                         $hashPath = $this->getDeletedHashPath( $key );
 107                         $path = "$root/$hashPath$key";
 108                         $dbw->startAtomic( __METHOD__ );
 109                         // Check for usage in deleted/hidden files and preemptively
 110                         // lock the key to avoid any future use until we are finished.
 111                         $deleted = $this->deletedFileHasKey( $key, 'lock' );
 112                         $hidden = $this->hiddenFileHasKey( $key, 'lock' );
 113                         if ( !$deleted && !$hidden ) { // not in use now
 114                                 wfDebug( __METHOD__ . ": deleting $key\n" );
 115                                 $op = [ 'op' => 'delete', 'src' => $path ];
 116                                 if ( !$backend->doOperation( $op )->isOK() ) {
 117                                         $status->error( 'undelete-cleanup-error', $path );
 118                                         $status->failCount++;
 119                                 }
 120                         } else {
 121                                 wfDebug( __METHOD__ . ": $key still in use\n" );
 122                                 $status->successCount++;
 123                         }
 124                         $dbw->endAtomic( __METHOD__ );
 125                 }
 126
 127                 return $status;
 128         }
 129
 130         /**
 131          * Check if a deleted (filearchive) file has this sha1 key
 132          *
 133          * @param string $key File storage key (base-36 sha1 key with file extension)
 134          * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
 135          * @return bool File with this key is in use
 136          */
 137         protected function deletedFileHasKey( $key, $lock = null ) {
 138                 $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
 139
 140                 $dbw = $this->getMasterDB();
 141
 142                 return (bool)$dbw->selectField( 'filearchive', '1',
 143                         [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ],
 144                         __METHOD__, $options
 145                 );
 146         }
 147
 148         /**
 149          * Check if a hidden (revision delete) file has this sha1 key
 150          *
 151          * @param string $key File storage key (base-36 sha1 key with file extension)
 152          * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
 153          * @return bool File with this key is in use
 154          */
 155         protected function hiddenFileHasKey( $key, $lock = null ) {
 156                 $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
 157
 158                 $sha1 = self::getHashFromKey( $key );
 159                 $ext = File::normalizeExtension( substr( $key, strcspn( $key, '.' ) + 1 ) );
 160
 161                 $dbw = $this->getMasterDB();
 162
 163                 return (bool)$dbw->selectField( 'oldimage', '1',
 164                         [ 'oi_sha1' => $sha1,
 165                                 'oi_archive_name ' . $dbw->buildLike( $dbw->anyString(), ".$ext" ),
 166                                 $dbw->bitAnd( 'oi_deleted', File::DELETED_FILE ) => File::DELETED_FILE ],
 167                         __METHOD__, $options
 168                 );
 169         }
 170
 171         /**
 172          * Gets the SHA1 hash from a storage key
 173          *
 174          * @param string $key
 175          * @return string
 176          */
 177         public static function getHashFromKey( $key ) {
 178                 return strtok( $key, '.' );
 179         }
 180
 181         /**
 182          * Checks if there is a redirect named as $title
 183          *
 184          * @param Title $title Title of file
 185          * @return bool|Title
 186          */
 187         function checkRedirect( Title $title ) {
 188                 $title = File::normalizeTitle( $title, 'exception' );
 189
 190                 $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 191                 if ( $memcKey === false ) {
 192                         $memcKey = $this->getLocalCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 193                         $expiry = 300; // no invalidation, 5 minutes
 194                 } else {
 195                         $expiry = 86400; // has invalidation, 1 day
 196                 }
 197
 198                 $method = __METHOD__;
 199                 $redirDbKey = ObjectCache::getMainWANInstance()->getWithSetCallback(
 200                         $memcKey,
 201                         $expiry,
 202                         function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) {
 203                                 $dbr = $this->getReplicaDB(); // possibly remote DB
 204
 205                                 $setOpts += Database::getCacheSetOptions( $dbr );
 206
 207                                 if ( $title instanceof Title ) {
 208                                         $row = $dbr->selectRow(
 209                                                 [ 'page', 'redirect' ],
 210                                                 [ 'rd_namespace', 'rd_title' ],
 211                                                 [
 212                                                         'page_namespace' => $title->getNamespace(),
 213                                                         'page_title' => $title->getDBkey(),
 214                                                         'rd_from = page_id'
 215                                                 ],
 216                                                 $method
 217                                         );
 218                                 } else {
 219                                         $row = false;
 220                                 }
 221
 222                                 return ( $row && $row->rd_namespace == NS_FILE )
 223                                         ? Title::makeTitle( $row->rd_namespace, $row->rd_title )->getDBkey()
 224                                         : ''; // negative cache
 225                         },
 226                         [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
 227                 );
 228
 229                 // @note: also checks " " for b/c
 230                 if ( $redirDbKey !== ' ' && strval( $redirDbKey ) !== '' ) {
 231                         // Page is a redirect to another file
 232                         return Title::newFromText( $redirDbKey, NS_FILE );
 233                 }
 234
 235                 return false; // no redirect
 236         }
 237
 238         public function findFiles( array $items, $flags = 0 ) {
 239                 $finalFiles = []; // map of (DB key => corresponding File) for matches
 240
 241                 $searchSet = []; // map of (normalized DB key => search params)
 242                 foreach ( $items as $item ) {
 243                         if ( is_array( $item ) ) {
 244                                 $title = File::normalizeTitle( $item['title'] );
 245                                 if ( $title ) {
 246                                         $searchSet[$title->getDBkey()] = $item;
 247                                 }
 248                         } else {
 249                                 $title = File::normalizeTitle( $item );
 250                                 if ( $title ) {
 251                                         $searchSet[$title->getDBkey()] = [];
 252                                 }
 253                         }
 254                 }
 255
 256                 $fileMatchesSearch = function ( File $file, array $search ) {
 257                         // Note: file name comparison done elsewhere (to handle redirects)
 258                         $user = ( !empty( $search['private'] ) && $search['private'] instanceof User )
 259                                 ? $search['private']
 260                                 : null;
 261
 262                         return (
 263                                 $file->exists() &&
 264                                 (
 265                                         ( empty( $search['time'] ) && !$file->isOld() ) ||
 266                                         ( !empty( $search['time'] ) && $search['time'] === $file->getTimestamp() )
 267                                 ) &&
 268                                 ( !empty( $search['private'] ) || !$file->isDeleted( File::DELETED_FILE ) ) &&
 269                                 $file->userCan( File::DELETED_FILE, $user )
 270                         );
 271                 };
 272
 273                 $that = $this;
 274                 $applyMatchingFiles = function ( ResultWrapper $res, &$searchSet, &$finalFiles )
 275                         use ( $that, $fileMatchesSearch, $flags )
 276                 {
 277                         global $wgContLang;
 278                         $info = $that->getInfo();
 279                         foreach ( $res as $row ) {
 280                                 $file = $that->newFileFromRow( $row );
 281                                 // There must have been a search for this DB key, but this has to handle the
 282                                 // cases were title capitalization is different on the client and repo wikis.
 283                                 $dbKeysLook = [ strtr( $file->getName(), ' ', '_' ) ];
 284                                 if ( !empty( $info['initialCapital'] ) ) {
 285                                         // Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
 286                                         $dbKeysLook[] = $wgContLang->lcfirst( $file->getName() );
 287                                 }
 288                                 foreach ( $dbKeysLook as $dbKey ) {
 289                                         if ( isset( $searchSet[$dbKey] )
 290                                                 && $fileMatchesSearch( $file, $searchSet[$dbKey] )
 291                                         ) {
 292                                                 $finalFiles[$dbKey] = ( $flags & FileRepo::NAME_AND_TIME_ONLY )
 293                                                         ? [ 'title' => $dbKey, 'timestamp' => $file->getTimestamp() ]
 294                                                         : $file;
 295                                                 unset( $searchSet[$dbKey] );
 296                                         }
 297                                 }
 298                         }
 299                 };
 300
 301                 $dbr = $this->getReplicaDB();
 302
 303                 // Query image table
 304                 $imgNames = [];
 305                 foreach ( array_keys( $searchSet ) as $dbKey ) {
 306                         $imgNames[] = $this->getNameFromTitle( File::normalizeTitle( $dbKey ) );
 307                 }
 308
 309                 if ( count( $imgNames ) ) {
 310                         $res = $dbr->select( 'image',
 311                                 LocalFile::selectFields(), [ 'img_name' => $imgNames ], __METHOD__ );
 312                         $applyMatchingFiles( $res, $searchSet, $finalFiles );
 313                 }
 314
 315                 // Query old image table
 316                 $oiConds = []; // WHERE clause array for each file
 317                 foreach ( $searchSet as $dbKey => $search ) {
 318                         if ( isset( $search['time'] ) ) {
 319                                 $oiConds[] = $dbr->makeList(
 320                                         [
 321                                                 'oi_name' => $this->getNameFromTitle( File::normalizeTitle( $dbKey ) ),
 322                                                 'oi_timestamp' => $dbr->timestamp( $search['time'] )
 323                                         ],
 324                                         LIST_AND
 325                                 );
 326                         }
 327                 }
 328
 329                 if ( count( $oiConds ) ) {
 330                         $res = $dbr->select( 'oldimage',
 331                                 OldLocalFile::selectFields(), $dbr->makeList( $oiConds, LIST_OR ), __METHOD__ );
 332                         $applyMatchingFiles( $res, $searchSet, $finalFiles );
 333                 }
 334
 335                 // Check for redirects...
 336                 foreach ( $searchSet as $dbKey => $search ) {
 337                         if ( !empty( $search['ignoreRedirect'] ) ) {
 338                                 continue;
 339                         }
 340
 341                         $title = File::normalizeTitle( $dbKey );
 342                         $redir = $this->checkRedirect( $title ); // hopefully hits memcached
 343
 344                         if ( $redir && $redir->getNamespace() == NS_FILE ) {
 345                                 $file = $this->newFile( $redir );
 346                                 if ( $file && $fileMatchesSearch( $file, $search ) ) {
 347                                         $file->redirectedFrom( $title->getDBkey() );
 348                                         if ( $flags & FileRepo::NAME_AND_TIME_ONLY ) {
 349                                                 $finalFiles[$dbKey] = [
 350                                                         'title' => $file->getTitle()->getDBkey(),
 351                                                         'timestamp' => $file->getTimestamp()
 352                                                 ];
 353                                         } else {
 354                                                 $finalFiles[$dbKey] = $file;
 355                                         }
 356                                 }
 357                         }
 358                 }
 359
 360                 return $finalFiles;
 361         }
 362
 363         /**
 364          * Get an array or iterator of file objects for files that have a given
 365          * SHA-1 content hash.
 366          *
 367          * @param string $hash A sha1 hash to look for
 368          * @return File[]
 369          */
 370         function findBySha1( $hash ) {
 371                 $dbr = $this->getReplicaDB();
 372                 $res = $dbr->select(
 373                         'image',
 374                         LocalFile::selectFields(),
 375                         [ 'img_sha1' => $hash ],
 376                         __METHOD__,
 377                         [ 'ORDER BY' => 'img_name' ]
 378                 );
 379
 380                 $result = [];
 381                 foreach ( $res as $row ) {
 382                         $result[] = $this->newFileFromRow( $row );
 383                 }
 384                 $res->free();
 385
 386                 return $result;
 387         }
 388
 389         /**
 390          * Get an array of arrays or iterators of file objects for files that
 391          * have the given SHA-1 content hashes.
 392          *
 393          * Overrides generic implementation in FileRepo for performance reason
 394          *
 395          * @param array $hashes An array of hashes
 396          * @return array An Array of arrays or iterators of file objects and the hash as key
 397          */
 398         function findBySha1s( array $hashes ) {
 399                 if ( !count( $hashes ) ) {
 400                         return []; // empty parameter
 401                 }
 402
 403                 $dbr = $this->getReplicaDB();
 404                 $res = $dbr->select(
 405                         'image',
 406                         LocalFile::selectFields(),
 407                         [ 'img_sha1' => $hashes ],
 408                         __METHOD__,
 409                         [ 'ORDER BY' => 'img_name' ]
 410                 );
 411
 412                 $result = [];
 413                 foreach ( $res as $row ) {
 414                         $file = $this->newFileFromRow( $row );
 415                         $result[$file->getSha1()][] = $file;
 416                 }
 417                 $res->free();
 418
 419                 return $result;
 420         }
 421
 422         /**
 423          * Return an array of files where the name starts with $prefix.
 424          *
 425          * @param string $prefix The prefix to search for
 426          * @param int $limit The maximum amount of files to return
 427          * @return array
 428          */
 429         public function findFilesByPrefix( $prefix, $limit ) {
 430                 $selectOptions = [ 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ];
 431
 432                 // Query database
 433                 $dbr = $this->getReplicaDB();
 434                 $res = $dbr->select(
 435                         'image',
 436                         LocalFile::selectFields(),
 437                         'img_name ' . $dbr->buildLike( $prefix, $dbr->anyString() ),
 438                         __METHOD__,
 439                         $selectOptions
 440                 );
 441
 442                 // Build file objects
 443                 $files = [];
 444                 foreach ( $res as $row ) {
 445                         $files[] = $this->newFileFromRow( $row );
 446                 }
 447
 448                 return $files;
 449         }
 450
 451         /**
 452          * Get a connection to the replica DB
 453          * @return IDatabase
 454          */
 455         function getReplicaDB() {
 456                 return wfGetDB( DB_REPLICA );
 457         }
 458
 459         /**
 460          * Alias for getReplicaDB()
 461          *
 462          * @return IDatabase
 463          * @deprecated Since 1.29
 464          */
 465         function getSlaveDB() {
 466                 return $this->getReplicaDB();
 467         }
 468
 469         /**
 470          * Get a connection to the master DB
 471          * @return IDatabase
 472          */
 473         function getMasterDB() {
 474                 return wfGetDB( DB_MASTER );
 475         }
 476
 477         /**
 478          * Get a callback to get a DB handle given an index (DB_REPLICA/DB_MASTER)
 479          * @return Closure
 480          */
 481         protected function getDBFactory() {
 482                 return function( $index ) {
 483                         return wfGetDB( $index );
 484                 };
 485         }
 486
 487         /**
 488          * Get a key on the primary cache for this repository.
 489          * Returns false if the repository's cache is not accessible at this site.
 490          * The parameters are the parts of the key, as for wfMemcKey().
 491          *
 492          * @return string
 493          */
 494         function getSharedCacheKey( /*...*/ ) {
 495                 $args = func_get_args();
 496
 497                 return call_user_func_array( 'wfMemcKey', $args );
 498         }
 499
 500         /**
 501          * Invalidates image redirect cache related to that image
 502          *
 503          * @param Title $title Title of page
 504          * @return void
 505          */
 506         function invalidateImageRedirect( Title $title ) {
 507                 $key = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 508                 if ( $key ) {
 509                         $this->getMasterDB()->onTransactionPreCommitOrIdle(
 510                                 function () use ( $key ) {
 511                                         ObjectCache::getMainWANInstance()->delete( $key );
 512                                 },
 513                                 __METHOD__
 514                         );
 515                 }
 516         }
 517
 518         /**
 519          * Return information about the repository.
 520          *
 521          * @return array
 522          * @since 1.22
 523          */
 524         function getInfo() {
 525                 global $wgFavicon;
 526
 527                 return array_merge( parent::getInfo(), [
 528                         'favicon' => wfExpandUrl( $wgFavicon ),
 529                 ] );
 530         }
 531
 532         public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
 533                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 534         }
 535
 536         public function storeBatch( array $triplets, $flags = 0 ) {
 537                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 538         }
 539
 540         public function cleanupBatch( array $files, $flags = 0 ) {
 541                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 542         }
 543
 544         public function publish(
 545                 $src,
 546                 $dstRel,
 547                 $archiveRel,
 548                 $flags = 0,
 549                 array $options = []
 550         ) {
 551                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 552         }
 553
 554         public function publishBatch( array $ntuples, $flags = 0 ) {
 555                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 556         }
 557
 558         public function delete( $srcRel, $archiveRel ) {
 559                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 560         }
 561
 562         public function deleteBatch( array $sourceDestPairs ) {
 563                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 564         }
 565
 566         /**
 567          * Skips the write operation if storage is sha1-based, executes it normally otherwise
 568          *
 569          * @param string $function
 570          * @param array $args
 571          * @return Status
 572          */
 573         protected function skipWriteOperationIfSha1( $function, array $args ) {
 574                 $this->assertWritableRepo(); // fail out if read-only
 575
 576                 if ( $this->hasSha1Storage() ) {
 577                         wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
 578                         return Status::newGood();
 579                 } else {
 580                         return call_user_func_array( 'parent::' . $function, $args );
 581                 }
 582         }
 583 }