includes/filerepo/LocalRepo.php

   1 <?php
   2 /**
   3  * Local repository that stores files in the local filesystem and registers them
   4  * in the wiki's own database.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19  * http://www.gnu.org/copyleft/gpl.html
  20  *
  21  * @file
  22  * @ingroup FileRepo
  23  */
  24
  25 use MediaWiki\MediaWikiServices;
  26 use Wikimedia\Rdbms\IResultWrapper;
  27 use Wikimedia\Rdbms\Database;
  28 use Wikimedia\Rdbms\IDatabase;
  29
  30 /**
  31  * A repository that stores files in the local filesystem and registers them
  32  * in the wiki's own database. This is the most commonly used repository class.
  33  *
  34  * @ingroup FileRepo
  35  * @method LocalFile|null newFile( $title, $time = false )
  36  */
  37 class LocalRepo extends FileRepo {
  38         /** @var callable */
  39         protected $fileFactory = [ LocalFile::class, 'newFromTitle' ];
  40         /** @var callable */
  41         protected $fileFactoryKey = [ LocalFile::class, 'newFromKey' ];
  42         /** @var callable */
  43         protected $fileFromRowFactory = [ LocalFile::class, 'newFromRow' ];
  44         /** @var callable */
  45         protected $oldFileFromRowFactory = [ OldLocalFile::class, 'newFromRow' ];
  46         /** @var callable */
  47         protected $oldFileFactory = [ OldLocalFile::class, 'newFromTitle' ];
  48         /** @var callable */
  49         protected $oldFileFactoryKey = [ OldLocalFile::class, 'newFromKey' ];
  50
  51         function __construct( array $info = null ) {
  52                 parent::__construct( $info );
  53
  54                 $this->hasSha1Storage = isset( $info['storageLayout'] )
  55                         && $info['storageLayout'] === 'sha1';
  56
  57                 if ( $this->hasSha1Storage() ) {
  58                         $this->backend = new FileBackendDBRepoWrapper( [
  59                                 'backend'         => $this->backend,
  60                                 'repoName'        => $this->name,
  61                                 'dbHandleFactory' => $this->getDBFactory()
  62                         ] );
  63                 }
  64         }
  65
  66         /**
  67          * @throws MWException
  68          * @param stdClass $row
  69          * @return LocalFile
  70          */
  71         function newFileFromRow( $row ) {
  72                 if ( isset( $row->img_name ) ) {
  73                         return call_user_func( $this->fileFromRowFactory, $row, $this );
  74                 } elseif ( isset( $row->oi_name ) ) {
  75                         return call_user_func( $this->oldFileFromRowFactory, $row, $this );
  76                 } else {
  77                         throw new MWException( __METHOD__ . ': invalid row' );
  78                 }
  79         }
  80
  81         /**
  82          * @param Title $title
  83          * @param string $archiveName
  84          * @return OldLocalFile
  85          */
  86         function newFromArchiveName( $title, $archiveName ) {
  87                 return OldLocalFile::newFromArchiveName( $title, $this, $archiveName );
  88         }
  89
  90         /**
  91          * Delete files in the deleted directory if they are not referenced in the
  92          * filearchive table. This needs to be done in the repo because it needs to
  93          * interleave database locks with file operations, which is potentially a
  94          * remote operation.
  95          *
  96          * @param string[] $storageKeys
  97          *
  98          * @return Status
  99          */
 100         function cleanupDeletedBatch( array $storageKeys ) {
 101                 if ( $this->hasSha1Storage() ) {
 102                         wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
 103                         return Status::newGood();
 104                 }
 105
 106                 $backend = $this->backend; // convenience
 107                 $root = $this->getZonePath( 'deleted' );
 108                 $dbw = $this->getMasterDB();
 109                 $status = $this->newGood();
 110                 $storageKeys = array_unique( $storageKeys );
 111                 foreach ( $storageKeys as $key ) {
 112                         $hashPath = $this->getDeletedHashPath( $key );
 113                         $path = "$root/$hashPath$key";
 114                         $dbw->startAtomic( __METHOD__ );
 115                         // Check for usage in deleted/hidden files and preemptively
 116                         // lock the key to avoid any future use until we are finished.
 117                         $deleted = $this->deletedFileHasKey( $key, 'lock' );
 118                         $hidden = $this->hiddenFileHasKey( $key, 'lock' );
 119                         if ( !$deleted && !$hidden ) { // not in use now
 120                                 wfDebug( __METHOD__ . ": deleting $key\n" );
 121                                 $op = [ 'op' => 'delete', 'src' => $path ];
 122                                 if ( !$backend->doOperation( $op )->isOK() ) {
 123                                         $status->error( 'undelete-cleanup-error', $path );
 124                                         $status->failCount++;
 125                                 }
 126                         } else {
 127                                 wfDebug( __METHOD__ . ": $key still in use\n" );
 128                                 $status->successCount++;
 129                         }
 130                         $dbw->endAtomic( __METHOD__ );
 131                 }
 132
 133                 return $status;
 134         }
 135
 136         /**
 137          * Check if a deleted (filearchive) file has this sha1 key
 138          *
 139          * @param string $key File storage key (base-36 sha1 key with file extension)
 140          * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
 141          * @return bool File with this key is in use
 142          */
 143         protected function deletedFileHasKey( $key, $lock = null ) {
 144                 $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
 145
 146                 $dbw = $this->getMasterDB();
 147
 148                 return (bool)$dbw->selectField( 'filearchive', '1',
 149                         [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ],
 150                         __METHOD__, $options
 151                 );
 152         }
 153
 154         /**
 155          * Check if a hidden (revision delete) file has this sha1 key
 156          *
 157          * @param string $key File storage key (base-36 sha1 key with file extension)
 158          * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
 159          * @return bool File with this key is in use
 160          */
 161         protected function hiddenFileHasKey( $key, $lock = null ) {
 162                 $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
 163
 164                 $sha1 = self::getHashFromKey( $key );
 165                 $ext = File::normalizeExtension( substr( $key, strcspn( $key, '.' ) + 1 ) );
 166
 167                 $dbw = $this->getMasterDB();
 168
 169                 return (bool)$dbw->selectField( 'oldimage', '1',
 170                         [ 'oi_sha1' => $sha1,
 171                                 'oi_archive_name ' . $dbw->buildLike( $dbw->anyString(), ".$ext" ),
 172                                 $dbw->bitAnd( 'oi_deleted', File::DELETED_FILE ) => File::DELETED_FILE ],
 173                         __METHOD__, $options
 174                 );
 175         }
 176
 177         /**
 178          * Gets the SHA1 hash from a storage key
 179          *
 180          * @param string $key
 181          * @return string
 182          */
 183         public static function getHashFromKey( $key ) {
 184                 $sha1 = strtok( $key, '.' );
 185                 if ( is_string( $sha1 ) && strlen( $sha1 ) === 32 && $sha1[0] === '0' ) {
 186                         $sha1 = substr( $sha1, 1 );
 187                 }
 188                 return $sha1;
 189         }
 190
 191         /**
 192          * Checks if there is a redirect named as $title
 193          *
 194          * @param Title $title Title of file
 195          * @return bool|Title
 196          */
 197         function checkRedirect( Title $title ) {
 198                 $title = File::normalizeTitle( $title, 'exception' );
 199
 200                 $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 201                 if ( $memcKey === false ) {
 202                         $memcKey = $this->getLocalCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 203                         $expiry = 300; // no invalidation, 5 minutes
 204                 } else {
 205                         $expiry = 86400; // has invalidation, 1 day
 206                 }
 207
 208                 $method = __METHOD__;
 209                 $redirDbKey = $this->wanCache->getWithSetCallback(
 210                         $memcKey,
 211                         $expiry,
 212                         function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) {
 213                                 $dbr = $this->getReplicaDB(); // possibly remote DB
 214
 215                                 $setOpts += Database::getCacheSetOptions( $dbr );
 216
 217                                 $row = $dbr->selectRow(
 218                                         [ 'page', 'redirect' ],
 219                                         [ 'rd_namespace', 'rd_title' ],
 220                                         [
 221                                                 'page_namespace' => $title->getNamespace(),
 222                                                 'page_title' => $title->getDBkey(),
 223                                                 'rd_from = page_id'
 224                                         ],
 225                                         $method
 226                                 );
 227
 228                                 return ( $row && $row->rd_namespace == NS_FILE )
 229                                         ? Title::makeTitle( $row->rd_namespace, $row->rd_title )->getDBkey()
 230                                         : ''; // negative cache
 231                         },
 232                         [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
 233                 );
 234
 235                 // @note: also checks " " for b/c
 236                 if ( $redirDbKey !== ' ' && strval( $redirDbKey ) !== '' ) {
 237                         // Page is a redirect to another file
 238                         return Title::newFromText( $redirDbKey, NS_FILE );
 239                 }
 240
 241                 return false; // no redirect
 242         }
 243
 244         public function findFiles( array $items, $flags = 0 ) {
 245                 $finalFiles = []; // map of (DB key => corresponding File) for matches
 246
 247                 $searchSet = []; // map of (normalized DB key => search params)
 248                 foreach ( $items as $item ) {
 249                         if ( is_array( $item ) ) {
 250                                 $title = File::normalizeTitle( $item['title'] );
 251                                 if ( $title ) {
 252                                         $searchSet[$title->getDBkey()] = $item;
 253                                 }
 254                         } else {
 255                                 $title = File::normalizeTitle( $item );
 256                                 if ( $title ) {
 257                                         $searchSet[$title->getDBkey()] = [];
 258                                 }
 259                         }
 260                 }
 261
 262                 $fileMatchesSearch = function ( File $file, array $search ) {
 263                         // Note: file name comparison done elsewhere (to handle redirects)
 264                         $user = ( !empty( $search['private'] ) && $search['private'] instanceof User )
 265                                 ? $search['private']
 266                                 : null;
 267
 268                         return (
 269                                 $file->exists() &&
 270                                 (
 271                                         ( empty( $search['time'] ) && !$file->isOld() ) ||
 272                                         ( !empty( $search['time'] ) && $search['time'] === $file->getTimestamp() )
 273                                 ) &&
 274                                 ( !empty( $search['private'] ) || !$file->isDeleted( File::DELETED_FILE ) ) &&
 275                                 $file->userCan( File::DELETED_FILE, $user )
 276                         );
 277                 };
 278
 279                 $applyMatchingFiles = function ( IResultWrapper $res, &$searchSet, &$finalFiles )
 280                         use ( $fileMatchesSearch, $flags )
 281                 {
 282                         $contLang = MediaWikiServices::getInstance()->getContentLanguage();
 283                         $info = $this->getInfo();
 284                         foreach ( $res as $row ) {
 285                                 $file = $this->newFileFromRow( $row );
 286                                 // There must have been a search for this DB key, but this has to handle the
 287                                 // cases were title capitalization is different on the client and repo wikis.
 288                                 $dbKeysLook = [ strtr( $file->getName(), ' ', '_' ) ];
 289                                 if ( !empty( $info['initialCapital'] ) ) {
 290                                         // Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
 291                                         $dbKeysLook[] = $contLang->lcfirst( $file->getName() );
 292                                 }
 293                                 foreach ( $dbKeysLook as $dbKey ) {
 294                                         if ( isset( $searchSet[$dbKey] )
 295                                                 && $fileMatchesSearch( $file, $searchSet[$dbKey] )
 296                                         ) {
 297                                                 $finalFiles[$dbKey] = ( $flags & FileRepo::NAME_AND_TIME_ONLY )
 298                                                         ? [ 'title' => $dbKey, 'timestamp' => $file->getTimestamp() ]
 299                                                         : $file;
 300                                                 unset( $searchSet[$dbKey] );
 301                                         }
 302                                 }
 303                         }
 304                 };
 305
 306                 $dbr = $this->getReplicaDB();
 307
 308                 // Query image table
 309                 $imgNames = [];
 310                 foreach ( array_keys( $searchSet ) as $dbKey ) {
 311                         $imgNames[] = $this->getNameFromTitle( File::normalizeTitle( $dbKey ) );
 312                 }
 313
 314                 if ( count( $imgNames ) ) {
 315                         $fileQuery = LocalFile::getQueryInfo();
 316                         $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'], [ 'img_name' => $imgNames ],
 317                                 __METHOD__, [], $fileQuery['joins'] );
 318                         $applyMatchingFiles( $res, $searchSet, $finalFiles );
 319                 }
 320
 321                 // Query old image table
 322                 $oiConds = []; // WHERE clause array for each file
 323                 foreach ( $searchSet as $dbKey => $search ) {
 324                         if ( isset( $search['time'] ) ) {
 325                                 $oiConds[] = $dbr->makeList(
 326                                         [
 327                                                 'oi_name' => $this->getNameFromTitle( File::normalizeTitle( $dbKey ) ),
 328                                                 'oi_timestamp' => $dbr->timestamp( $search['time'] )
 329                                         ],
 330                                         LIST_AND
 331                                 );
 332                         }
 333                 }
 334
 335                 if ( count( $oiConds ) ) {
 336                         $fileQuery = OldLocalFile::getQueryInfo();
 337                         $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'],
 338                                 $dbr->makeList( $oiConds, LIST_OR ),
 339                                 __METHOD__, [], $fileQuery['joins'] );
 340                         $applyMatchingFiles( $res, $searchSet, $finalFiles );
 341                 }
 342
 343                 // Check for redirects...
 344                 foreach ( $searchSet as $dbKey => $search ) {
 345                         if ( !empty( $search['ignoreRedirect'] ) ) {
 346                                 continue;
 347                         }
 348
 349                         $title = File::normalizeTitle( $dbKey );
 350                         $redir = $this->checkRedirect( $title ); // hopefully hits memcached
 351
 352                         if ( $redir && $redir->getNamespace() == NS_FILE ) {
 353                                 $file = $this->newFile( $redir );
 354                                 if ( $file && $fileMatchesSearch( $file, $search ) ) {
 355                                         $file->redirectedFrom( $title->getDBkey() );
 356                                         if ( $flags & FileRepo::NAME_AND_TIME_ONLY ) {
 357                                                 $finalFiles[$dbKey] = [
 358                                                         'title' => $file->getTitle()->getDBkey(),
 359                                                         'timestamp' => $file->getTimestamp()
 360                                                 ];
 361                                         } else {
 362                                                 $finalFiles[$dbKey] = $file;
 363                                         }
 364                                 }
 365                         }
 366                 }
 367
 368                 return $finalFiles;
 369         }
 370
 371         /**
 372          * Get an array or iterator of file objects for files that have a given
 373          * SHA-1 content hash.
 374          *
 375          * @param string $hash A sha1 hash to look for
 376          * @return LocalFile[]
 377          */
 378         function findBySha1( $hash ) {
 379                 $dbr = $this->getReplicaDB();
 380                 $fileQuery = LocalFile::getQueryInfo();
 381                 $res = $dbr->select(
 382                         $fileQuery['tables'],
 383                         $fileQuery['fields'],
 384                         [ 'img_sha1' => $hash ],
 385                         __METHOD__,
 386                         [ 'ORDER BY' => 'img_name' ],
 387                         $fileQuery['joins']
 388                 );
 389
 390                 $result = [];
 391                 foreach ( $res as $row ) {
 392                         $result[] = $this->newFileFromRow( $row );
 393                 }
 394                 $res->free();
 395
 396                 return $result;
 397         }
 398
 399         /**
 400          * Get an array of arrays or iterators of file objects for files that
 401          * have the given SHA-1 content hashes.
 402          *
 403          * Overrides generic implementation in FileRepo for performance reason
 404          *
 405          * @param string[] $hashes An array of hashes
 406          * @return array[] An Array of arrays or iterators of file objects and the hash as key
 407          */
 408         function findBySha1s( array $hashes ) {
 409                 if ( $hashes === [] ) {
 410                         return []; // empty parameter
 411                 }
 412
 413                 $dbr = $this->getReplicaDB();
 414                 $fileQuery = LocalFile::getQueryInfo();
 415                 $res = $dbr->select(
 416                         $fileQuery['tables'],
 417                         $fileQuery['fields'],
 418                         [ 'img_sha1' => $hashes ],
 419                         __METHOD__,
 420                         [ 'ORDER BY' => 'img_name' ],
 421                         $fileQuery['joins']
 422                 );
 423
 424                 $result = [];
 425                 foreach ( $res as $row ) {
 426                         $file = $this->newFileFromRow( $row );
 427                         $result[$file->getSha1()][] = $file;
 428                 }
 429                 $res->free();
 430
 431                 return $result;
 432         }
 433
 434         /**
 435          * Return an array of files where the name starts with $prefix.
 436          *
 437          * @param string $prefix The prefix to search for
 438          * @param int $limit The maximum amount of files to return
 439          * @return LocalFile[]
 440          */
 441         public function findFilesByPrefix( $prefix, $limit ) {
 442                 $selectOptions = [ 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ];
 443
 444                 // Query database
 445                 $dbr = $this->getReplicaDB();
 446                 $fileQuery = LocalFile::getQueryInfo();
 447                 $res = $dbr->select(
 448                         $fileQuery['tables'],
 449                         $fileQuery['fields'],
 450                         'img_name ' . $dbr->buildLike( $prefix, $dbr->anyString() ),
 451                         __METHOD__,
 452                         $selectOptions,
 453                         $fileQuery['joins']
 454                 );
 455
 456                 // Build file objects
 457                 $files = [];
 458                 foreach ( $res as $row ) {
 459                         $files[] = $this->newFileFromRow( $row );
 460                 }
 461
 462                 return $files;
 463         }
 464
 465         /**
 466          * Get a connection to the replica DB
 467          * @return IDatabase
 468          */
 469         function getReplicaDB() {
 470                 return wfGetDB( DB_REPLICA );
 471         }
 472
 473         /**
 474          * Alias for getReplicaDB()
 475          *
 476          * @return IDatabase
 477          * @deprecated Since 1.29
 478          */
 479         function getSlaveDB() {
 480                 return $this->getReplicaDB();
 481         }
 482
 483         /**
 484          * Get a connection to the master DB
 485          * @return IDatabase
 486          */
 487         function getMasterDB() {
 488                 return wfGetDB( DB_MASTER );
 489         }
 490
 491         /**
 492          * Get a callback to get a DB handle given an index (DB_REPLICA/DB_MASTER)
 493          * @return Closure
 494          */
 495         protected function getDBFactory() {
 496                 return function ( $index ) {
 497                         return wfGetDB( $index );
 498                 };
 499         }
 500
 501         /**
 502          * Get a key on the primary cache for this repository.
 503          * Returns false if the repository's cache is not accessible at this site.
 504          * The parameters are the parts of the key.
 505          *
 506          * @return string
 507          */
 508         function getSharedCacheKey( /*...*/ ) {
 509                 $args = func_get_args();
 510
 511                 return $this->wanCache->makeKey( ...$args );
 512         }
 513
 514         /**
 515          * Invalidates image redirect cache related to that image
 516          *
 517          * @param Title $title Title of page
 518          * @return void
 519          */
 520         function invalidateImageRedirect( Title $title ) {
 521                 $key = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 522                 if ( $key ) {
 523                         $this->getMasterDB()->onTransactionPreCommitOrIdle(
 524                                 function () use ( $key ) {
 525                                         $this->wanCache->delete( $key );
 526                                 },
 527                                 __METHOD__
 528                         );
 529                 }
 530         }
 531
 532         /**
 533          * Return information about the repository.
 534          *
 535          * @return array
 536          * @since 1.22
 537          */
 538         function getInfo() {
 539                 global $wgFavicon;
 540
 541                 return array_merge( parent::getInfo(), [
 542                         'favicon' => wfExpandUrl( $wgFavicon ),
 543                 ] );
 544         }
 545
 546         public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
 547                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 548         }
 549
 550         public function storeBatch( array $triplets, $flags = 0 ) {
 551                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 552         }
 553
 554         public function cleanupBatch( array $files, $flags = 0 ) {
 555                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 556         }
 557
 558         public function publish(
 559                 $src,
 560                 $dstRel,
 561                 $archiveRel,
 562                 $flags = 0,
 563                 array $options = []
 564         ) {
 565                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 566         }
 567
 568         public function publishBatch( array $ntuples, $flags = 0 ) {
 569                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 570         }
 571
 572         public function delete( $srcRel, $archiveRel ) {
 573                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 574         }
 575
 576         public function deleteBatch( array $sourceDestPairs ) {
 577                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 578         }
 579
 580         /**
 581          * Skips the write operation if storage is sha1-based, executes it normally otherwise
 582          *
 583          * @param string $function
 584          * @param array $args
 585          * @return Status
 586          */
 587         protected function skipWriteOperationIfSha1( $function, array $args ) {
 588                 $this->assertWritableRepo(); // fail out if read-only
 589
 590                 if ( $this->hasSha1Storage() ) {
 591                         wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
 592                         return Status::newGood();
 593                 } else {
 594                         return parent::$function( ...$args );
 595                 }
 596         }
 597 }