includes/filerepo/LocalRepo.php

   1 <?php
   2 /**
   3  * Local repository that stores files in the local filesystem and registers them
   4  * in the wiki's own database.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19  * http://www.gnu.org/copyleft/gpl.html
  20  *
  21  * @file
  22  * @ingroup FileRepo
  23  */
  24
  25 use MediaWiki\MediaWikiServices;
  26 use Wikimedia\Rdbms\IResultWrapper;
  27 use Wikimedia\Rdbms\Database;
  28 use Wikimedia\Rdbms\IDatabase;
  29
  30 /**
  31  * A repository that stores files in the local filesystem and registers them
  32  * in the wiki's own database. This is the most commonly used repository class.
  33  *
  34  * @ingroup FileRepo
  35  */
  36 class LocalRepo extends FileRepo {
  37         /** @var callable */
  38         protected $fileFactory = [ LocalFile::class, 'newFromTitle' ];
  39         /** @var callable */
  40         protected $fileFactoryKey = [ LocalFile::class, 'newFromKey' ];
  41         /** @var callable */
  42         protected $fileFromRowFactory = [ LocalFile::class, 'newFromRow' ];
  43         /** @var callable */
  44         protected $oldFileFromRowFactory = [ OldLocalFile::class, 'newFromRow' ];
  45         /** @var callable */
  46         protected $oldFileFactory = [ OldLocalFile::class, 'newFromTitle' ];
  47         /** @var callable */
  48         protected $oldFileFactoryKey = [ OldLocalFile::class, 'newFromKey' ];
  49
  50         function __construct( array $info = null ) {
  51                 parent::__construct( $info );
  52
  53                 $this->hasSha1Storage = isset( $info['storageLayout'] )
  54                         && $info['storageLayout'] === 'sha1';
  55
  56                 if ( $this->hasSha1Storage() ) {
  57                         $this->backend = new FileBackendDBRepoWrapper( [
  58                                 'backend'         => $this->backend,
  59                                 'repoName'        => $this->name,
  60                                 'dbHandleFactory' => $this->getDBFactory()
  61                         ] );
  62                 }
  63         }
  64
  65         /**
  66          * @throws MWException
  67          * @param stdClass $row
  68          * @return LocalFile
  69          */
  70         function newFileFromRow( $row ) {
  71                 if ( isset( $row->img_name ) ) {
  72                         return call_user_func( $this->fileFromRowFactory, $row, $this );
  73                 } elseif ( isset( $row->oi_name ) ) {
  74                         return call_user_func( $this->oldFileFromRowFactory, $row, $this );
  75                 } else {
  76                         throw new MWException( __METHOD__ . ': invalid row' );
  77                 }
  78         }
  79
  80         /**
  81          * @param Title $title
  82          * @param string $archiveName
  83          * @return OldLocalFile
  84          */
  85         function newFromArchiveName( $title, $archiveName ) {
  86                 return OldLocalFile::newFromArchiveName( $title, $this, $archiveName );
  87         }
  88
  89         /**
  90          * Delete files in the deleted directory if they are not referenced in the
  91          * filearchive table. This needs to be done in the repo because it needs to
  92          * interleave database locks with file operations, which is potentially a
  93          * remote operation.
  94          *
  95          * @param string[] $storageKeys
  96          *
  97          * @return Status
  98          */
  99         function cleanupDeletedBatch( array $storageKeys ) {
 100                 if ( $this->hasSha1Storage() ) {
 101                         wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
 102                         return Status::newGood();
 103                 }
 104
 105                 $backend = $this->backend; // convenience
 106                 $root = $this->getZonePath( 'deleted' );
 107                 $dbw = $this->getMasterDB();
 108                 $status = $this->newGood();
 109                 $storageKeys = array_unique( $storageKeys );
 110                 foreach ( $storageKeys as $key ) {
 111                         $hashPath = $this->getDeletedHashPath( $key );
 112                         $path = "$root/$hashPath$key";
 113                         $dbw->startAtomic( __METHOD__ );
 114                         // Check for usage in deleted/hidden files and preemptively
 115                         // lock the key to avoid any future use until we are finished.
 116                         $deleted = $this->deletedFileHasKey( $key, 'lock' );
 117                         $hidden = $this->hiddenFileHasKey( $key, 'lock' );
 118                         if ( !$deleted && !$hidden ) { // not in use now
 119                                 wfDebug( __METHOD__ . ": deleting $key\n" );
 120                                 $op = [ 'op' => 'delete', 'src' => $path ];
 121                                 if ( !$backend->doOperation( $op )->isOK() ) {
 122                                         $status->error( 'undelete-cleanup-error', $path );
 123                                         $status->failCount++;
 124                                 }
 125                         } else {
 126                                 wfDebug( __METHOD__ . ": $key still in use\n" );
 127                                 $status->successCount++;
 128                         }
 129                         $dbw->endAtomic( __METHOD__ );
 130                 }
 131
 132                 return $status;
 133         }
 134
 135         /**
 136          * Check if a deleted (filearchive) file has this sha1 key
 137          *
 138          * @param string $key File storage key (base-36 sha1 key with file extension)
 139          * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
 140          * @return bool File with this key is in use
 141          */
 142         protected function deletedFileHasKey( $key, $lock = null ) {
 143                 $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
 144
 145                 $dbw = $this->getMasterDB();
 146
 147                 return (bool)$dbw->selectField( 'filearchive', '1',
 148                         [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ],
 149                         __METHOD__, $options
 150                 );
 151         }
 152
 153         /**
 154          * Check if a hidden (revision delete) file has this sha1 key
 155          *
 156          * @param string $key File storage key (base-36 sha1 key with file extension)
 157          * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
 158          * @return bool File with this key is in use
 159          */
 160         protected function hiddenFileHasKey( $key, $lock = null ) {
 161                 $options = ( $lock === 'lock' ) ? [ 'FOR UPDATE' ] : [];
 162
 163                 $sha1 = self::getHashFromKey( $key );
 164                 $ext = File::normalizeExtension( substr( $key, strcspn( $key, '.' ) + 1 ) );
 165
 166                 $dbw = $this->getMasterDB();
 167
 168                 return (bool)$dbw->selectField( 'oldimage', '1',
 169                         [ 'oi_sha1' => $sha1,
 170                                 'oi_archive_name ' . $dbw->buildLike( $dbw->anyString(), ".$ext" ),
 171                                 $dbw->bitAnd( 'oi_deleted', File::DELETED_FILE ) => File::DELETED_FILE ],
 172                         __METHOD__, $options
 173                 );
 174         }
 175
 176         /**
 177          * Gets the SHA1 hash from a storage key
 178          *
 179          * @param string $key
 180          * @return string
 181          */
 182         public static function getHashFromKey( $key ) {
 183                 return strtok( $key, '.' );
 184         }
 185
 186         /**
 187          * Checks if there is a redirect named as $title
 188          *
 189          * @param Title $title Title of file
 190          * @return bool|Title
 191          */
 192         function checkRedirect( Title $title ) {
 193                 $title = File::normalizeTitle( $title, 'exception' );
 194
 195                 $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 196                 if ( $memcKey === false ) {
 197                         $memcKey = $this->getLocalCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 198                         $expiry = 300; // no invalidation, 5 minutes
 199                 } else {
 200                         $expiry = 86400; // has invalidation, 1 day
 201                 }
 202
 203                 $method = __METHOD__;
 204                 $redirDbKey = $this->wanCache->getWithSetCallback(
 205                         $memcKey,
 206                         $expiry,
 207                         function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) {
 208                                 $dbr = $this->getReplicaDB(); // possibly remote DB
 209
 210                                 $setOpts += Database::getCacheSetOptions( $dbr );
 211
 212                                 $row = $dbr->selectRow(
 213                                         [ 'page', 'redirect' ],
 214                                         [ 'rd_namespace', 'rd_title' ],
 215                                         [
 216                                                 'page_namespace' => $title->getNamespace(),
 217                                                 'page_title' => $title->getDBkey(),
 218                                                 'rd_from = page_id'
 219                                         ],
 220                                         $method
 221                                 );
 222
 223                                 return ( $row && $row->rd_namespace == NS_FILE )
 224                                         ? Title::makeTitle( $row->rd_namespace, $row->rd_title )->getDBkey()
 225                                         : ''; // negative cache
 226                         },
 227                         [ 'pcTTL' => WANObjectCache::TTL_PROC_LONG ]
 228                 );
 229
 230                 // @note: also checks " " for b/c
 231                 if ( $redirDbKey !== ' ' && strval( $redirDbKey ) !== '' ) {
 232                         // Page is a redirect to another file
 233                         return Title::newFromText( $redirDbKey, NS_FILE );
 234                 }
 235
 236                 return false; // no redirect
 237         }
 238
 239         public function findFiles( array $items, $flags = 0 ) {
 240                 $finalFiles = []; // map of (DB key => corresponding File) for matches
 241
 242                 $searchSet = []; // map of (normalized DB key => search params)
 243                 foreach ( $items as $item ) {
 244                         if ( is_array( $item ) ) {
 245                                 $title = File::normalizeTitle( $item['title'] );
 246                                 if ( $title ) {
 247                                         $searchSet[$title->getDBkey()] = $item;
 248                                 }
 249                         } else {
 250                                 $title = File::normalizeTitle( $item );
 251                                 if ( $title ) {
 252                                         $searchSet[$title->getDBkey()] = [];
 253                                 }
 254                         }
 255                 }
 256
 257                 $fileMatchesSearch = function ( File $file, array $search ) {
 258                         // Note: file name comparison done elsewhere (to handle redirects)
 259                         $user = ( !empty( $search['private'] ) && $search['private'] instanceof User )
 260                                 ? $search['private']
 261                                 : null;
 262
 263                         return (
 264                                 $file->exists() &&
 265                                 (
 266                                         ( empty( $search['time'] ) && !$file->isOld() ) ||
 267                                         ( !empty( $search['time'] ) && $search['time'] === $file->getTimestamp() )
 268                                 ) &&
 269                                 ( !empty( $search['private'] ) || !$file->isDeleted( File::DELETED_FILE ) ) &&
 270                                 $file->userCan( File::DELETED_FILE, $user )
 271                         );
 272                 };
 273
 274                 $applyMatchingFiles = function ( IResultWrapper $res, &$searchSet, &$finalFiles )
 275                         use ( $fileMatchesSearch, $flags )
 276                 {
 277                         $contLang = MediaWikiServices::getInstance()->getContentLanguage();
 278                         $info = $this->getInfo();
 279                         foreach ( $res as $row ) {
 280                                 $file = $this->newFileFromRow( $row );
 281                                 // There must have been a search for this DB key, but this has to handle the
 282                                 // cases were title capitalization is different on the client and repo wikis.
 283                                 $dbKeysLook = [ strtr( $file->getName(), ' ', '_' ) ];
 284                                 if ( !empty( $info['initialCapital'] ) ) {
 285                                         // Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
 286                                         $dbKeysLook[] = $contLang->lcfirst( $file->getName() );
 287                                 }
 288                                 foreach ( $dbKeysLook as $dbKey ) {
 289                                         if ( isset( $searchSet[$dbKey] )
 290                                                 && $fileMatchesSearch( $file, $searchSet[$dbKey] )
 291                                         ) {
 292                                                 $finalFiles[$dbKey] = ( $flags & FileRepo::NAME_AND_TIME_ONLY )
 293                                                         ? [ 'title' => $dbKey, 'timestamp' => $file->getTimestamp() ]
 294                                                         : $file;
 295                                                 unset( $searchSet[$dbKey] );
 296                                         }
 297                                 }
 298                         }
 299                 };
 300
 301                 $dbr = $this->getReplicaDB();
 302
 303                 // Query image table
 304                 $imgNames = [];
 305                 foreach ( array_keys( $searchSet ) as $dbKey ) {
 306                         $imgNames[] = $this->getNameFromTitle( File::normalizeTitle( $dbKey ) );
 307                 }
 308
 309                 if ( count( $imgNames ) ) {
 310                         $fileQuery = LocalFile::getQueryInfo();
 311                         $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'], [ 'img_name' => $imgNames ],
 312                                 __METHOD__, [], $fileQuery['joins'] );
 313                         $applyMatchingFiles( $res, $searchSet, $finalFiles );
 314                 }
 315
 316                 // Query old image table
 317                 $oiConds = []; // WHERE clause array for each file
 318                 foreach ( $searchSet as $dbKey => $search ) {
 319                         if ( isset( $search['time'] ) ) {
 320                                 $oiConds[] = $dbr->makeList(
 321                                         [
 322                                                 'oi_name' => $this->getNameFromTitle( File::normalizeTitle( $dbKey ) ),
 323                                                 'oi_timestamp' => $dbr->timestamp( $search['time'] )
 324                                         ],
 325                                         LIST_AND
 326                                 );
 327                         }
 328                 }
 329
 330                 if ( count( $oiConds ) ) {
 331                         $fileQuery = OldLocalFile::getQueryInfo();
 332                         $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'],
 333                                 $dbr->makeList( $oiConds, LIST_OR ),
 334                                 __METHOD__, [], $fileQuery['joins'] );
 335                         $applyMatchingFiles( $res, $searchSet, $finalFiles );
 336                 }
 337
 338                 // Check for redirects...
 339                 foreach ( $searchSet as $dbKey => $search ) {
 340                         if ( !empty( $search['ignoreRedirect'] ) ) {
 341                                 continue;
 342                         }
 343
 344                         $title = File::normalizeTitle( $dbKey );
 345                         $redir = $this->checkRedirect( $title ); // hopefully hits memcached
 346
 347                         if ( $redir && $redir->getNamespace() == NS_FILE ) {
 348                                 $file = $this->newFile( $redir );
 349                                 if ( $file && $fileMatchesSearch( $file, $search ) ) {
 350                                         $file->redirectedFrom( $title->getDBkey() );
 351                                         if ( $flags & FileRepo::NAME_AND_TIME_ONLY ) {
 352                                                 $finalFiles[$dbKey] = [
 353                                                         'title' => $file->getTitle()->getDBkey(),
 354                                                         'timestamp' => $file->getTimestamp()
 355                                                 ];
 356                                         } else {
 357                                                 $finalFiles[$dbKey] = $file;
 358                                         }
 359                                 }
 360                         }
 361                 }
 362
 363                 return $finalFiles;
 364         }
 365
 366         /**
 367          * Get an array or iterator of file objects for files that have a given
 368          * SHA-1 content hash.
 369          *
 370          * @param string $hash A sha1 hash to look for
 371          * @return LocalFile[]
 372          */
 373         function findBySha1( $hash ) {
 374                 $dbr = $this->getReplicaDB();
 375                 $fileQuery = LocalFile::getQueryInfo();
 376                 $res = $dbr->select(
 377                         $fileQuery['tables'],
 378                         $fileQuery['fields'],
 379                         [ 'img_sha1' => $hash ],
 380                         __METHOD__,
 381                         [ 'ORDER BY' => 'img_name' ],
 382                         $fileQuery['joins']
 383                 );
 384
 385                 $result = [];
 386                 foreach ( $res as $row ) {
 387                         $result[] = $this->newFileFromRow( $row );
 388                 }
 389                 $res->free();
 390
 391                 return $result;
 392         }
 393
 394         /**
 395          * Get an array of arrays or iterators of file objects for files that
 396          * have the given SHA-1 content hashes.
 397          *
 398          * Overrides generic implementation in FileRepo for performance reason
 399          *
 400          * @param string[] $hashes An array of hashes
 401          * @return array[] An Array of arrays or iterators of file objects and the hash as key
 402          */
 403         function findBySha1s( array $hashes ) {
 404                 if ( $hashes === [] ) {
 405                         return []; // empty parameter
 406                 }
 407
 408                 $dbr = $this->getReplicaDB();
 409                 $fileQuery = LocalFile::getQueryInfo();
 410                 $res = $dbr->select(
 411                         $fileQuery['tables'],
 412                         $fileQuery['fields'],
 413                         [ 'img_sha1' => $hashes ],
 414                         __METHOD__,
 415                         [ 'ORDER BY' => 'img_name' ],
 416                         $fileQuery['joins']
 417                 );
 418
 419                 $result = [];
 420                 foreach ( $res as $row ) {
 421                         $file = $this->newFileFromRow( $row );
 422                         $result[$file->getSha1()][] = $file;
 423                 }
 424                 $res->free();
 425
 426                 return $result;
 427         }
 428
 429         /**
 430          * Return an array of files where the name starts with $prefix.
 431          *
 432          * @param string $prefix The prefix to search for
 433          * @param int $limit The maximum amount of files to return
 434          * @return LocalFile[]
 435          */
 436         public function findFilesByPrefix( $prefix, $limit ) {
 437                 $selectOptions = [ 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ];
 438
 439                 // Query database
 440                 $dbr = $this->getReplicaDB();
 441                 $fileQuery = LocalFile::getQueryInfo();
 442                 $res = $dbr->select(
 443                         $fileQuery['tables'],
 444                         $fileQuery['fields'],
 445                         'img_name ' . $dbr->buildLike( $prefix, $dbr->anyString() ),
 446                         __METHOD__,
 447                         $selectOptions,
 448                         $fileQuery['joins']
 449                 );
 450
 451                 // Build file objects
 452                 $files = [];
 453                 foreach ( $res as $row ) {
 454                         $files[] = $this->newFileFromRow( $row );
 455                 }
 456
 457                 return $files;
 458         }
 459
 460         /**
 461          * Get a connection to the replica DB
 462          * @return IDatabase
 463          */
 464         function getReplicaDB() {
 465                 return wfGetDB( DB_REPLICA );
 466         }
 467
 468         /**
 469          * Alias for getReplicaDB()
 470          *
 471          * @return IDatabase
 472          * @deprecated Since 1.29
 473          */
 474         function getSlaveDB() {
 475                 return $this->getReplicaDB();
 476         }
 477
 478         /**
 479          * Get a connection to the master DB
 480          * @return IDatabase
 481          */
 482         function getMasterDB() {
 483                 return wfGetDB( DB_MASTER );
 484         }
 485
 486         /**
 487          * Get a callback to get a DB handle given an index (DB_REPLICA/DB_MASTER)
 488          * @return Closure
 489          */
 490         protected function getDBFactory() {
 491                 return function ( $index ) {
 492                         return wfGetDB( $index );
 493                 };
 494         }
 495
 496         /**
 497          * Get a key on the primary cache for this repository.
 498          * Returns false if the repository's cache is not accessible at this site.
 499          * The parameters are the parts of the key.
 500          *
 501          * @return string
 502          */
 503         function getSharedCacheKey( /*...*/ ) {
 504                 $args = func_get_args();
 505
 506                 return $this->wanCache->makeKey( ...$args );
 507         }
 508
 509         /**
 510          * Invalidates image redirect cache related to that image
 511          *
 512          * @param Title $title Title of page
 513          * @return void
 514          */
 515         function invalidateImageRedirect( Title $title ) {
 516                 $key = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
 517                 if ( $key ) {
 518                         $this->getMasterDB()->onTransactionPreCommitOrIdle(
 519                                 function () use ( $key ) {
 520                                         $this->wanCache->delete( $key );
 521                                 },
 522                                 __METHOD__
 523                         );
 524                 }
 525         }
 526
 527         /**
 528          * Return information about the repository.
 529          *
 530          * @return array
 531          * @since 1.22
 532          */
 533         function getInfo() {
 534                 global $wgFavicon;
 535
 536                 return array_merge( parent::getInfo(), [
 537                         'favicon' => wfExpandUrl( $wgFavicon ),
 538                 ] );
 539         }
 540
 541         public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
 542                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 543         }
 544
 545         public function storeBatch( array $triplets, $flags = 0 ) {
 546                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 547         }
 548
 549         public function cleanupBatch( array $files, $flags = 0 ) {
 550                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 551         }
 552
 553         public function publish(
 554                 $src,
 555                 $dstRel,
 556                 $archiveRel,
 557                 $flags = 0,
 558                 array $options = []
 559         ) {
 560                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 561         }
 562
 563         public function publishBatch( array $ntuples, $flags = 0 ) {
 564                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 565         }
 566
 567         public function delete( $srcRel, $archiveRel ) {
 568                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 569         }
 570
 571         public function deleteBatch( array $sourceDestPairs ) {
 572                 return $this->skipWriteOperationIfSha1( __FUNCTION__, func_get_args() );
 573         }
 574
 575         /**
 576          * Skips the write operation if storage is sha1-based, executes it normally otherwise
 577          *
 578          * @param string $function
 579          * @param array $args
 580          * @return Status
 581          */
 582         protected function skipWriteOperationIfSha1( $function, array $args ) {
 583                 $this->assertWritableRepo(); // fail out if read-only
 584
 585                 if ( $this->hasSha1Storage() ) {
 586                         wfDebug( __METHOD__ . ": skipped because storage uses sha1 paths\n" );
 587                         return Status::newGood();
 588                 } else {
 589                         return parent::$function( ...$args );
 590                 }
 591         }
 592 }