includes/filebackend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * Proxy backend that mirrors writes to several internal backends.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Proxy backend that mirrors writes to several internal backends.
  27  *
  28  * This class defines a multi-write backend. Multiple backends can be
  29  * registered to this proxy backend and it will act as a single backend.
  30  * Use this when all access to those backends is through this proxy backend.
  31  * At least one of the backends must be declared the "master" backend.
  32  *
  33  * Only use this class when transitioning from one storage system to another.
  34  *
  35  * Read operations are only done on the 'master' backend for consistency.
  36  * Write operations are performed on all backends, in the order defined.
  37  * If an operation fails on one backend it will be rolled back from the others.
  38  *
  39  * @ingroup FileBackend
  40  * @since 1.19
  41  */
  42 class FileBackendMultiWrite extends FileBackend {
  43         /** @var Array Prioritized list of FileBackendStore objects */
  44         protected $backends = array(); // array of (backend index => backends)
  45         protected $masterIndex = -1; // integer; index of master backend
  46         protected $syncChecks = 0; // integer bitfield
  47
  48         /* Possible internal backend consistency checks */
  49         const CHECK_SIZE = 1;
  50         const CHECK_TIME = 2;
  51         const CHECK_SHA1 = 4;
  52
  53         /**
  54          * Construct a proxy backend that consists of several internal backends.
  55          * Locking, journaling, and read-only checks are handled by the proxy backend.
  56          *
  57          * Additional $config params include:
  58          *   - backends   : Array of backend config and multi-backend settings.
  59          *                  Each value is the config used in the constructor of a
  60          *                  FileBackendStore class, but with these additional settings:
  61          *                    - class         : The name of the backend class
  62          *                    - isMultiMaster : This must be set for one backend.
  63          *                    - template:     : If given a backend name, this will use
  64          *                                      the config of that backend as a template.
  65          *                                      Values specified here take precedence.
  66          *   - syncChecks : Integer bitfield of internal backend sync checks to perform.
  67          *                  Possible bits include the FileBackendMultiWrite::CHECK_* constants.
  68          *                  There are constants for SIZE, TIME, and SHA1.
  69          *                  The checks are done before allowing any file operations.
  70          * @param $config Array
  71          * @throws MWException
  72          */
  73         public function __construct( array $config ) {
  74                 parent::__construct( $config );
  75                 $namesUsed = array();
  76                 // Construct backends here rather than via registration
  77                 // to keep these backends hidden from outside the proxy.
  78                 foreach ( $config['backends'] as $index => $config ) {
  79                         if ( isset( $config['template'] ) ) {
  80                                 // Config is just a modified version of a registered backend's.
  81                                 // This should only be used when that config is used only by this backend.
  82                                 $config = $config + FileBackendGroup::singleton()->config( $config['template'] );
  83                         }
  84                         $name = $config['name'];
  85                         if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
  86                                 throw new MWException( "Two or more backends defined with the name $name." );
  87                         }
  88                         $namesUsed[$name] = 1;
  89                         // Alter certain sub-backend settings for sanity
  90                         unset( $config['readOnly'] ); // use proxy backend setting
  91                         unset( $config['fileJournal'] ); // use proxy backend journal
  92                         $config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID
  93                         $config['lockManager'] = 'nullLockManager'; // lock under proxy backend
  94                         if ( !empty( $config['isMultiMaster'] ) ) {
  95                                 if ( $this->masterIndex >= 0 ) {
  96                                         throw new MWException( 'More than one master backend defined.' );
  97                                 }
  98                                 $this->masterIndex = $index; // this is the "master"
  99                                 $config['fileJournal'] = $this->fileJournal; // log under proxy backend
 100                         }
 101                         // Create sub-backend object
 102                         if ( !isset( $config['class'] ) ) {
 103                                 throw new MWException( 'No class given for a backend config.' );
 104                         }
 105                         $class = $config['class'];
 106                         $this->backends[$index] = new $class( $config );
 107                 }
 108                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
 109                         throw new MWException( 'No master backend defined.' );
 110                 }
 111                 $this->syncChecks = isset( $config['syncChecks'] )
 112                         ? $config['syncChecks']
 113                         : self::CHECK_SIZE;
 114         }
 115
 116         /**
 117          * @see FileBackend::doOperationsInternal()
 118          * @return Status
 119          */
 120         final protected function doOperationsInternal( array $ops, array $opts ) {
 121                 $status = Status::newGood();
 122
 123                 $mbe = $this->backends[$this->masterIndex]; // convenience
 124
 125                 // Get the paths to lock from the master backend
 126                 $realOps = $this->substOpBatchPaths( $ops, $mbe );
 127                 $paths = $mbe->getPathsToLockForOpsInternal( $mbe->getOperationsInternal( $realOps ) );
 128                 // Get the paths under the proxy backend's name
 129                 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
 130                 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
 131                 // Try to lock those files for the scope of this function...
 132                 if ( empty( $opts['nonLocking'] ) ) {
 133                         // Try to lock those files for the scope of this function...
 134                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 135                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 136                         if ( !$status->isOK() ) {
 137                                 return $status; // abort
 138                         }
 139                 }
 140                 // Clear any cache entries (after locks acquired)
 141                 $this->clearCache();
 142                 // Do a consistency check to see if the backends agree
 143                 $status->merge( $this->consistencyCheck( $this->fileStoragePathsForOps( $ops ) ) );
 144                 if ( !$status->isOK() ) {
 145                         return $status; // abort
 146                 }
 147                 // Actually attempt the operation batch on the master backend...
 148                 $masterStatus = $mbe->doOperations( $realOps, $opts );
 149                 $status->merge( $masterStatus );
 150                 // Propagate the operations to the clone backends...
 151                 foreach ( $this->backends as $index => $backend ) {
 152                         if ( $index !== $this->masterIndex ) { // not done already
 153                                 $realOps = $this->substOpBatchPaths( $ops, $backend );
 154                                 $status->merge( $backend->doOperations( $realOps, $opts ) );
 155                         }
 156                 }
 157                 // Make 'success', 'successCount', and 'failCount' fields reflect
 158                 // the overall operation, rather than all the batches for each backend.
 159                 // Do this by only using success values from the master backend's batch.
 160                 $status->success = $masterStatus->success;
 161                 $status->successCount = $masterStatus->successCount;
 162                 $status->failCount = $masterStatus->failCount;
 163
 164                 return $status;
 165         }
 166
 167         /**
 168          * Check that a set of files are consistent across all internal backends
 169          *
 170          * @param $paths Array
 171          * @return Status
 172          */
 173         public function consistencyCheck( array $paths ) {
 174                 $status = Status::newGood();
 175                 if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
 176                         return $status; // skip checks
 177                 }
 178
 179                 $mBackend = $this->backends[$this->masterIndex];
 180                 foreach ( array_unique( $paths ) as $path ) {
 181                         $params = array( 'src' => $path, 'latest' => true );
 182                         $mParams = $this->substOpPaths( $params, $mBackend );
 183                         // Stat the file on the 'master' backend
 184                         $mStat = $mBackend->getFileStat( $mParams );
 185                         if ( $this->syncChecks & self::CHECK_SHA1 ) {
 186                                 $mSha1 = $mBackend->getFileSha1( $mParams );
 187                         } else {
 188                                 $mSha1 = false;
 189                         }
 190                         $mUsable = $mBackend->isPathUsableInternal( $mParams['src'] );
 191                         // Check of all clone backends agree with the master...
 192                         foreach ( $this->backends as $index => $cBackend ) {
 193                                 if ( $index === $this->masterIndex ) {
 194                                         continue; // master
 195                                 }
 196                                 $cParams = $this->substOpPaths( $params, $cBackend );
 197                                 $cStat = $cBackend->getFileStat( $cParams );
 198                                 if ( $mStat ) { // file is in master
 199                                         if ( !$cStat ) { // file should exist
 200                                                 $status->fatal( 'backend-fail-synced', $path );
 201                                                 continue;
 202                                         }
 203                                         if ( $this->syncChecks & self::CHECK_SIZE ) {
 204                                                 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
 205                                                         $status->fatal( 'backend-fail-synced', $path );
 206                                                         continue;
 207                                                 }
 208                                         }
 209                                         if ( $this->syncChecks & self::CHECK_TIME ) {
 210                                                 $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
 211                                                 $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
 212                                                 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
 213                                                         $status->fatal( 'backend-fail-synced', $path );
 214                                                         continue;
 215                                                 }
 216                                         }
 217                                         if ( $this->syncChecks & self::CHECK_SHA1 ) {
 218                                                 if ( $cBackend->getFileSha1( $cParams ) !== $mSha1 ) { // wrong SHA1
 219                                                         $status->fatal( 'backend-fail-synced', $path );
 220                                                         continue;
 221                                                 }
 222                                         }
 223                                 } else { // file is not in master
 224                                         if ( $cStat ) { // file should not exist
 225                                                 $status->fatal( 'backend-fail-synced', $path );
 226                                         }
 227                                 }
 228                                 if ( $mUsable !== $cBackend->isPathUsableInternal( $cParams['src'] ) ) {
 229                                         $status->fatal( 'backend-fail-synced', $path );
 230                                 }
 231                         }
 232                 }
 233
 234                 return $status;
 235         }
 236
 237         /**
 238          * Get a list of file storage paths to read or write for a list of operations
 239          *
 240          * @param $ops Array Same format as doOperations()
 241          * @return Array List of storage paths to files (does not include directories)
 242          */
 243         protected function fileStoragePathsForOps( array $ops ) {
 244                 $paths = array();
 245                 foreach ( $ops as $op ) {
 246                         if ( isset( $op['src'] ) ) {
 247                                 $paths[] = $op['src'];
 248                         }
 249                         if ( isset( $op['srcs'] ) ) {
 250                                 $paths = array_merge( $paths, $op['srcs'] );
 251                         }
 252                         if ( isset( $op['dst'] ) ) {
 253                                 $paths[] = $op['dst'];
 254                         }
 255                 }
 256                 return array_unique( $paths );
 257         }
 258
 259         /**
 260          * Substitute the backend name in storage path parameters
 261          * for a set of operations with that of a given internal backend.
 262          *
 263          * @param $ops Array List of file operation arrays
 264          * @param $backend FileBackendStore
 265          * @return Array
 266          */
 267         protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
 268                 $newOps = array(); // operations
 269                 foreach ( $ops as $op ) {
 270                         $newOp = $op; // operation
 271                         foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
 272                                 if ( isset( $newOp[$par] ) ) { // string or array
 273                                         $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
 274                                 }
 275                         }
 276                         $newOps[] = $newOp;
 277                 }
 278                 return $newOps;
 279         }
 280
 281         /**
 282          * Same as substOpBatchPaths() but for a single operation
 283          *
 284          * @param $ops array File operation array
 285          * @param $backend FileBackendStore
 286          * @return Array
 287          */
 288         protected function substOpPaths( array $ops, FileBackendStore $backend ) {
 289                 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
 290                 return $newOps[0];
 291         }
 292
 293         /**
 294          * Substitute the backend of storage paths with an internal backend's name
 295          *
 296          * @param $paths Array|string List of paths or single string path
 297          * @param $backend FileBackendStore
 298          * @return Array|string
 299          */
 300         protected function substPaths( $paths, FileBackendStore $backend ) {
 301                 return preg_replace(
 302                         '!^mwstore://' . preg_quote( $this->name ) . '/!',
 303                         StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
 304                         $paths // string or array
 305                 );
 306         }
 307
 308         /**
 309          * Substitute the backend of internal storage paths with the proxy backend's name
 310          *
 311          * @param $paths Array|string List of paths or single string path
 312          * @return Array|string
 313          */
 314         protected function unsubstPaths( $paths ) {
 315                 return preg_replace(
 316                         '!^mwstore://([^/]+)!',
 317                         StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ),
 318                         $paths // string or array
 319                 );
 320         }
 321
 322         /**
 323          * @see FileBackend::doQuickOperationsInternal()
 324          * @return Status
 325          */
 326         protected function doQuickOperationsInternal( array $ops ) {
 327                 $status = Status::newGood();
 328                 // Do the operations on the master backend; setting Status fields...
 329                 $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
 330                 $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
 331                 $status->merge( $masterStatus );
 332                 // Propagate the operations to the clone backends...
 333                 foreach ( $this->backends as $index => $backend ) {
 334                         if ( $index !== $this->masterIndex ) { // not done already
 335                                 $realOps = $this->substOpBatchPaths( $ops, $backend );
 336                                 $status->merge( $backend->doQuickOperations( $realOps ) );
 337                         }
 338                 }
 339                 // Make 'success', 'successCount', and 'failCount' fields reflect
 340                 // the overall operation, rather than all the batches for each backend.
 341                 // Do this by only using success values from the master backend's batch.
 342                 $status->success = $masterStatus->success;
 343                 $status->successCount = $masterStatus->successCount;
 344                 $status->failCount = $masterStatus->failCount;
 345                 return $status;
 346         }
 347
 348         /**
 349          * @see FileBackend::doPrepare()
 350          * @return Status
 351          */
 352         protected function doPrepare( array $params ) {
 353                 $status = Status::newGood();
 354                 foreach ( $this->backends as $backend ) {
 355                         $realParams = $this->substOpPaths( $params, $backend );
 356                         $status->merge( $backend->doPrepare( $realParams ) );
 357                 }
 358                 return $status;
 359         }
 360
 361         /**
 362          * @see FileBackend::doSecure()
 363          * @param $params array
 364          * @return Status
 365          */
 366         protected function doSecure( array $params ) {
 367                 $status = Status::newGood();
 368                 foreach ( $this->backends as $backend ) {
 369                         $realParams = $this->substOpPaths( $params, $backend );
 370                         $status->merge( $backend->doSecure( $realParams ) );
 371                 }
 372                 return $status;
 373         }
 374
 375         /**
 376          * @see FileBackend::doPublish()
 377          * @param $params array
 378          * @return Status
 379          */
 380         protected function doPublish( array $params ) {
 381                 $status = Status::newGood();
 382                 foreach ( $this->backends as $backend ) {
 383                         $realParams = $this->substOpPaths( $params, $backend );
 384                         $status->merge( $backend->doPublish( $realParams ) );
 385                 }
 386                 return $status;
 387         }
 388
 389         /**
 390          * @see FileBackend::doClean()
 391          * @param $params array
 392          * @return Status
 393          */
 394         protected function doClean( array $params ) {
 395                 $status = Status::newGood();
 396                 foreach ( $this->backends as $backend ) {
 397                         $realParams = $this->substOpPaths( $params, $backend );
 398                         $status->merge( $backend->doClean( $realParams ) );
 399                 }
 400                 return $status;
 401         }
 402
 403         /**
 404          * @see FileBackend::concatenate()
 405          * @param $params array
 406          * @return Status
 407          */
 408         public function concatenate( array $params ) {
 409                 // We are writing to an FS file, so we don't need to do this per-backend
 410                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 411                 return $this->backends[$this->masterIndex]->concatenate( $realParams );
 412         }
 413
 414         /**
 415          * @see FileBackend::fileExists()
 416          * @param $params array
 417          */
 418         public function fileExists( array $params ) {
 419                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 420                 return $this->backends[$this->masterIndex]->fileExists( $realParams );
 421         }
 422
 423         /**
 424          * @see FileBackend::getFileTimestamp()
 425          * @param $params array
 426          * @return bool|string
 427          */
 428         public function getFileTimestamp( array $params ) {
 429                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 430                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 431         }
 432
 433         /**
 434          * @see FileBackend::getFileSize()
 435          * @param $params array
 436          * @return bool|int
 437          */
 438         public function getFileSize( array $params ) {
 439                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 440                 return $this->backends[$this->masterIndex]->getFileSize( $realParams );
 441         }
 442
 443         /**
 444          * @see FileBackend::getFileStat()
 445          * @param $params array
 446          * @return Array|bool|null
 447          */
 448         public function getFileStat( array $params ) {
 449                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 450                 return $this->backends[$this->masterIndex]->getFileStat( $realParams );
 451         }
 452
 453         /**
 454          * @see FileBackend::getFileContents()
 455          * @param $params array
 456          * @return bool|string
 457          */
 458         public function getFileContents( array $params ) {
 459                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 460                 return $this->backends[$this->masterIndex]->getFileContents( $realParams );
 461         }
 462
 463         /**
 464          * @see FileBackend::getFileSha1Base36()
 465          * @param $params array
 466          * @return bool|string
 467          */
 468         public function getFileSha1Base36( array $params ) {
 469                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 470                 return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
 471         }
 472
 473         /**
 474          * @see FileBackend::getFileProps()
 475          * @param $params array
 476          * @return Array
 477          */
 478         public function getFileProps( array $params ) {
 479                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 480                 return $this->backends[$this->masterIndex]->getFileProps( $realParams );
 481         }
 482
 483         /**
 484          * @see FileBackend::streamFile()
 485          * @param $params array
 486          * @return \Status
 487          */
 488         public function streamFile( array $params ) {
 489                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 490                 return $this->backends[$this->masterIndex]->streamFile( $realParams );
 491         }
 492
 493         /**
 494          * @see FileBackend::getLocalReference()
 495          * @param $params array
 496          * @return FSFile|null
 497          */
 498         public function getLocalReference( array $params ) {
 499                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 500                 return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
 501         }
 502
 503         /**
 504          * @see FileBackend::getLocalCopy()
 505          * @param $params array
 506          * @return null|TempFSFile
 507          */
 508         public function getLocalCopy( array $params ) {
 509                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 510                 return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
 511         }
 512
 513         /**
 514          * @see FileBackend::directoryExists()
 515          * @param $params array
 516          * @return bool|null
 517          */
 518         public function directoryExists( array $params ) {
 519                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 520                 return $this->backends[$this->masterIndex]->directoryExists( $realParams );
 521         }
 522
 523         /**
 524          * @see FileBackend::getSubdirectoryList()
 525          * @param $params array
 526          * @return Array|null|Traversable
 527          */
 528         public function getDirectoryList( array $params ) {
 529                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 530                 return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
 531         }
 532
 533         /**
 534          * @see FileBackend::getFileList()
 535          * @param $params array
 536          * @return Array|null|\Traversable
 537          */
 538         public function getFileList( array $params ) {
 539                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 540                 return $this->backends[$this->masterIndex]->getFileList( $realParams );
 541         }
 542
 543         /**
 544          * @see FileBackend::clearCache()
 545          */
 546         public function clearCache( array $paths = null ) {
 547                 foreach ( $this->backends as $backend ) {
 548                         $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
 549                         $backend->clearCache( $realPaths );
 550                 }
 551         }
 552
 553         /**
 554          * @see FileBackend::getScopedLocksForOps()
 555          */
 556         public function getScopedLocksForOps( array $ops, Status $status ) {
 557                 $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $ops );
 558                 // Get the paths to lock from the master backend
 559                 $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
 560                 // Get the paths under the proxy backend's name
 561                 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
 562                 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
 563                 return array(
 564                         $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
 565                         $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
 566                 );
 567         }
 568 }