includes/filerepo/backend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * This class defines a multi-write backend. Multiple backends can be
  10  * registered to this proxy backend and it will act as a single backend.
  11  * Use this when all access to those backends is through this proxy backend.
  12  * At least one of the backends must be declared the "master" backend.
  13  *
  14  * Only use this class when transitioning from one storage system to another.
  15  *
  16  * Read operations are only done on the 'master' backend for consistency.
  17  * Write operations are performed on all backends, in the order defined.
  18  * If an operation fails on one backend it will be rolled back from the others.
  19  *
  20  * @ingroup FileBackend
  21  * @since 1.19
  22  */
  23 class FileBackendMultiWrite extends FileBackend {
  24         /** @var Array Prioritized list of FileBackendStore objects */
  25         protected $backends = array(); // array of (backend index => backends)
  26         protected $masterIndex = -1; // integer; index of master backend
  27         protected $syncChecks = 0; // integer bitfield
  28
  29         /* Possible internal backend consistency checks */
  30         const CHECK_SIZE = 1;
  31         const CHECK_TIME = 2;
  32
  33         /**
  34          * Construct a proxy backend that consists of several internal backends.
  35          * Additional $config params include:
  36          *     'backends'    : Array of backend config and multi-backend settings.
  37          *                     Each value is the config used in the constructor of a
  38          *                     FileBackendStore class, but with these additional settings:
  39          *                         'class'         : The name of the backend class
  40          *                         'isMultiMaster' : This must be set for one backend.
  41          *     'syncChecks'  : Integer bitfield of internal backend sync checks to perform.
  42          *                     Possible bits include self::CHECK_SIZE and self::CHECK_TIME.
  43          *                     The checks are done before allowing any file operations.
  44          * @param $config Array
  45          */
  46         public function __construct( array $config ) {
  47                 parent::__construct( $config );
  48                 $namesUsed = array();
  49                 // Construct backends here rather than via registration
  50                 // to keep these backends hidden from outside the proxy.
  51                 foreach ( $config['backends'] as $index => $config ) {
  52                         $name = $config['name'];
  53                         if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
  54                                 throw new MWException( "Two or more backends defined with the name $name." );
  55                         }
  56                         $namesUsed[$name] = 1;
  57                         if ( !isset( $config['class'] ) ) {
  58                                 throw new MWException( 'No class given for a backend config.' );
  59                         }
  60                         $class = $config['class'];
  61                         $this->backends[$index] = new $class( $config );
  62                         if ( !empty( $config['isMultiMaster'] ) ) {
  63                                 if ( $this->masterIndex >= 0 ) {
  64                                         throw new MWException( 'More than one master backend defined.' );
  65                                 }
  66                                 $this->masterIndex = $index;
  67                         }
  68                 }
  69                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
  70                         throw new MWException( 'No master backend defined.' );
  71                 }
  72                 $this->syncChecks = isset( $config['syncChecks'] )
  73                         ? $config['syncChecks']
  74                         : self::CHECK_SIZE;
  75         }
  76
  77         /**
  78          * @see FileBackend::doOperationsInternal()
  79          * @return Status
  80          */
  81         final protected function doOperationsInternal( array $ops, array $opts ) {
  82                 $status = Status::newGood();
  83
  84                 $performOps = array(); // list of FileOp objects
  85                 $filesRead = $filesChanged = array(); // storage paths used
  86                 // Build up a list of FileOps. The list will have all the ops
  87                 // for one backend, then all the ops for the next, and so on.
  88                 // These batches of ops are all part of a continuous array.
  89                 // Also build up a list of files read/changed...
  90                 foreach ( $this->backends as $index => $backend ) {
  91                         $backendOps = $this->substOpBatchPaths( $ops, $backend );
  92                         // Add on the operation batch for this backend
  93                         $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
  94                         if ( $index == 0 ) { // first batch
  95                                 // Get the files used for these operations. Each backend has a batch of
  96                                 // the same operations, so we only need to get them from the first batch.
  97                                 foreach ( $performOps as $fileOp ) {
  98                                         $filesRead = array_merge( $filesRead, $fileOp->storagePathsRead() );
  99                                         $filesChanged = array_merge( $filesChanged, $fileOp->storagePathsChanged() );
 100                                 }
 101                                 // Get the paths under the proxy backend's name
 102                                 $filesRead = $this->unsubstPaths( $filesRead );
 103                                 $filesChanged = $this->unsubstPaths( $filesChanged );
 104                         }
 105                 }
 106
 107                 // Try to lock those files for the scope of this function...
 108                 if ( empty( $opts['nonLocking'] ) ) {
 109                         $filesLockSh = array_diff( $filesRead, $filesChanged ); // optimization
 110                         $filesLockEx = $filesChanged;
 111                         // Get a shared lock on the parent directory of each path changed
 112                         $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
 113                         // Try to lock those files for the scope of this function...
 114                         $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
 115                         $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 116                         if ( !$status->isOK() ) {
 117                                 return $status; // abort
 118                         }
 119                 }
 120
 121                 // Clear any cache entries (after locks acquired)
 122                 $this->clearCache();
 123
 124                 // Do a consistency check to see if the backends agree
 125                 if ( count( $this->backends ) > 1 ) {
 126                         $status->merge( $this->consistencyCheck( array_merge( $filesRead, $filesChanged ) ) );
 127                         if ( !$status->isOK() ) {
 128                                 return $status; // abort
 129                         }
 130                 }
 131
 132                 // Actually attempt the operation batch...
 133                 $subStatus = FileOp::attemptBatch( $performOps, $opts );
 134
 135                 $success = array();
 136                 $failCount = $successCount = 0;
 137                 // Make 'success', 'successCount', and 'failCount' fields reflect
 138                 // the overall operation, rather than all the batches for each backend.
 139                 // Do this by only using success values from the master backend's batch.
 140                 $batchStart = $this->masterIndex * count( $ops );
 141                 $batchEnd = $batchStart + count( $ops ) - 1;
 142                 for ( $i = $batchStart; $i <= $batchEnd; $i++ ) {
 143                         if ( !isset( $subStatus->success[$i] ) ) {
 144                                 break; // failed out before trying this op
 145                         } elseif ( $subStatus->success[$i] ) {
 146                                 ++$successCount;
 147                         } else {
 148                                 ++$failCount;
 149                         }
 150                         $success[] = $subStatus->success[$i];
 151                 }
 152                 $subStatus->success = $success;
 153                 $subStatus->successCount = $successCount;
 154                 $subStatus->failCount = $failCount;
 155
 156                 // Merge errors into status fields
 157                 $status->merge( $subStatus );
 158                 $status->success = $subStatus->success; // not done in merge()
 159
 160                 return $status;
 161         }
 162
 163         /**
 164          * Check that a set of files are consistent across all internal backends
 165          *
 166          * @param $paths Array
 167          * @return Status
 168          */
 169         public function consistencyCheck( array $paths ) {
 170                 $status = Status::newGood();
 171                 if ( $this->syncChecks == 0 ) {
 172                         return $status; // skip checks
 173                 }
 174
 175                 $mBackend = $this->backends[$this->masterIndex];
 176                 foreach ( array_unique( $paths ) as $path ) {
 177                         $params = array( 'src' => $path, 'latest' => true );
 178                         // Stat the file on the 'master' backend
 179                         $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
 180                         // Check of all clone backends agree with the master...
 181                         foreach ( $this->backends as $index => $cBackend ) {
 182                                 if ( $index === $this->masterIndex ) {
 183                                         continue; // master
 184                                 }
 185                                 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
 186                                 if ( $mStat ) { // file is in master
 187                                         if ( !$cStat ) { // file should exist
 188                                                 $status->fatal( 'backend-fail-synced', $path );
 189                                                 continue;
 190                                         }
 191                                         if ( $this->syncChecks & self::CHECK_SIZE ) {
 192                                                 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
 193                                                         $status->fatal( 'backend-fail-synced', $path );
 194                                                         continue;
 195                                                 }
 196                                         }
 197                                         if ( $this->syncChecks & self::CHECK_TIME ) {
 198                                                 $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
 199                                                 $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
 200                                                 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
 201                                                         $status->fatal( 'backend-fail-synced', $path );
 202                                                         continue;
 203                                                 }
 204                                         }
 205                                 } else { // file is not in master
 206                                         if ( $cStat ) { // file should not exist
 207                                                 $status->fatal( 'backend-fail-synced', $path );
 208                                         }
 209                                 }
 210                         }
 211                 }
 212
 213                 return $status;
 214         }
 215
 216         /**
 217          * Substitute the backend name in storage path parameters
 218          * for a set of operations with that of a given internal backend.
 219          *
 220          * @param $ops Array List of file operation arrays
 221          * @param $backend FileBackendStore
 222          * @return Array
 223          */
 224         protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
 225                 $newOps = array(); // operations
 226                 foreach ( $ops as $op ) {
 227                         $newOp = $op; // operation
 228                         foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
 229                                 if ( isset( $newOp[$par] ) ) { // string or array
 230                                         $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
 231                                 }
 232                         }
 233                         $newOps[] = $newOp;
 234                 }
 235                 return $newOps;
 236         }
 237
 238         /**
 239          * Same as substOpBatchPaths() but for a single operation
 240          *
 241          * @param $op File operation array
 242          * @param $backend FileBackendStore
 243          * @return Array
 244          */
 245         protected function substOpPaths( array $ops, FileBackendStore $backend ) {
 246                 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
 247                 return $newOps[0];
 248         }
 249
 250         /**
 251          * Substitute the backend of storage paths with an internal backend's name
 252          *
 253          * @param $paths Array|string List of paths or single string path
 254          * @param $backend FileBackendStore
 255          * @return Array|string
 256          */
 257         protected function substPaths( $paths, FileBackendStore $backend ) {
 258                 return preg_replace(
 259                         '!^mwstore://' . preg_quote( $this->name ) . '/!',
 260                         StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
 261                         $paths // string or array
 262                 );
 263         }
 264
 265         /**
 266          * Substitute the backend of internal storage paths with the proxy backend's name
 267          *
 268          * @param $paths Array|string List of paths or single string path
 269          * @return Array|string
 270          */
 271         protected function unsubstPaths( $paths ) {
 272                 return preg_replace(
 273                         '!^mwstore://([^/]+)!',
 274                         StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ),
 275                         $paths // string or array
 276                 );
 277         }
 278
 279         /**
 280          * @see FileBackend::doPrepare()
 281          * @return Status
 282          */
 283         public function doPrepare( array $params ) {
 284                 $status = Status::newGood();
 285                 foreach ( $this->backends as $backend ) {
 286                         $realParams = $this->substOpPaths( $params, $backend );
 287                         $status->merge( $backend->doPrepare( $realParams ) );
 288                 }
 289                 return $status;
 290         }
 291
 292         /**
 293          * @see FileBackend::doSecure()
 294          * @return Status
 295          */
 296         public function doSecure( array $params ) {
 297                 $status = Status::newGood();
 298                 foreach ( $this->backends as $backend ) {
 299                         $realParams = $this->substOpPaths( $params, $backend );
 300                         $status->merge( $backend->doSecure( $realParams ) );
 301                 }
 302                 return $status;
 303         }
 304
 305         /**
 306          * @see FileBackend::doClean()
 307          * @return Status
 308          */
 309         public function doClean( array $params ) {
 310                 $status = Status::newGood();
 311                 foreach ( $this->backends as $backend ) {
 312                         $realParams = $this->substOpPaths( $params, $backend );
 313                         $status->merge( $backend->doClean( $realParams ) );
 314                 }
 315                 return $status;
 316         }
 317
 318         /**
 319          * @see FileBackend::getFileList()
 320          */
 321         public function concatenate( array $params ) {
 322                 // We are writing to an FS file, so we don't need to do this per-backend
 323                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 324                 return $this->backends[$this->masterIndex]->concatenate( $realParams );
 325         }
 326
 327         /**
 328          * @see FileBackend::fileExists()
 329          */
 330         public function fileExists( array $params ) {
 331                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 332                 return $this->backends[$this->masterIndex]->fileExists( $realParams );
 333         }
 334
 335         /**
 336          * @see FileBackend::getFileTimestamp()
 337          */
 338         public function getFileTimestamp( array $params ) {
 339                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 340                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 341         }
 342
 343         /**
 344          * @see FileBackend::getFileSize()
 345          */
 346         public function getFileSize( array $params ) {
 347                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 348                 return $this->backends[$this->masterIndex]->getFileSize( $realParams );
 349         }
 350
 351         /**
 352          * @see FileBackend::getFileStat()
 353          */
 354         public function getFileStat( array $params ) {
 355                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 356                 return $this->backends[$this->masterIndex]->getFileStat( $realParams );
 357         }
 358
 359         /**
 360          * @see FileBackend::getFileContents()
 361          */
 362         public function getFileContents( array $params ) {
 363                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 364                 return $this->backends[$this->masterIndex]->getFileContents( $realParams );
 365         }
 366
 367         /**
 368          * @see FileBackend::getFileSha1Base36()
 369          */
 370         public function getFileSha1Base36( array $params ) {
 371                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 372                 return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
 373         }
 374
 375         /**
 376          * @see FileBackend::getFileProps()
 377          */
 378         public function getFileProps( array $params ) {
 379                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 380                 return $this->backends[$this->masterIndex]->getFileProps( $realParams );
 381         }
 382
 383         /**
 384          * @see FileBackend::streamFile()
 385          */
 386         public function streamFile( array $params ) {
 387                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 388                 return $this->backends[$this->masterIndex]->streamFile( $realParams );
 389         }
 390
 391         /**
 392          * @see FileBackend::getLocalReference()
 393          */
 394         public function getLocalReference( array $params ) {
 395                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 396                 return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
 397         }
 398
 399         /**
 400          * @see FileBackend::getLocalCopy()
 401          */
 402         public function getLocalCopy( array $params ) {
 403                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 404                 return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
 405         }
 406
 407         /**
 408          * @see FileBackend::getFileList()
 409          */
 410         public function getFileList( array $params ) {
 411                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 412                 return $this->backends[$this->masterIndex]->getFileList( $realParams );
 413         }
 414
 415         /**
 416          * @see FileBackend::clearCache()
 417          */
 418         public function clearCache( array $paths = null ) {
 419                 foreach ( $this->backends as $backend ) {
 420                         $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
 421                         $backend->clearCache( $realPaths );
 422                 }
 423         }
 424 }