includes/filerepo/backend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * This class defines a multi-write backend. Multiple backends can be
  10  * registered to this proxy backend and it will act as a single backend.
  11  * Use this when all access to those backends is through this proxy backend.
  12  * At least one of the backends must be declared the "master" backend.
  13  *
  14  * Only use this class when transitioning from one storage system to another.
  15  *
  16  * Read operations are only done on the 'master' backend for consistency.
  17  * Write operations are performed on all backends, in the order defined.
  18  * If an operation fails on one backend it will be rolled back from the others.
  19  *
  20  * @ingroup FileBackend
  21  * @since 1.19
  22  */
  23 class FileBackendMultiWrite extends FileBackend {
  24         /** @var Array Prioritized list of FileBackendStore objects */
  25         protected $backends = array(); // array of (backend index => backends)
  26         protected $masterIndex = -1; // integer; index of master backend
  27         protected $syncChecks = 0; // integer bitfield
  28
  29         /* Possible internal backend consistency checks */
  30         const CHECK_SIZE = 1;
  31         const CHECK_TIME = 2;
  32
  33         /**
  34          * Construct a proxy backend that consists of several internal backends.
  35          * Additional $config params include:
  36          *     'backends'    : Array of backend config and multi-backend settings.
  37          *                     Each value is the config used in the constructor of a
  38          *                     FileBackendStore class, but with these additional settings:
  39          *                         'class'         : The name of the backend class
  40          *                         'isMultiMaster' : This must be set for one backend.
  41          *     'syncChecks'  : Integer bitfield of internal backend sync checks to perform.
  42          *                     Possible bits include self::CHECK_SIZE and self::CHECK_TIME.
  43          *                     The checks are done before allowing any file operations.
  44          * @param $config Array
  45          */
  46         public function __construct( array $config ) {
  47                 parent::__construct( $config );
  48                 $namesUsed = array();
  49                 // Construct backends here rather than via registration
  50                 // to keep these backends hidden from outside the proxy.
  51                 foreach ( $config['backends'] as $index => $config ) {
  52                         $name = $config['name'];
  53                         if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
  54                                 throw new MWException( "Two or more backends defined with the name $name." );
  55                         }
  56                         $namesUsed[$name] = 1;
  57                         if ( !isset( $config['class'] ) ) {
  58                                 throw new MWException( 'No class given for a backend config.' );
  59                         }
  60                         $class = $config['class'];
  61                         $this->backends[$index] = new $class( $config );
  62                         if ( !empty( $config['isMultiMaster'] ) ) {
  63                                 if ( $this->masterIndex >= 0 ) {
  64                                         throw new MWException( 'More than one master backend defined.' );
  65                                 }
  66                                 $this->masterIndex = $index;
  67                         }
  68                 }
  69                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
  70                         throw new MWException( 'No master backend defined.' );
  71                 }
  72                 $this->syncChecks = isset( $config['syncChecks'] )
  73                         ? $config['syncChecks']
  74                         : self::CHECK_SIZE;
  75         }
  76
  77         /**
  78          * @see FileBackend::doOperationsInternal()
  79          */
  80         final protected function doOperationsInternal( array $ops, array $opts ) {
  81                 $status = Status::newGood();
  82
  83                 $performOps = array(); // list of FileOp objects
  84                 $filesRead = $filesChanged = array(); // storage paths used
  85                 // Build up a list of FileOps. The list will have all the ops
  86                 // for one backend, then all the ops for the next, and so on.
  87                 // These batches of ops are all part of a continuous array.
  88                 // Also build up a list of files read/changed...
  89                 foreach ( $this->backends as $index => $backend ) {
  90                         $backendOps = $this->substOpBatchPaths( $ops, $backend );
  91                         // Add on the operation batch for this backend
  92                         $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
  93                         if ( $index == 0 ) { // first batch
  94                                 // Get the files used for these operations. Each backend has a batch of
  95                                 // the same operations, so we only need to get them from the first batch.
  96                                 foreach ( $performOps as $fileOp ) {
  97                                         $filesRead = array_merge( $filesRead, $fileOp->storagePathsRead() );
  98                                         $filesChanged = array_merge( $filesChanged, $fileOp->storagePathsChanged() );
  99                                 }
 100                                 // Get the paths under the proxy backend's name
 101                                 $filesRead = $this->unsubstPaths( $filesRead );
 102                                 $filesChanged = $this->unsubstPaths( $filesChanged );
 103                         }
 104                 }
 105
 106                 // Try to lock those files for the scope of this function...
 107                 if ( empty( $opts['nonLocking'] ) ) {
 108                         $filesLockSh = array_diff( $filesRead, $filesChanged ); // optimization
 109                         $filesLockEx = $filesChanged;
 110                         // Get a shared lock on the parent directory of each path changed
 111                         $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
 112                         // Try to lock those files for the scope of this function...
 113                         $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
 114                         $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 115                         if ( !$status->isOK() ) {
 116                                 return $status; // abort
 117                         }
 118                 }
 119
 120                 // Clear any cache entries (after locks acquired)
 121                 $this->clearCache();
 122
 123                 // Do a consistency check to see if the backends agree
 124                 if ( count( $this->backends ) > 1 ) {
 125                         $status->merge( $this->consistencyCheck( array_merge( $filesRead, $filesChanged ) ) );
 126                         if ( !$status->isOK() ) {
 127                                 return $status; // abort
 128                         }
 129                 }
 130
 131                 // Actually attempt the operation batch...
 132                 $subStatus = FileOp::attemptBatch( $performOps, $opts );
 133
 134                 $success = array();
 135                 $failCount = $successCount = 0;
 136                 // Make 'success', 'successCount', and 'failCount' fields reflect
 137                 // the overall operation, rather than all the batches for each backend.
 138                 // Do this by only using success values from the master backend's batch.
 139                 $batchStart = $this->masterIndex * count( $ops );
 140                 $batchEnd = $batchStart + count( $ops ) - 1;
 141                 for ( $i = $batchStart; $i <= $batchEnd; $i++ ) {
 142                         if ( !isset( $subStatus->success[$i] ) ) {
 143                                 break; // failed out before trying this op
 144                         } elseif ( $subStatus->success[$i] ) {
 145                                 ++$successCount;
 146                         } else {
 147                                 ++$failCount;
 148                         }
 149                         $success[] = $subStatus->success[$i];
 150                 }
 151                 $subStatus->success = $success;
 152                 $subStatus->successCount = $successCount;
 153                 $subStatus->failCount = $failCount;
 154
 155                 // Merge errors into status fields
 156                 $status->merge( $subStatus );
 157                 $status->success = $subStatus->success; // not done in merge()
 158
 159                 return $status;
 160         }
 161
 162         /**
 163          * Check that a set of files are consistent across all internal backends
 164          *
 165          * @param $paths Array
 166          * @return Status
 167          */
 168         public function consistencyCheck( array $paths ) {
 169                 $status = Status::newGood();
 170                 if ( $this->syncChecks == 0 ) {
 171                         return $status; // skip checks
 172                 }
 173
 174                 $mBackend = $this->backends[$this->masterIndex];
 175                 foreach ( array_unique( $paths ) as $path ) {
 176                         $params = array( 'src' => $path, 'latest' => true );
 177                         // Stat the file on the 'master' backend
 178                         $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
 179                         // Check of all clone backends agree with the master...
 180                         foreach ( $this->backends as $index => $cBackend ) {
 181                                 if ( $index === $this->masterIndex ) {
 182                                         continue; // master
 183                                 }
 184                                 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
 185                                 if ( $mStat ) { // file is in master
 186                                         if ( !$cStat ) { // file should exist
 187                                                 $status->fatal( 'backend-fail-synced', $path );
 188                                                 continue;
 189                                         }
 190                                         if ( $this->syncChecks & self::CHECK_SIZE ) {
 191                                                 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
 192                                                         $status->fatal( 'backend-fail-synced', $path );
 193                                                         continue;
 194                                                 }
 195                                         }
 196                                         if ( $this->syncChecks & self::CHECK_TIME ) {
 197                                                 $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
 198                                                 $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
 199                                                 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
 200                                                         $status->fatal( 'backend-fail-synced', $path );
 201                                                         continue;
 202                                                 }
 203                                         }
 204                                 } else { // file is not in master
 205                                         if ( $cStat ) { // file should not exist
 206                                                 $status->fatal( 'backend-fail-synced', $path );
 207                                         }
 208                                 }
 209                         }
 210                 }
 211
 212                 return $status;
 213         }
 214
 215         /**
 216          * Substitute the backend name in storage path parameters
 217          * for a set of operations with that of a given internal backend.
 218          *
 219          * @param $ops Array List of file operation arrays
 220          * @param $backend FileBackendStore
 221          * @return Array
 222          */
 223         protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
 224                 $newOps = array(); // operations
 225                 foreach ( $ops as $op ) {
 226                         $newOp = $op; // operation
 227                         foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
 228                                 if ( isset( $newOp[$par] ) ) { // string or array
 229                                         $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
 230                                 }
 231                         }
 232                         $newOps[] = $newOp;
 233                 }
 234                 return $newOps;
 235         }
 236
 237         /**
 238          * Same as substOpBatchPaths() but for a single operation
 239          *
 240          * @param $op File operation array
 241          * @param $backend FileBackendStore
 242          * @return Array
 243          */
 244         protected function substOpPaths( array $ops, FileBackendStore $backend ) {
 245                 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
 246                 return $newOps[0];
 247         }
 248
 249         /**
 250          * Substitute the backend of storage paths with an internal backend's name
 251          *
 252          * @param $paths Array|string List of paths or single string path
 253          * @param $backend FileBackendStore
 254          * @return Array|string
 255          */
 256         protected function substPaths( $paths, FileBackendStore $backend ) {
 257                 return preg_replace(
 258                         '!^mwstore://' . preg_quote( $this->name ) . '/!',
 259                         'mwstore://' . $backend->getName() . '/',
 260                         $paths // string or array
 261                 );
 262         }
 263
 264         /**
 265          * Substitute the backend of internal storage paths with the proxy backend's name
 266          *
 267          * @param $paths Array|string List of paths or single string path
 268          * @return Array|string
 269          */
 270         protected function unsubstPaths( $paths ) {
 271                 return preg_replace(
 272                         '!^mwstore://([^/]+)!',
 273                         "mwstore://{$this->name}",
 274                         $paths // string or array
 275                 );
 276         }
 277
 278         /**
 279          * @see FileBackend::doPrepare()
 280          */
 281         public function doPrepare( array $params ) {
 282                 $status = Status::newGood();
 283                 foreach ( $this->backends as $backend ) {
 284                         $realParams = $this->substOpPaths( $params, $backend );
 285                         $status->merge( $backend->doPrepare( $realParams ) );
 286                 }
 287                 return $status;
 288         }
 289
 290         /**
 291          * @see FileBackend::doSecure()
 292          */
 293         public function doSecure( array $params ) {
 294                 $status = Status::newGood();
 295                 foreach ( $this->backends as $backend ) {
 296                         $realParams = $this->substOpPaths( $params, $backend );
 297                         $status->merge( $backend->doSecure( $realParams ) );
 298                 }
 299                 return $status;
 300         }
 301
 302         /**
 303          * @see FileBackend::doClean()
 304          */
 305         public function doClean( array $params ) {
 306                 $status = Status::newGood();
 307                 foreach ( $this->backends as $backend ) {
 308                         $realParams = $this->substOpPaths( $params, $backend );
 309                         $status->merge( $backend->doClean( $realParams ) );
 310                 }
 311                 return $status;
 312         }
 313
 314         /**
 315          * @see FileBackend::getFileList()
 316          */
 317         public function concatenate( array $params ) {
 318                 // We are writing to an FS file, so we don't need to do this per-backend
 319                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 320                 return $this->backends[$this->masterIndex]->concatenate( $realParams );
 321         }
 322
 323         /**
 324          * @see FileBackend::fileExists()
 325          */
 326         public function fileExists( array $params ) {
 327                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 328                 return $this->backends[$this->masterIndex]->fileExists( $realParams );
 329         }
 330
 331         /**
 332          * @see FileBackend::getFileTimestamp()
 333          */
 334         public function getFileTimestamp( array $params ) {
 335                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 336                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 337         }
 338
 339         /**
 340          * @see FileBackend::getFileSize()
 341          */
 342         public function getFileSize( array $params ) {
 343                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 344                 return $this->backends[$this->masterIndex]->getFileSize( $realParams );
 345         }
 346
 347         /**
 348          * @see FileBackend::getFileStat()
 349          */
 350         public function getFileStat( array $params ) {
 351                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 352                 return $this->backends[$this->masterIndex]->getFileStat( $realParams );
 353         }
 354
 355         /**
 356          * @see FileBackend::getFileContents()
 357          */
 358         public function getFileContents( array $params ) {
 359                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 360                 return $this->backends[$this->masterIndex]->getFileContents( $realParams );
 361         }
 362
 363         /**
 364          * @see FileBackend::getFileSha1Base36()
 365          */
 366         public function getFileSha1Base36( array $params ) {
 367                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 368                 return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
 369         }
 370
 371         /**
 372          * @see FileBackend::getFileProps()
 373          */
 374         public function getFileProps( array $params ) {
 375                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 376                 return $this->backends[$this->masterIndex]->getFileProps( $realParams );
 377         }
 378
 379         /**
 380          * @see FileBackend::streamFile()
 381          */
 382         public function streamFile( array $params ) {
 383                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 384                 return $this->backends[$this->masterIndex]->streamFile( $realParams );
 385         }
 386
 387         /**
 388          * @see FileBackend::getLocalReference()
 389          */
 390         public function getLocalReference( array $params ) {
 391                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 392                 return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
 393         }
 394
 395         /**
 396          * @see FileBackend::getLocalCopy()
 397          */
 398         public function getLocalCopy( array $params ) {
 399                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 400                 return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
 401         }
 402
 403         /**
 404          * @see FileBackend::getFileList()
 405          */
 406         public function getFileList( array $params ) {
 407                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 408                 return $this->backends[$this->masterIndex]->getFileList( $realParams );
 409         }
 410
 411         /**
 412          * @see FileBackend::clearCache()
 413          */
 414         public function clearCache( array $paths = null ) {
 415                 foreach ( $this->backends as $backend ) {
 416                         $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
 417                         $backend->clearCache( $realPaths );
 418                 }
 419         }
 420 }