includes/filerepo/backend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * This class defines a multi-write backend. Multiple backends can be
  10  * registered to this proxy backend and it will act as a single backend.
  11  * Use this when all access to those backends is through this proxy backend.
  12  * At least one of the backends must be declared the "master" backend.
  13  *
  14  * Only use this class when transitioning from one storage system to another.
  15  *
  16  * The order that the backends are defined sets the priority of which
  17  * backend is read from or written to first. Functions like fileExists()
  18  * and getFileProps() will return information based on the first backend
  19  * that has the file. Special cases are listed below:
  20  *     a) getFileTimestamp() will always check only the master backend to
  21  *        avoid confusing and inconsistent results.
  22  *
  23  * All write operations are performed on all backends.
  24  * If an operation fails on one backend it will be rolled back from the others.
  25  *
  26  * @ingroup FileBackend
  27  */
  28 class FileBackendMultiWrite extends FileBackendBase {
  29         /** @var Array Prioritized list of FileBackend objects */
  30         protected $fileBackends = array(); // array of (backend index => backends)
  31         protected $masterIndex = -1; // index of master backend
  32
  33         /**
  34          * Construct a proxy backend that consists of several internal backends.
  35          * $config contains:
  36          *     'name'        : The name of the proxy backend
  37          *     'lockManager' : Registered name of the file lock manager to use
  38          *     'backends'    : Array of backend config and multi-backend settings.
  39          *                     Each value is the config used in the constructor of a
  40          *                     FileBackend class, but with these additional settings:
  41          *                         'class'         : The name of the backend class
  42          *                         'isMultiMaster' : This must be set for one backend.
  43          * @param $config Array
  44          */
  45         public function __construct( array $config ) {
  46                 parent::__construct( $config );
  47                 // Construct backends here rather than via registration
  48                 // to keep these backends hidden from outside the proxy.
  49                 foreach ( $config['backends'] as $index => $config ) {
  50                         if ( !isset( $config['class'] ) ) {
  51                                 throw new MWException( 'No class given for a backend config.' );
  52                         }
  53                         $class = $config['class'];
  54                         $this->fileBackends[$index] = new $class( $config );
  55                         if ( !empty( $config['isMultiMaster'] ) ) {
  56                                 if ( $this->masterIndex >= 0 ) {
  57                                         throw new MWException( 'More than one master backend defined.' );
  58                                 }
  59                                 $this->masterIndex = $index;
  60                         }
  61                 }
  62                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
  63                         throw new MWException( 'No master backend defined.' );
  64                 }
  65         }
  66
  67         /**
  68          * @see FileBackendBase::doOperationsInternal()
  69          */
  70         final protected function doOperationsInternal( array $ops, array $opts ) {
  71                 $status = Status::newGood();
  72
  73                 $performOps = array(); // list of FileOp objects
  74                 $filesLockEx = $filesLockSh = array(); // storage paths to lock
  75                 // Build up a list of FileOps. The list will have all the ops
  76                 // for one backend, then all the ops for the next, and so on.
  77                 // These batches of ops are all part of a continuous array.
  78                 // Also build up a list of files to lock...
  79                 foreach ( $this->fileBackends as $index => $backend ) {
  80                         $backendOps = $this->substOpPaths( $ops, $backend );
  81                         $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
  82                         if ( $index == 0 && empty( $opts['nonLocking'] ) ) {
  83                                 // Set "files to lock" from the first batch so we don't try to set all
  84                                 // locks two or three times over (depending on the number of backends).
  85                                 // A lock on one storage path is a lock on all the backends.
  86                                 foreach ( $performOps as $index => $fileOp ) {
  87                                         $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
  88                                         $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
  89                                 }
  90                                 // Optimization: if doing an EX lock anyway, don't also set an SH one
  91                                 $filesLockSh = array_diff( $filesLockSh, $filesLockEx );
  92                                 // Lock the paths under the proxy backend's name
  93                                 $this->unsubstPaths( $filesLockSh );
  94                                 $this->unsubstPaths( $filesLockEx );
  95                         }
  96                 }
  97
  98                 // Try to lock those files for the scope of this function...
  99                 $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
 100                 $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 101                 if ( !$status->isOK() ) {
 102                         return $status; // abort
 103                 }
 104
 105                 // Clear any cache entries (after locks acquired)
 106                 foreach ( $this->fileBackends as $backend ) {
 107                         $backend->clearCache();
 108                 }
 109                 // Actually attempt the operation batch...
 110                 $status->merge( FileOp::attemptBatch( $performOps, $opts ) );
 111
 112                 return $status;
 113         }
 114
 115         /**
 116          * Substitute the backend name in storage path parameters
 117          * for a set of operations with a that of a given backend.
 118          *
 119          * @param $ops Array List of file operation arrays
 120          * @param $backend FileBackend
 121          * @return Array
 122          */
 123         protected function substOpPaths( array $ops, FileBackend $backend ) {
 124                 $newOps = array(); // operations
 125                 foreach ( $ops as $op ) {
 126                         $newOp = $op; // operation
 127                         foreach ( array( 'src', 'srcs', 'dst' ) as $par ) {
 128                                 if ( isset( $newOp[$par] ) ) {
 129                                         $newOp[$par] = preg_replace(
 130                                                 '!^mwstore://' . preg_quote( $this->name ) . '/!',
 131                                                 'mwstore://' . $backend->getName() . '/',
 132                                                 $newOp[$par] // string or array
 133                                         );
 134                                 }
 135                         }
 136                         $newOps[] = $newOp;
 137                 }
 138                 return $newOps;
 139         }
 140
 141         /**
 142          * Replace the backend part of storage paths with this backend's name
 143          *
 144          * @param &$paths Array
 145          * @return void
 146          */
 147         protected function unsubstPaths( array &$paths ) {
 148                 foreach ( $paths as &$path ) {
 149                         $path = preg_replace( '!^mwstore://([^/]+)!', "mwstore://{$this->name}", $path );
 150                 }
 151         }
 152
 153         /**
 154          * @see FileBackendBase::prepare()
 155          */
 156         function prepare( array $params ) {
 157                 $status = Status::newGood();
 158                 foreach ( $this->backends as $backend ) {
 159                         $realParams = $this->substOpPaths( $params, $backend );
 160                         $status->merge( $backend->prepare( $realParams ) );
 161                 }
 162                 return $status;
 163         }
 164
 165         /**
 166          * @see FileBackendBase::secure()
 167          */
 168         function secure( array $params ) {
 169                 $status = Status::newGood();
 170                 foreach ( $this->backends as $backend ) {
 171                         $realParams = $this->substOpPaths( $params, $backend );
 172                         $status->merge( $backend->secure( $realParams ) );
 173                 }
 174                 return $status;
 175         }
 176
 177         /**
 178          * @see FileBackendBase::clean()
 179          */
 180         function clean( array $params ) {
 181                 $status = Status::newGood();
 182                 foreach ( $this->backends as $backend ) {
 183                         $realParams = $this->substOpPaths( $params, $backend );
 184                         $status->merge( $backend->clean( $realParams ) );
 185                 }
 186                 return $status;
 187         }
 188
 189         /**
 190          * @see FileBackendBase::fileExists()
 191          */
 192         function fileExists( array $params ) {
 193                 # Hit all backends in case of failed operations (out of sync)
 194                 foreach ( $this->backends as $backend ) {
 195                         $realParams = $this->substOpPaths( $params, $backend );
 196                         if ( $backend->fileExists( $realParams ) ) {
 197                                 return true;
 198                         }
 199                 }
 200                 return false;
 201         }
 202
 203         /**
 204          * @see FileBackendBase::getFileTimestamp()
 205          */
 206         function getFileTimestamp( array $params ) {
 207                 // Skip non-master for consistent timestamps
 208                 $realParams = $this->substOpPaths( $params, $backend );
 209                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 210         }
 211
 212         /**
 213          * @see FileBackendBase::getFileSha1Base36()
 214          */
 215         function getFileSha1Base36( array $params ) {
 216                 # Hit all backends in case of failed operations (out of sync)
 217                 foreach ( $this->backends as $backend ) {
 218                         $realParams = $this->substOpPaths( $params, $backend );
 219                         $hash = $backend->getFileSha1Base36( $realParams );
 220                         if ( $hash !== false ) {
 221                                 return $hash;
 222                         }
 223                 }
 224                 return false;
 225         }
 226
 227         /**
 228          * @see FileBackendBase::getFileProps()
 229          */
 230         function getFileProps( array $params ) {
 231                 # Hit all backends in case of failed operations (out of sync)
 232                 foreach ( $this->backends as $backend ) {
 233                         $realParams = $this->substOpPaths( $params, $backend );
 234                         $props = $backend->getFileProps( $realParams );
 235                         if ( $props !== null ) {
 236                                 return $props;
 237                         }
 238                 }
 239                 return null;
 240         }
 241
 242         /**
 243          * @see FileBackendBase::streamFile()
 244          */
 245         function streamFile( array $params ) {
 246                 $status = Status::newGood();
 247                 foreach ( $this->backends as $backend ) {
 248                         $realParams = $this->substOpPaths( $params, $backend );
 249                         $subStatus = $backend->streamFile( $realParams );
 250                         $status->merge( $subStatus );
 251                         if ( $subStatus->isOK() ) {
 252                                 // Pass isOK() despite fatals from other backends
 253                                 $status->setResult( true );
 254                                 return $status;
 255                         } else { // failure
 256                                 if ( headers_sent() ) {
 257                                         return $status; // died mid-stream...so this is already fubar
 258                                 } elseif ( strval( ob_get_contents() ) !== '' ) {
 259                                         ob_clean(); // output was buffered but not sent; clear it
 260                                 }
 261                         }
 262                 }
 263                 return $status;
 264         }
 265
 266         /**
 267          * @see FileBackendBase::getLocalReference()
 268          */
 269         function getLocalReference( array $params ) {
 270                 # Hit all backends in case of failed operations (out of sync)
 271                 foreach ( $this->backends as $backend ) {
 272                         $realParams = $this->substOpPaths( $params, $backend );
 273                         $fsFile = $backend->getLocalReference( $realParams );
 274                         if ( $fsFile ) {
 275                                 return $fsFile;
 276                         }
 277                 }
 278                 return null;
 279         }
 280
 281         /**
 282          * @see FileBackendBase::getLocalCopy()
 283          */
 284         function getLocalCopy( array $params ) {
 285                 # Hit all backends in case of failed operations (out of sync)
 286                 foreach ( $this->backends as $backend ) {
 287                         $realParams = $this->substOpPaths( $params, $backend );
 288                         $tmpFile = $backend->getLocalCopy( $realParams );
 289                         if ( $tmpFile ) {
 290                                 return $tmpFile;
 291                         }
 292                 }
 293                 return null;
 294         }
 295
 296         /**
 297          * @see FileBackendBase::getFileList()
 298          */
 299         function getFileList( array $params ) {
 300                 foreach ( $this->backends as $index => $backend ) {
 301                         # Get results from the first backend
 302                         $realParams = $this->substOpPaths( $params, $backend );
 303                         return $backend->getFileList( $realParams );
 304                 }
 305                 return array(); // sanity
 306         }
 307 }