includes/filerepo/backend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * This class defines a multi-write backend. Multiple backends can be
  10  * registered to this proxy backend and it will act as a single backend.
  11  * Use this when all access to those backends is through this proxy backend.
  12  * At least one of the backends must be declared the "master" backend.
  13  *
  14  * The order that the backends are defined sets the priority of which
  15  * backend is read from or written to first. Functions like fileExists()
  16  * and getFileProps() will return information based on the first backend
  17  * that has the file. Special cases are listed below:
  18  *     a) getFileTimestamp() will always check only the master backend to
  19  *        avoid confusing and inconsistent results.
  20  *
  21  * All write operations are performed on all backends.
  22  * If an operation fails on one backend it will be rolled back from the others.
  23  *
  24  * @ingroup FileBackend
  25  */
  26 class FileBackendMultiWrite extends FileBackendBase {
  27         /** @var Array Prioritized list of FileBackend objects */
  28         protected $fileBackends = array(); // array of (backend index => backends)
  29         protected $masterIndex = -1; // index of master backend
  30
  31         /**
  32          * Construct a proxy backend that consists of several internal backends.
  33          * $config contains:
  34          *     'name'        : The name of the proxy backend
  35          *     'lockManager' : Registered name of the file lock manager to use
  36          *     'backends'    : Array of backend config and multi-backend settings.
  37          *                     Each value is the config used in the constructor of a
  38          *                     FileBackend class, but with these additional settings:
  39          *                         'class'         : The name of the backend class
  40          *                         'isMultiMaster' : This must be set for one backend.
  41          * @param $config Array
  42          */
  43         public function __construct( array $config ) {
  44                 parent::__construct( $config );
  45                 // Construct backends here rather than via registration
  46                 // to keep these backends hidden from outside the proxy.
  47                 foreach ( $config['backends'] as $index => $config ) {
  48                         if ( !isset( $config['class'] ) ) {
  49                                 throw new MWException( 'No class given for a backend config.' );
  50                         }
  51                         $class = $config['class'];
  52                         $this->fileBackends[$index] = new $class( $config );
  53                         if ( !empty( $config['isMultiMaster'] ) ) {
  54                                 if ( $this->masterIndex >= 0 ) {
  55                                         throw new MWException( 'More than one master backend defined.' );
  56                                 }
  57                                 $this->masterIndex = $index;
  58                         }
  59                 }
  60                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
  61                         throw new MWException( 'No master backend defined.' );
  62                 }
  63         }
  64
  65         final public function doOperations( array $ops, array $opts = array() ) {
  66                 $status = Status::newGood();
  67
  68                 $performOps = array(); // list of FileOp objects
  69                 $filesLockEx = $filesLockSh = array(); // storage paths to lock
  70                 // Build up a list of FileOps. The list will have all the ops
  71                 // for one backend, then all the ops for the next, and so on.
  72                 // These batches of ops are all part of a continuous array.
  73                 // Also build up a list of files to lock...
  74                 foreach ( $this->fileBackends as $index => $backend ) {
  75                         $backendOps = $this->substOpPaths( $ops, $backend );
  76                         $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
  77                         if ( $index == 0 && empty( $opts['nonLocking'] ) ) {
  78                                 // Set "files to lock" from the first batch so we don't try to set all
  79                                 // locks two or three times over (depending on the number of backends).
  80                                 // A lock on one storage path is a lock on all the backends.
  81                                 foreach ( $performOps as $index => $fileOp ) {
  82                                         $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
  83                                         $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
  84                                 }
  85                                 // Lock the paths under the proxy backend's name
  86                                 $this->unsubstPaths( $filesLockSh );
  87                                 $this->unsubstPaths( $filesLockEx );
  88                         }
  89                 }
  90
  91                 // Try to lock those files for the scope of this function...
  92                 $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
  93                 $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
  94                 if ( !$status->isOK() ) {
  95                         return $status; // abort
  96                 }
  97
  98                 // Clear any cache entries (after locks acquired)
  99                 foreach ( $this->fileBackends as $backend ) {
 100                         $backend->clearCache();
 101                 }
 102                 // Actually attempt the operation batch...
 103                 $status->merge( FileOp::attemptBatch( $performOps, $opts ) );
 104
 105                 return $status;
 106         }
 107
 108         /**
 109          * Substitute the backend name in storage path parameters
 110          * for a set of operations with a that of a given backend.
 111          *
 112          * @param $ops Array List of file operation arrays
 113          * @param $backend FileBackend
 114          * @return Array
 115          */
 116         protected function substOpPaths( array $ops, FileBackend $backend ) {
 117                 $newOps = array(); // operations
 118                 foreach ( $ops as $op ) {
 119                         $newOp = $op; // operation
 120                         foreach ( array( 'src', 'srcs', 'dst' ) as $par ) {
 121                                 if ( isset( $newOp[$par] ) ) {
 122                                         $newOp[$par] = preg_replace(
 123                                                 '!^mwstore://' . preg_quote( $this->name ) . '/!',
 124                                                 'mwstore://' . $backend->getName() . '/',
 125                                                 $newOp[$par] // string or array
 126                                         );
 127                                 }
 128                         }
 129                         $newOps[] = $newOp;
 130                 }
 131                 return $newOps;
 132         }
 133
 134         /**
 135          * Replace the backend part of storage paths with this backend's name
 136          *
 137          * @param &$paths Array
 138          * @return void
 139          */
 140         protected function unsubstPaths( array &$paths ) {
 141                 foreach ( $paths as &$path ) {
 142                         $path = preg_replace( '!^mwstore://([^/]+)!', "mwstore://{$this->name}", $path );
 143                 }
 144         }
 145
 146         function prepare( array $params ) {
 147                 $status = Status::newGood();
 148                 foreach ( $this->backends as $backend ) {
 149                         $realParams = $this->substOpPaths( $params, $backend );
 150                         $status->merge( $backend->prepare( $realParams ) );
 151                 }
 152                 return $status;
 153         }
 154
 155         function secure( array $params ) {
 156                 $status = Status::newGood();
 157                 foreach ( $this->backends as $backend ) {
 158                         $realParams = $this->substOpPaths( $params, $backend );
 159                         $status->merge( $backend->secure( $realParams ) );
 160                 }
 161                 return $status;
 162         }
 163
 164         function clean( array $params ) {
 165                 $status = Status::newGood();
 166                 foreach ( $this->backends as $backend ) {
 167                         $realParams = $this->substOpPaths( $params, $backend );
 168                         $status->merge( $backend->clean( $realParams ) );
 169                 }
 170                 return $status;
 171         }
 172
 173         function fileExists( array $params ) {
 174                 # Hit all backends in case of failed operations (out of sync)
 175                 foreach ( $this->backends as $backend ) {
 176                         $realParams = $this->substOpPaths( $params, $backend );
 177                         if ( $backend->fileExists( $realParams ) ) {
 178                                 return true;
 179                         }
 180                 }
 181                 return false;
 182         }
 183
 184         function getFileTimestamp( array $params ) {
 185                 // Skip non-master for consistent timestamps
 186                 $realParams = $this->substOpPaths( $params, $backend );
 187                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 188         }
 189
 190         function getFileSha1Base36( array $params ) {
 191                 # Hit all backends in case of failed operations (out of sync)
 192                 foreach ( $this->backends as $backend ) {
 193                         $realParams = $this->substOpPaths( $params, $backend );
 194                         $hash = $backend->getFileSha1Base36( $realParams );
 195                         if ( $hash !== false ) {
 196                                 return $hash;
 197                         }
 198                 }
 199                 return false;
 200         }
 201
 202         function getFileProps( array $params ) {
 203                 # Hit all backends in case of failed operations (out of sync)
 204                 foreach ( $this->backends as $backend ) {
 205                         $realParams = $this->substOpPaths( $params, $backend );
 206                         $props = $backend->getFileProps( $realParams );
 207                         if ( $props !== null ) {
 208                                 return $props;
 209                         }
 210                 }
 211                 return null;
 212         }
 213
 214         function streamFile( array $params ) {
 215                 $status = Status::newGood();
 216                 foreach ( $this->backends as $backend ) {
 217                         $realParams = $this->substOpPaths( $params, $backend );
 218                         $subStatus = $backend->streamFile( $realParams );
 219                         $status->merge( $subStatus );
 220                         if ( $subStatus->isOK() ) {
 221                                 // Pass isOK() despite fatals from other backends
 222                                 $status->setResult( true );
 223                                 return $status;
 224                         } else { // failure
 225                                 if ( headers_sent() ) {
 226                                         return $status; // died mid-stream...so this is already fubar
 227                                 } elseif ( strval( ob_get_contents() ) !== '' ) {
 228                                         ob_clean(); // output was buffered but not sent; clear it
 229                                 }
 230                         }
 231                 }
 232                 return $status;
 233         }
 234
 235         function getLocalReference( array $params ) {
 236                 # Hit all backends in case of failed operations (out of sync)
 237                 foreach ( $this->backends as $backend ) {
 238                         $realParams = $this->substOpPaths( $params, $backend );
 239                         $fsFile = $backend->getLocalReference( $realParams );
 240                         if ( $fsFile ) {
 241                                 return $fsFile;
 242                         }
 243                 }
 244                 return null;
 245         }
 246
 247         function getLocalCopy( array $params ) {
 248                 # Hit all backends in case of failed operations (out of sync)
 249                 foreach ( $this->backends as $backend ) {
 250                         $realParams = $this->substOpPaths( $params, $backend );
 251                         $tmpFile = $backend->getLocalCopy( $realParams );
 252                         if ( $tmpFile ) {
 253                                 return $tmpFile;
 254                         }
 255                 }
 256                 return null;
 257         }
 258
 259         function getFileList( array $params ) {
 260                 foreach ( $this->backends as $index => $backend ) {
 261                         # Get results from the first backend
 262                         $realParams = $this->substOpPaths( $params, $backend );
 263                         return $backend->getFileList( $realParams );
 264                 }
 265                 return array(); // sanity
 266         }
 267 }