9 * @brief Proxy backend that mirrors writes to several internal backends.
11 * This class defines a multi-write backend. Multiple backends can be
12 * registered to this proxy backend and it will act as a single backend.
13 * Use this when all access to those backends is through this proxy backend.
14 * At least one of the backends must be declared the "master" backend.
16 * Only use this class when transitioning from one storage system to another.
18 * Read operations are only done on the 'master' backend for consistency.
19 * Write operations are performed on all backends, in the order defined.
20 * If an operation fails on one backend it will be rolled back from the others.
22 * @ingroup FileBackend
25 class FileBackendMultiWrite
extends FileBackend
{
26 /** @var Array Prioritized list of FileBackendStore objects */
27 protected $backends = array(); // array of (backend index => backends)
28 protected $masterIndex = -1; // integer; index of master backend
29 protected $syncChecks = 0; // integer bitfield
31 /* Possible internal backend consistency checks */
36 * Construct a proxy backend that consists of several internal backends.
37 * Additional $config params include:
38 * 'backends' : Array of backend config and multi-backend settings.
39 * Each value is the config used in the constructor of a
40 * FileBackendStore class, but with these additional settings:
41 * 'class' : The name of the backend class
42 * 'isMultiMaster' : This must be set for one backend.
43 * 'syncChecks' : Integer bitfield of internal backend sync checks to perform.
44 * Possible bits include self::CHECK_SIZE and self::CHECK_TIME.
45 * The checks are done before allowing any file operations.
46 * @param $config Array
48 public function __construct( array $config ) {
49 parent
::__construct( $config );
51 // Construct backends here rather than via registration
52 // to keep these backends hidden from outside the proxy.
53 foreach ( $config['backends'] as $index => $config ) {
54 $name = $config['name'];
55 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
56 throw new MWException( "Two or more backends defined with the name $name." );
58 $namesUsed[$name] = 1;
59 if ( !isset( $config['class'] ) ) {
60 throw new MWException( 'No class given for a backend config.' );
62 $class = $config['class'];
63 $this->backends
[$index] = new $class( $config );
64 if ( !empty( $config['isMultiMaster'] ) ) {
65 if ( $this->masterIndex
>= 0 ) {
66 throw new MWException( 'More than one master backend defined.' );
68 $this->masterIndex
= $index;
71 if ( $this->masterIndex
< 0 ) { // need backends and must have a master
72 throw new MWException( 'No master backend defined.' );
74 $this->syncChecks
= isset( $config['syncChecks'] )
75 ?
$config['syncChecks']
80 * @see FileBackend::doOperationsInternal()
83 final protected function doOperationsInternal( array $ops, array $opts ) {
84 $status = Status
::newGood();
86 $performOps = array(); // list of FileOp objects
87 $paths = array(); // storage paths read from or written to
88 // Build up a list of FileOps. The list will have all the ops
89 // for one backend, then all the ops for the next, and so on.
90 // These batches of ops are all part of a continuous array.
91 // Also build up a list of files read/changed...
92 foreach ( $this->backends
as $index => $backend ) {
93 $backendOps = $this->substOpBatchPaths( $ops, $backend );
94 // Add on the operation batch for this backend
95 $performOps = array_merge( $performOps,
96 $backend->getOperationsInternal( $backendOps ) );
97 if ( $index == 0 ) { // first batch
98 // Get the files used for these operations. Each backend has a batch of
99 // the same operations, so we only need to get them from the first batch.
100 $paths = $backend->getPathsToLockForOpsInternal( $performOps );
101 // Get the paths under the proxy backend's name
102 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
103 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
107 // Try to lock those files for the scope of this function...
108 if ( empty( $opts['nonLocking'] ) ) {
109 // Try to lock those files for the scope of this function...
110 $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status );
111 $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status );
112 if ( !$status->isOK() ) {
113 return $status; // abort
117 // Clear any cache entries (after locks acquired)
120 // Do a consistency check to see if the backends agree
121 if ( count( $this->backends
) > 1 ) {
122 $status->merge( $this->consistencyCheck( array_merge( $paths['sh'], $paths['ex'] ) ) );
123 if ( !$status->isOK() ) {
124 return $status; // abort
128 // Actually attempt the operation batch...
129 $subStatus = FileOp
::attemptBatch( $performOps, $opts, $this->fileJournal
);
134 // Make 'success', 'successCount', and 'failCount' fields reflect
135 // the overall operation, rather than all the batches for each backend.
136 // Do this by only using success values from the master backend's batch.
137 $batchStart = $this->masterIndex
* count( $ops );
138 $batchEnd = $batchStart +
count( $ops ) - 1;
139 for ( $i = $batchStart; $i <= $batchEnd; $i++
) {
140 if ( !isset( $subStatus->success
[$i] ) ) {
141 break; // failed out before trying this op
142 } elseif ( $subStatus->success
[$i] ) {
147 $success[] = $subStatus->success
[$i];
149 $subStatus->success
= $success;
150 $subStatus->successCount
= $successCount;
151 $subStatus->failCount
= $failCount;
153 // Merge errors into status fields
154 $status->merge( $subStatus );
155 $status->success
= $subStatus->success
; // not done in merge()
161 * Check that a set of files are consistent across all internal backends
163 * @param $paths Array
166 public function consistencyCheck( array $paths ) {
167 $status = Status
::newGood();
168 if ( $this->syncChecks
== 0 ) {
169 return $status; // skip checks
172 $mBackend = $this->backends
[$this->masterIndex
];
173 foreach ( array_unique( $paths ) as $path ) {
174 $params = array( 'src' => $path, 'latest' => true );
175 // Stat the file on the 'master' backend
176 $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
177 // Check of all clone backends agree with the master...
178 foreach ( $this->backends
as $index => $cBackend ) {
179 if ( $index === $this->masterIndex
) {
182 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
183 if ( $mStat ) { // file is in master
184 if ( !$cStat ) { // file should exist
185 $status->fatal( 'backend-fail-synced', $path );
188 if ( $this->syncChecks
& self
::CHECK_SIZE
) {
189 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
190 $status->fatal( 'backend-fail-synced', $path );
194 if ( $this->syncChecks
& self
::CHECK_TIME
) {
195 $mTs = wfTimestamp( TS_UNIX
, $mStat['mtime'] );
196 $cTs = wfTimestamp( TS_UNIX
, $cStat['mtime'] );
197 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
198 $status->fatal( 'backend-fail-synced', $path );
202 } else { // file is not in master
203 if ( $cStat ) { // file should not exist
204 $status->fatal( 'backend-fail-synced', $path );
214 * Substitute the backend name in storage path parameters
215 * for a set of operations with that of a given internal backend.
217 * @param $ops Array List of file operation arrays
218 * @param $backend FileBackendStore
221 protected function substOpBatchPaths( array $ops, FileBackendStore
$backend ) {
222 $newOps = array(); // operations
223 foreach ( $ops as $op ) {
224 $newOp = $op; // operation
225 foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
226 if ( isset( $newOp[$par] ) ) { // string or array
227 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
236 * Same as substOpBatchPaths() but for a single operation
238 * @param $op File operation array
239 * @param $backend FileBackendStore
242 protected function substOpPaths( array $ops, FileBackendStore
$backend ) {
243 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
248 * Substitute the backend of storage paths with an internal backend's name
250 * @param $paths Array|string List of paths or single string path
251 * @param $backend FileBackendStore
252 * @return Array|string
254 protected function substPaths( $paths, FileBackendStore
$backend ) {
256 '!^mwstore://' . preg_quote( $this->name
) . '/!',
257 StringUtils
::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
258 $paths // string or array
263 * Substitute the backend of internal storage paths with the proxy backend's name
265 * @param $paths Array|string List of paths or single string path
266 * @return Array|string
268 protected function unsubstPaths( $paths ) {
270 '!^mwstore://([^/]+)!',
271 StringUtils
::escapeRegexReplacement( "mwstore://{$this->name}" ),
272 $paths // string or array
277 * @see FileBackend::doPrepare()
280 protected function doPrepare( array $params ) {
281 $status = Status
::newGood();
282 foreach ( $this->backends
as $backend ) {
283 $realParams = $this->substOpPaths( $params, $backend );
284 $status->merge( $backend->doPrepare( $realParams ) );
290 * @see FileBackend::doSecure()
293 protected function doSecure( array $params ) {
294 $status = Status
::newGood();
295 foreach ( $this->backends
as $backend ) {
296 $realParams = $this->substOpPaths( $params, $backend );
297 $status->merge( $backend->doSecure( $realParams ) );
303 * @see FileBackend::doClean()
306 protected function doClean( array $params ) {
307 $status = Status
::newGood();
308 foreach ( $this->backends
as $backend ) {
309 $realParams = $this->substOpPaths( $params, $backend );
310 $status->merge( $backend->doClean( $realParams ) );
316 * @see FileBackend::concatenate()
318 public function concatenate( array $params ) {
319 // We are writing to an FS file, so we don't need to do this per-backend
320 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
321 return $this->backends
[$this->masterIndex
]->concatenate( $realParams );
325 * @see FileBackend::fileExists()
327 public function fileExists( array $params ) {
328 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
329 return $this->backends
[$this->masterIndex
]->fileExists( $realParams );
333 * @see FileBackend::getFileTimestamp()
335 public function getFileTimestamp( array $params ) {
336 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
337 return $this->backends
[$this->masterIndex
]->getFileTimestamp( $realParams );
341 * @see FileBackend::getFileSize()
343 public function getFileSize( array $params ) {
344 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
345 return $this->backends
[$this->masterIndex
]->getFileSize( $realParams );
349 * @see FileBackend::getFileStat()
351 public function getFileStat( array $params ) {
352 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
353 return $this->backends
[$this->masterIndex
]->getFileStat( $realParams );
357 * @see FileBackend::getFileContents()
359 public function getFileContents( array $params ) {
360 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
361 return $this->backends
[$this->masterIndex
]->getFileContents( $realParams );
365 * @see FileBackend::getFileSha1Base36()
367 public function getFileSha1Base36( array $params ) {
368 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
369 return $this->backends
[$this->masterIndex
]->getFileSha1Base36( $realParams );
373 * @see FileBackend::getFileProps()
375 public function getFileProps( array $params ) {
376 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
377 return $this->backends
[$this->masterIndex
]->getFileProps( $realParams );
381 * @see FileBackend::streamFile()
383 public function streamFile( array $params ) {
384 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
385 return $this->backends
[$this->masterIndex
]->streamFile( $realParams );
389 * @see FileBackend::getLocalReference()
391 public function getLocalReference( array $params ) {
392 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
393 return $this->backends
[$this->masterIndex
]->getLocalReference( $realParams );
397 * @see FileBackend::getLocalCopy()
399 public function getLocalCopy( array $params ) {
400 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
401 return $this->backends
[$this->masterIndex
]->getLocalCopy( $realParams );
405 * @see FileBackend::directoryExists()
407 public function directoryExists( array $params ) {
408 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
409 return $this->backends
[$this->masterIndex
]->directoryExists( $realParams );
413 * @see FileBackend::getSubdirectoryList()
415 public function getDirectoryList( array $params ) {
416 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
417 return $this->backends
[$this->masterIndex
]->getDirectoryList( $realParams );
421 * @see FileBackend::getFileList()
423 public function getFileList( array $params ) {
424 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
425 return $this->backends
[$this->masterIndex
]->getFileList( $realParams );
429 * @see FileBackend::clearCache()
431 public function clearCache( array $paths = null ) {
432 foreach ( $this->backends
as $backend ) {
433 $realPaths = is_array( $paths ) ?
$this->substPaths( $paths, $backend ) : null;
434 $backend->clearCache( $realPaths );