3 * Proxy backend that mirrors writes to several internal backends.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
22 * @author Aaron Schulz
26 * @brief Proxy backend that mirrors writes to several internal backends.
28 * This class defines a multi-write backend. Multiple backends can be
29 * registered to this proxy backend and it will act as a single backend.
30 * Use this when all access to those backends is through this proxy backend.
31 * At least one of the backends must be declared the "master" backend.
33 * Only use this class when transitioning from one storage system to another.
35 * Read operations are only done on the 'master' backend for consistency.
36 * Write operations are performed on all backends, in the order defined.
37 * If an operation fails on one backend it will be rolled back from the others.
39 * @ingroup FileBackend
42 class FileBackendMultiWrite
extends FileBackend
{
43 /** @var Array Prioritized list of FileBackendStore objects */
44 protected $backends = array(); // array of (backend index => backends)
45 protected $masterIndex = -1; // integer; index of master backend
46 protected $syncChecks = 0; // integer; bitfield
48 protected $noPushDirConts = array();
49 protected $noPushQuickOps = false; // boolean
51 /* Possible internal backend consistency checks */
57 * Construct a proxy backend that consists of several internal backends.
58 * Locking, journaling, and read-only checks are handled by the proxy backend.
60 * Additional $config params include:
61 * - backends : Array of backend config and multi-backend settings.
62 * Each value is the config used in the constructor of a
63 * FileBackendStore class, but with these additional settings:
64 * - class : The name of the backend class
65 * - isMultiMaster : This must be set for one backend.
66 * - template: : If given a backend name, this will use
67 * the config of that backend as a template.
68 * Values specified here take precedence.
69 * - syncChecks : Integer bitfield of internal backend sync checks to perform.
70 * Possible bits include the FileBackendMultiWrite::CHECK_* constants.
71 * There are constants for SIZE, TIME, and SHA1.
72 * The checks are done before allowing any file operations.
73 * - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend.
74 * - noPushDirConts : (hack) Only apply directory functions to the master backend.
76 * @param $config Array
79 public function __construct( array $config ) {
80 parent
::__construct( $config );
81 $this->syncChecks
= isset( $config['syncChecks'] )
82 ?
$config['syncChecks']
84 $this->noPushQuickOps
= isset( $config['noPushQuickOps'] )
85 ?
$config['noPushQuickOps']
87 $this->noPushDirConts
= isset( $config['noPushDirConts'] )
88 ?
$config['noPushDirConts']
90 // Construct backends here rather than via registration
91 // to keep these backends hidden from outside the proxy.
93 foreach ( $config['backends'] as $index => $config ) {
94 if ( isset( $config['template'] ) ) {
95 // Config is just a modified version of a registered backend's.
96 // This should only be used when that config is used only by this backend.
97 $config = $config + FileBackendGroup
::singleton()->config( $config['template'] );
99 $name = $config['name'];
100 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
101 throw new MWException( "Two or more backends defined with the name $name." );
103 $namesUsed[$name] = 1;
104 // Alter certain sub-backend settings for sanity
105 unset( $config['readOnly'] ); // use proxy backend setting
106 unset( $config['fileJournal'] ); // use proxy backend journal
107 $config['wikiId'] = $this->wikiId
; // use the proxy backend wiki ID
108 $config['lockManager'] = 'nullLockManager'; // lock under proxy backend
109 if ( !empty( $config['isMultiMaster'] ) ) {
110 if ( $this->masterIndex
>= 0 ) {
111 throw new MWException( 'More than one master backend defined.' );
113 $this->masterIndex
= $index; // this is the "master"
114 $config['fileJournal'] = $this->fileJournal
; // log under proxy backend
116 // Create sub-backend object
117 if ( !isset( $config['class'] ) ) {
118 throw new MWException( 'No class given for a backend config.' );
120 $class = $config['class'];
121 $this->backends
[$index] = new $class( $config );
123 if ( $this->masterIndex
< 0 ) { // need backends and must have a master
124 throw new MWException( 'No master backend defined.' );
129 * @see FileBackend::doOperationsInternal()
132 final protected function doOperationsInternal( array $ops, array $opts ) {
133 $status = Status
::newGood();
135 $mbe = $this->backends
[$this->masterIndex
]; // convenience
137 // Get the paths to lock from the master backend
138 $realOps = $this->substOpBatchPaths( $ops, $mbe );
139 $paths = $mbe->getPathsToLockForOpsInternal( $mbe->getOperationsInternal( $realOps ) );
140 // Get the paths under the proxy backend's name
141 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
142 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
143 // Try to lock those files for the scope of this function...
144 if ( empty( $opts['nonLocking'] ) ) {
145 // Try to lock those files for the scope of this function...
146 $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status );
147 $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status );
148 if ( !$status->isOK() ) {
149 return $status; // abort
152 // Clear any cache entries (after locks acquired)
154 $opts['preserveCache'] = true; // only locked files are cached
155 // Do a consistency check to see if the backends agree
156 $status->merge( $this->consistencyCheck( $this->fileStoragePathsForOps( $ops ) ) );
157 if ( !$status->isOK() ) {
158 return $status; // abort
160 // Actually attempt the operation batch on the master backend...
161 $masterStatus = $mbe->doOperations( $realOps, $opts );
162 $status->merge( $masterStatus );
163 // Propagate the operations to the clone backends if there were no fatal errors.
164 // If $ops only had one operation, this might avoid backend inconsistencies.
165 if ( !count( $masterStatus->getErrorsArray() ) ) {
166 foreach ( $this->backends
as $index => $backend ) {
167 if ( $index !== $this->masterIndex
) { // not done already
168 $realOps = $this->substOpBatchPaths( $ops, $backend );
169 $status->merge( $backend->doOperations( $realOps, $opts ) );
173 // Make 'success', 'successCount', and 'failCount' fields reflect
174 // the overall operation, rather than all the batches for each backend.
175 // Do this by only using success values from the master backend's batch.
176 $status->success
= $masterStatus->success
;
177 $status->successCount
= $masterStatus->successCount
;
178 $status->failCount
= $masterStatus->failCount
;
184 * Check that a set of files are consistent across all internal backends
186 * @param $paths Array List of storage paths
189 public function consistencyCheck( array $paths ) {
190 $status = Status
::newGood();
191 if ( $this->syncChecks
== 0 ||
count( $this->backends
) <= 1 ) {
192 return $status; // skip checks
195 $mBackend = $this->backends
[$this->masterIndex
];
196 foreach ( array_unique( $paths ) as $path ) {
197 $params = array( 'src' => $path, 'latest' => true );
198 $mParams = $this->substOpPaths( $params, $mBackend );
199 // Stat the file on the 'master' backend
200 $mStat = $mBackend->getFileStat( $mParams );
201 if ( $this->syncChecks
& self
::CHECK_SHA1
) {
202 $mSha1 = $mBackend->getFileSha1Base36( $mParams );
206 $mUsable = $mBackend->isPathUsableInternal( $mParams['src'] );
207 // Check of all clone backends agree with the master...
208 foreach ( $this->backends
as $index => $cBackend ) {
209 if ( $index === $this->masterIndex
) {
212 $cParams = $this->substOpPaths( $params, $cBackend );
213 $cStat = $cBackend->getFileStat( $cParams );
214 if ( $mStat ) { // file is in master
215 if ( !$cStat ) { // file should exist
216 $status->fatal( 'backend-fail-synced', $path );
219 if ( $this->syncChecks
& self
::CHECK_SIZE
) {
220 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
221 $status->fatal( 'backend-fail-synced', $path );
225 if ( $this->syncChecks
& self
::CHECK_TIME
) {
226 $mTs = wfTimestamp( TS_UNIX
, $mStat['mtime'] );
227 $cTs = wfTimestamp( TS_UNIX
, $cStat['mtime'] );
228 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
229 $status->fatal( 'backend-fail-synced', $path );
233 if ( $this->syncChecks
& self
::CHECK_SHA1
) {
234 if ( $cBackend->getFileSha1Base36( $cParams ) !== $mSha1 ) { // wrong SHA1
235 $status->fatal( 'backend-fail-synced', $path );
239 } else { // file is not in master
240 if ( $cStat ) { // file should not exist
241 $status->fatal( 'backend-fail-synced', $path );
244 if ( $mUsable !== $cBackend->isPathUsableInternal( $cParams['src'] ) ) {
245 $status->fatal( 'backend-fail-synced', $path );
254 * Check that a set of files are consistent across all internal backends
255 * and re-synchronize those files againt the "multi master" if needed.
257 * @param $paths Array List of storage paths
260 public function resyncFiles( array $paths ) {
261 $status = Status
::newGood();
263 $mBackend = $this->backends
[$this->masterIndex
];
264 foreach ( $paths as $path ) {
265 $mPath = $this->substPaths( $path, $mBackend );
266 $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) );
267 $mExist = $mBackend->fileExists( array( 'src' => $mPath ) );
268 // Check of all clone backends agree with the master...
269 foreach ( $this->backends
as $index => $cBackend ) {
270 if ( $index === $this->masterIndex
) {
273 $cPath = $this->substPaths( $path, $cBackend );
274 $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath ) );
275 if ( $mSha1 === $cSha1 ) {
276 // already synced; nothing to do
277 } elseif ( $mSha1 ) { // file is in master
278 $fsFile = $mBackend->getLocalReference( array( 'src' => $mPath ) );
279 $status->merge( $cBackend->quickStore(
280 array( 'src' => $fsFile->getPath(), 'dst' => $cPath )
282 } elseif ( $mExist === false ) { // file is not in master
283 $status->merge( $cBackend->quickDelete( array( 'src' => $cPath ) ) );
292 * Get a list of file storage paths to read or write for a list of operations
294 * @param $ops Array Same format as doOperations()
295 * @return Array List of storage paths to files (does not include directories)
297 protected function fileStoragePathsForOps( array $ops ) {
299 foreach ( $ops as $op ) {
300 if ( isset( $op['src'] ) ) {
301 $paths[] = $op['src'];
303 if ( isset( $op['srcs'] ) ) {
304 $paths = array_merge( $paths, $op['srcs'] );
306 if ( isset( $op['dst'] ) ) {
307 $paths[] = $op['dst'];
310 return array_unique( $paths );
314 * Substitute the backend name in storage path parameters
315 * for a set of operations with that of a given internal backend.
317 * @param $ops Array List of file operation arrays
318 * @param $backend FileBackendStore
321 protected function substOpBatchPaths( array $ops, FileBackendStore
$backend ) {
322 $newOps = array(); // operations
323 foreach ( $ops as $op ) {
324 $newOp = $op; // operation
325 foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
326 if ( isset( $newOp[$par] ) ) { // string or array
327 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
336 * Same as substOpBatchPaths() but for a single operation
338 * @param $ops array File operation array
339 * @param $backend FileBackendStore
342 protected function substOpPaths( array $ops, FileBackendStore
$backend ) {
343 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
348 * Substitute the backend of storage paths with an internal backend's name
350 * @param $paths Array|string List of paths or single string path
351 * @param $backend FileBackendStore
352 * @return Array|string
354 protected function substPaths( $paths, FileBackendStore
$backend ) {
356 '!^mwstore://' . preg_quote( $this->name
) . '/!',
357 StringUtils
::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
358 $paths // string or array
363 * Substitute the backend of internal storage paths with the proxy backend's name
365 * @param $paths Array|string List of paths or single string path
366 * @return Array|string
368 protected function unsubstPaths( $paths ) {
370 '!^mwstore://([^/]+)!',
371 StringUtils
::escapeRegexReplacement( "mwstore://{$this->name}" ),
372 $paths // string or array
377 * @see FileBackend::doQuickOperationsInternal()
380 protected function doQuickOperationsInternal( array $ops ) {
381 $status = Status
::newGood();
382 // Do the operations on the master backend; setting Status fields...
383 $realOps = $this->substOpBatchPaths( $ops, $this->backends
[$this->masterIndex
] );
384 $masterStatus = $this->backends
[$this->masterIndex
]->doQuickOperations( $realOps );
385 $status->merge( $masterStatus );
386 // Propagate the operations to the clone backends...
387 if ( !$this->noPushQuickOps
) {
388 foreach ( $this->backends
as $index => $backend ) {
389 if ( $index !== $this->masterIndex
) { // not done already
390 $realOps = $this->substOpBatchPaths( $ops, $backend );
391 $status->merge( $backend->doQuickOperations( $realOps ) );
395 // Make 'success', 'successCount', and 'failCount' fields reflect
396 // the overall operation, rather than all the batches for each backend.
397 // Do this by only using success values from the master backend's batch.
398 $status->success
= $masterStatus->success
;
399 $status->successCount
= $masterStatus->successCount
;
400 $status->failCount
= $masterStatus->failCount
;
405 * @param $path string Storage path
406 * @return bool Path container should have dir changes pushed to all backends
408 protected function replicateContainerDirChanges( $path ) {
409 list( $b, $shortCont, $r ) = self
::splitStoragePath( $path );
410 return !in_array( $shortCont, $this->noPushDirConts
);
414 * @see FileBackend::doPrepare()
417 protected function doPrepare( array $params ) {
418 $status = Status
::newGood();
419 $replicate = $this->replicateContainerDirChanges( $params['dir'] );
420 foreach ( $this->backends
as $index => $backend ) {
421 if ( $replicate ||
$index == $this->masterIndex
) {
422 $realParams = $this->substOpPaths( $params, $backend );
423 $status->merge( $backend->doPrepare( $realParams ) );
430 * @see FileBackend::doSecure()
431 * @param $params array
434 protected function doSecure( array $params ) {
435 $status = Status
::newGood();
436 $replicate = $this->replicateContainerDirChanges( $params['dir'] );
437 foreach ( $this->backends
as $index => $backend ) {
438 if ( $replicate ||
$index == $this->masterIndex
) {
439 $realParams = $this->substOpPaths( $params, $backend );
440 $status->merge( $backend->doSecure( $realParams ) );
447 * @see FileBackend::doPublish()
448 * @param $params array
451 protected function doPublish( array $params ) {
452 $status = Status
::newGood();
453 $replicate = $this->replicateContainerDirChanges( $params['dir'] );
454 foreach ( $this->backends
as $index => $backend ) {
455 if ( $replicate ||
$index == $this->masterIndex
) {
456 $realParams = $this->substOpPaths( $params, $backend );
457 $status->merge( $backend->doPublish( $realParams ) );
464 * @see FileBackend::doClean()
465 * @param $params array
468 protected function doClean( array $params ) {
469 $status = Status
::newGood();
470 $replicate = $this->replicateContainerDirChanges( $params['dir'] );
471 foreach ( $this->backends
as $index => $backend ) {
472 if ( $replicate ||
$index == $this->masterIndex
) {
473 $realParams = $this->substOpPaths( $params, $backend );
474 $status->merge( $backend->doClean( $realParams ) );
481 * @see FileBackend::concatenate()
482 * @param $params array
485 public function concatenate( array $params ) {
486 // We are writing to an FS file, so we don't need to do this per-backend
487 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
488 return $this->backends
[$this->masterIndex
]->concatenate( $realParams );
492 * @see FileBackend::fileExists()
493 * @param $params array
495 public function fileExists( array $params ) {
496 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
497 return $this->backends
[$this->masterIndex
]->fileExists( $realParams );
501 * @see FileBackend::getFileTimestamp()
502 * @param $params array
503 * @return bool|string
505 public function getFileTimestamp( array $params ) {
506 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
507 return $this->backends
[$this->masterIndex
]->getFileTimestamp( $realParams );
511 * @see FileBackend::getFileSize()
512 * @param $params array
515 public function getFileSize( array $params ) {
516 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
517 return $this->backends
[$this->masterIndex
]->getFileSize( $realParams );
521 * @see FileBackend::getFileStat()
522 * @param $params array
523 * @return Array|bool|null
525 public function getFileStat( array $params ) {
526 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
527 return $this->backends
[$this->masterIndex
]->getFileStat( $realParams );
531 * @see FileBackend::getFileContents()
532 * @param $params array
533 * @return bool|string
535 public function getFileContents( array $params ) {
536 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
537 return $this->backends
[$this->masterIndex
]->getFileContents( $realParams );
541 * @see FileBackend::getFileSha1Base36()
542 * @param $params array
543 * @return bool|string
545 public function getFileSha1Base36( array $params ) {
546 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
547 return $this->backends
[$this->masterIndex
]->getFileSha1Base36( $realParams );
551 * @see FileBackend::getFileProps()
552 * @param $params array
555 public function getFileProps( array $params ) {
556 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
557 return $this->backends
[$this->masterIndex
]->getFileProps( $realParams );
561 * @see FileBackend::streamFile()
562 * @param $params array
565 public function streamFile( array $params ) {
566 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
567 return $this->backends
[$this->masterIndex
]->streamFile( $realParams );
571 * @see FileBackend::getLocalReference()
572 * @param $params array
573 * @return FSFile|null
575 public function getLocalReference( array $params ) {
576 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
577 return $this->backends
[$this->masterIndex
]->getLocalReference( $realParams );
581 * @see FileBackend::getLocalCopy()
582 * @param $params array
583 * @return null|TempFSFile
585 public function getLocalCopy( array $params ) {
586 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
587 return $this->backends
[$this->masterIndex
]->getLocalCopy( $realParams );
591 * @see FileBackend::directoryExists()
592 * @param $params array
595 public function directoryExists( array $params ) {
596 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
597 return $this->backends
[$this->masterIndex
]->directoryExists( $realParams );
601 * @see FileBackend::getSubdirectoryList()
602 * @param $params array
603 * @return Array|null|Traversable
605 public function getDirectoryList( array $params ) {
606 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
607 return $this->backends
[$this->masterIndex
]->getDirectoryList( $realParams );
611 * @see FileBackend::getFileList()
612 * @param $params array
613 * @return Array|null|\Traversable
615 public function getFileList( array $params ) {
616 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
617 return $this->backends
[$this->masterIndex
]->getFileList( $realParams );
621 * @see FileBackend::clearCache()
623 public function clearCache( array $paths = null ) {
624 foreach ( $this->backends
as $backend ) {
625 $realPaths = is_array( $paths ) ?
$this->substPaths( $paths, $backend ) : null;
626 $backend->clearCache( $realPaths );
631 * @see FileBackend::getScopedLocksForOps()
633 public function getScopedLocksForOps( array $ops, Status
$status ) {
634 $fileOps = $this->backends
[$this->masterIndex
]->getOperationsInternal( $ops );
635 // Get the paths to lock from the master backend
636 $paths = $this->backends
[$this->masterIndex
]->getPathsToLockForOpsInternal( $fileOps );
637 // Get the paths under the proxy backend's name
638 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
639 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
641 $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status ),
642 $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status )