e9136d5af88ba325aca5687d531e9fef831e60c5
3 * Proxy backend that mirrors writes to several internal backends.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
22 * @author Aaron Schulz
26 * @brief Proxy backend that mirrors writes to several internal backends.
28 * This class defines a multi-write backend. Multiple backends can be
29 * registered to this proxy backend and it will act as a single backend.
30 * Use this when all access to those backends is through this proxy backend.
31 * At least one of the backends must be declared the "master" backend.
33 * Only use this class when transitioning from one storage system to another.
35 * Read operations are only done on the 'master' backend for consistency.
36 * Write operations are performed on all backends, in the order defined.
37 * If an operation fails on one backend it will be rolled back from the others.
39 * @ingroup FileBackend
42 class FileBackendMultiWrite
extends FileBackend
{
43 /** @var Array Prioritized list of FileBackendStore objects */
44 protected $backends = array(); // array of (backend index => backends)
45 protected $masterIndex = -1; // integer; index of master backend
46 protected $syncChecks = 0; // integer bitfield
48 /* Possible internal backend consistency checks */
54 * Construct a proxy backend that consists of several internal backends.
55 * Locking, journaling, and read-only checks are handled by the proxy backend.
57 * Additional $config params include:
58 * - backends : Array of backend config and multi-backend settings.
59 * Each value is the config used in the constructor of a
60 * FileBackendStore class, but with these additional settings:
61 * - class : The name of the backend class
62 * - isMultiMaster : This must be set for one backend.
63 * - template: : If given a backend name, this will use
64 * the config of that backend as a template.
65 * Values specified here take precedence.
66 * - syncChecks : Integer bitfield of internal backend sync checks to perform.
67 * Possible bits include the FileBackendMultiWrite::CHECK_* constants.
68 * There are constants for SIZE, TIME, and SHA1.
69 * The checks are done before allowing any file operations.
70 * @param $config Array
73 public function __construct( array $config ) {
74 parent
::__construct( $config );
76 // Construct backends here rather than via registration
77 // to keep these backends hidden from outside the proxy.
78 foreach ( $config['backends'] as $index => $config ) {
79 if ( isset( $config['template'] ) ) {
80 // Config is just a modified version of a registered backend's.
81 // This should only be used when that config is used only by this backend.
82 $config = $config + FileBackendGroup
::singleton()->config( $config['template'] );
84 $name = $config['name'];
85 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
86 throw new MWException( "Two or more backends defined with the name $name." );
88 $namesUsed[$name] = 1;
89 // Alter certain sub-backend settings for sanity
90 unset( $config['readOnly'] ); // use proxy backend setting
91 unset( $config['fileJournal'] ); // use proxy backend journal
92 $config['wikiId'] = $this->wikiId
; // use the proxy backend wiki ID
93 $config['lockManager'] = 'nullLockManager'; // lock under proxy backend
94 if ( !empty( $config['isMultiMaster'] ) ) {
95 if ( $this->masterIndex
>= 0 ) {
96 throw new MWException( 'More than one master backend defined.' );
98 $this->masterIndex
= $index; // this is the "master"
99 $config['fileJournal'] = $this->fileJournal
; // log under proxy backend
101 // Create sub-backend object
102 if ( !isset( $config['class'] ) ) {
103 throw new MWException( 'No class given for a backend config.' );
105 $class = $config['class'];
106 $this->backends
[$index] = new $class( $config );
108 if ( $this->masterIndex
< 0 ) { // need backends and must have a master
109 throw new MWException( 'No master backend defined.' );
111 $this->syncChecks
= isset( $config['syncChecks'] )
112 ?
$config['syncChecks']
117 * @see FileBackend::doOperationsInternal()
120 final protected function doOperationsInternal( array $ops, array $opts ) {
121 $status = Status
::newGood();
123 $mbe = $this->backends
[$this->masterIndex
]; // convenience
125 // Get the paths to lock from the master backend
126 $realOps = $this->substOpBatchPaths( $ops, $mbe );
127 $paths = $mbe->getPathsToLockForOpsInternal( $mbe->getOperationsInternal( $realOps ) );
128 // Get the paths under the proxy backend's name
129 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
130 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
131 // Try to lock those files for the scope of this function...
132 if ( empty( $opts['nonLocking'] ) ) {
133 // Try to lock those files for the scope of this function...
134 $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status );
135 $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status );
136 if ( !$status->isOK() ) {
137 return $status; // abort
140 // Clear any cache entries (after locks acquired)
142 // Do a consistency check to see if the backends agree
143 $status->merge( $this->consistencyCheck( $this->fileStoragePathsForOps( $ops ) ) );
144 if ( !$status->isOK() ) {
145 return $status; // abort
147 // Actually attempt the operation batch on the master backend...
148 $masterStatus = $mbe->doOperations( $realOps, $opts );
149 $status->merge( $masterStatus );
150 // Propagate the operations to the clone backends...
151 foreach ( $this->backends
as $index => $backend ) {
152 if ( $index !== $this->masterIndex
) { // not done already
153 $realOps = $this->substOpBatchPaths( $ops, $backend );
154 $status->merge( $backend->doOperations( $realOps, $opts ) );
157 // Make 'success', 'successCount', and 'failCount' fields reflect
158 // the overall operation, rather than all the batches for each backend.
159 // Do this by only using success values from the master backend's batch.
160 $status->success
= $masterStatus->success
;
161 $status->successCount
= $masterStatus->successCount
;
162 $status->failCount
= $masterStatus->failCount
;
168 * Check that a set of files are consistent across all internal backends
170 * @param $paths Array
173 public function consistencyCheck( array $paths ) {
174 $status = Status
::newGood();
175 if ( $this->syncChecks
== 0 ||
count( $this->backends
) <= 1 ) {
176 return $status; // skip checks
179 $mBackend = $this->backends
[$this->masterIndex
];
180 foreach ( array_unique( $paths ) as $path ) {
181 $params = array( 'src' => $path, 'latest' => true );
182 $mParams = $this->substOpPaths( $params, $mBackend );
183 // Stat the file on the 'master' backend
184 $mStat = $mBackend->getFileStat( $mParams );
185 if ( $this->syncChecks
& self
::CHECK_SHA1
) {
186 $mSha1 = $mBackend->getFileSha1( $mParams );
190 $mUsable = $mBackend->isPathUsableInternal( $mParams['src'] );
191 // Check of all clone backends agree with the master...
192 foreach ( $this->backends
as $index => $cBackend ) {
193 if ( $index === $this->masterIndex
) {
196 $cParams = $this->substOpPaths( $params, $cBackend );
197 $cStat = $cBackend->getFileStat( $cParams );
198 if ( $mStat ) { // file is in master
199 if ( !$cStat ) { // file should exist
200 $status->fatal( 'backend-fail-synced', $path );
203 if ( $this->syncChecks
& self
::CHECK_SIZE
) {
204 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
205 $status->fatal( 'backend-fail-synced', $path );
209 if ( $this->syncChecks
& self
::CHECK_TIME
) {
210 $mTs = wfTimestamp( TS_UNIX
, $mStat['mtime'] );
211 $cTs = wfTimestamp( TS_UNIX
, $cStat['mtime'] );
212 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
213 $status->fatal( 'backend-fail-synced', $path );
217 if ( $this->syncChecks
& self
::CHECK_SHA1
) {
218 if ( $cBackend->getFileSha1( $cParams ) !== $mSha1 ) { // wrong SHA1
219 $status->fatal( 'backend-fail-synced', $path );
223 } else { // file is not in master
224 if ( $cStat ) { // file should not exist
225 $status->fatal( 'backend-fail-synced', $path );
228 if ( $mUsable !== $cBackend->isPathUsableInternal( $cParams['src'] ) ) {
229 $status->fatal( 'backend-fail-synced', $path );
238 * Get a list of file storage paths to read or write for a list of operations
240 * @param $ops Array Same format as doOperations()
241 * @return Array List of storage paths to files (does not include directories)
243 protected function fileStoragePathsForOps( array $ops ) {
245 foreach ( $ops as $op ) {
246 if ( isset( $op['src'] ) ) {
247 $paths[] = $op['src'];
249 if ( isset( $op['srcs'] ) ) {
250 $paths = array_merge( $paths, $op['srcs'] );
252 if ( isset( $op['dst'] ) ) {
253 $paths[] = $op['dst'];
256 return array_unique( $paths );
260 * Substitute the backend name in storage path parameters
261 * for a set of operations with that of a given internal backend.
263 * @param $ops Array List of file operation arrays
264 * @param $backend FileBackendStore
267 protected function substOpBatchPaths( array $ops, FileBackendStore
$backend ) {
268 $newOps = array(); // operations
269 foreach ( $ops as $op ) {
270 $newOp = $op; // operation
271 foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
272 if ( isset( $newOp[$par] ) ) { // string or array
273 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
282 * Same as substOpBatchPaths() but for a single operation
284 * @param $ops array File operation array
285 * @param $backend FileBackendStore
288 protected function substOpPaths( array $ops, FileBackendStore
$backend ) {
289 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
294 * Substitute the backend of storage paths with an internal backend's name
296 * @param $paths Array|string List of paths or single string path
297 * @param $backend FileBackendStore
298 * @return Array|string
300 protected function substPaths( $paths, FileBackendStore
$backend ) {
302 '!^mwstore://' . preg_quote( $this->name
) . '/!',
303 StringUtils
::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
304 $paths // string or array
309 * Substitute the backend of internal storage paths with the proxy backend's name
311 * @param $paths Array|string List of paths or single string path
312 * @return Array|string
314 protected function unsubstPaths( $paths ) {
316 '!^mwstore://([^/]+)!',
317 StringUtils
::escapeRegexReplacement( "mwstore://{$this->name}" ),
318 $paths // string or array
323 * @see FileBackend::doQuickOperationsInternal()
326 protected function doQuickOperationsInternal( array $ops ) {
327 $status = Status
::newGood();
328 // Do the operations on the master backend; setting Status fields...
329 $realOps = $this->substOpBatchPaths( $ops, $this->backends
[$this->masterIndex
] );
330 $masterStatus = $this->backends
[$this->masterIndex
]->doQuickOperations( $realOps );
331 $status->merge( $masterStatus );
332 // Propagate the operations to the clone backends...
333 foreach ( $this->backends
as $index => $backend ) {
334 if ( $index !== $this->masterIndex
) { // not done already
335 $realOps = $this->substOpBatchPaths( $ops, $backend );
336 $status->merge( $backend->doQuickOperations( $realOps ) );
339 // Make 'success', 'successCount', and 'failCount' fields reflect
340 // the overall operation, rather than all the batches for each backend.
341 // Do this by only using success values from the master backend's batch.
342 $status->success
= $masterStatus->success
;
343 $status->successCount
= $masterStatus->successCount
;
344 $status->failCount
= $masterStatus->failCount
;
349 * @see FileBackend::doPrepare()
352 protected function doPrepare( array $params ) {
353 $status = Status
::newGood();
354 foreach ( $this->backends
as $backend ) {
355 $realParams = $this->substOpPaths( $params, $backend );
356 $status->merge( $backend->doPrepare( $realParams ) );
362 * @see FileBackend::doSecure()
363 * @param $params array
366 protected function doSecure( array $params ) {
367 $status = Status
::newGood();
368 foreach ( $this->backends
as $backend ) {
369 $realParams = $this->substOpPaths( $params, $backend );
370 $status->merge( $backend->doSecure( $realParams ) );
376 * @see FileBackend::doPublish()
377 * @param $params array
380 protected function doPublish( array $params ) {
381 $status = Status
::newGood();
382 foreach ( $this->backends
as $backend ) {
383 $realParams = $this->substOpPaths( $params, $backend );
384 $status->merge( $backend->doPublish( $realParams ) );
390 * @see FileBackend::doClean()
391 * @param $params array
394 protected function doClean( array $params ) {
395 $status = Status
::newGood();
396 foreach ( $this->backends
as $backend ) {
397 $realParams = $this->substOpPaths( $params, $backend );
398 $status->merge( $backend->doClean( $realParams ) );
404 * @see FileBackend::concatenate()
405 * @param $params array
408 public function concatenate( array $params ) {
409 // We are writing to an FS file, so we don't need to do this per-backend
410 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
411 return $this->backends
[$this->masterIndex
]->concatenate( $realParams );
415 * @see FileBackend::fileExists()
416 * @param $params array
418 public function fileExists( array $params ) {
419 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
420 return $this->backends
[$this->masterIndex
]->fileExists( $realParams );
424 * @see FileBackend::getFileTimestamp()
425 * @param $params array
426 * @return bool|string
428 public function getFileTimestamp( array $params ) {
429 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
430 return $this->backends
[$this->masterIndex
]->getFileTimestamp( $realParams );
434 * @see FileBackend::getFileSize()
435 * @param $params array
438 public function getFileSize( array $params ) {
439 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
440 return $this->backends
[$this->masterIndex
]->getFileSize( $realParams );
444 * @see FileBackend::getFileStat()
445 * @param $params array
446 * @return Array|bool|null
448 public function getFileStat( array $params ) {
449 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
450 return $this->backends
[$this->masterIndex
]->getFileStat( $realParams );
454 * @see FileBackend::getFileContents()
455 * @param $params array
456 * @return bool|string
458 public function getFileContents( array $params ) {
459 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
460 return $this->backends
[$this->masterIndex
]->getFileContents( $realParams );
464 * @see FileBackend::getFileSha1Base36()
465 * @param $params array
466 * @return bool|string
468 public function getFileSha1Base36( array $params ) {
469 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
470 return $this->backends
[$this->masterIndex
]->getFileSha1Base36( $realParams );
474 * @see FileBackend::getFileProps()
475 * @param $params array
478 public function getFileProps( array $params ) {
479 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
480 return $this->backends
[$this->masterIndex
]->getFileProps( $realParams );
484 * @see FileBackend::streamFile()
485 * @param $params array
488 public function streamFile( array $params ) {
489 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
490 return $this->backends
[$this->masterIndex
]->streamFile( $realParams );
494 * @see FileBackend::getLocalReference()
495 * @param $params array
496 * @return FSFile|null
498 public function getLocalReference( array $params ) {
499 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
500 return $this->backends
[$this->masterIndex
]->getLocalReference( $realParams );
504 * @see FileBackend::getLocalCopy()
505 * @param $params array
506 * @return null|TempFSFile
508 public function getLocalCopy( array $params ) {
509 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
510 return $this->backends
[$this->masterIndex
]->getLocalCopy( $realParams );
514 * @see FileBackend::directoryExists()
515 * @param $params array
518 public function directoryExists( array $params ) {
519 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
520 return $this->backends
[$this->masterIndex
]->directoryExists( $realParams );
524 * @see FileBackend::getSubdirectoryList()
525 * @param $params array
526 * @return Array|null|Traversable
528 public function getDirectoryList( array $params ) {
529 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
530 return $this->backends
[$this->masterIndex
]->getDirectoryList( $realParams );
534 * @see FileBackend::getFileList()
535 * @param $params array
536 * @return Array|null|\Traversable
538 public function getFileList( array $params ) {
539 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
540 return $this->backends
[$this->masterIndex
]->getFileList( $realParams );
544 * @see FileBackend::clearCache()
546 public function clearCache( array $paths = null ) {
547 foreach ( $this->backends
as $backend ) {
548 $realPaths = is_array( $paths ) ?
$this->substPaths( $paths, $backend ) : null;
549 $backend->clearCache( $realPaths );
554 * @see FileBackend::getScopedLocksForOps()
556 public function getScopedLocksForOps( array $ops, Status
$status ) {
557 $fileOps = $this->backends
[$this->masterIndex
]->getOperationsInternal( $ops );
558 // Get the paths to lock from the master backend
559 $paths = $this->backends
[$this->masterIndex
]->getPathsToLockForOpsInternal( $fileOps );
560 // Get the paths under the proxy backend's name
561 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
562 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
564 $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status ),
565 $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status )