merging latest master
[lhc/web/wiklou.git] / includes / filebackend / FileBackendMultiWrite.php
1 <?php
2 /**
3 * Proxy backend that mirrors writes to several internal backends.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup FileBackend
22 * @author Aaron Schulz
23 */
24
25 /**
26 * @brief Proxy backend that mirrors writes to several internal backends.
27 *
28 * This class defines a multi-write backend. Multiple backends can be
29 * registered to this proxy backend and it will act as a single backend.
30 * Use this when all access to those backends is through this proxy backend.
31 * At least one of the backends must be declared the "master" backend.
32 *
33 * Only use this class when transitioning from one storage system to another.
34 *
35 * Read operations are only done on the 'master' backend for consistency.
36 * Write operations are performed on all backends, in the order defined.
37 * If an operation fails on one backend it will be rolled back from the others.
38 *
39 * @ingroup FileBackend
40 * @since 1.19
41 */
42 class FileBackendMultiWrite extends FileBackend {
43 /** @var Array Prioritized list of FileBackendStore objects */
44 protected $backends = array(); // array of (backend index => backends)
45 protected $masterIndex = -1; // integer; index of master backend
46 protected $syncChecks = 0; // integer bitfield
47
48 /* Possible internal backend consistency checks */
49 const CHECK_SIZE = 1;
50 const CHECK_TIME = 2;
51 const CHECK_SHA1 = 4;
52
53 /**
54 * Construct a proxy backend that consists of several internal backends.
55 * Locking, journaling, and read-only checks are handled by the proxy backend.
56 *
57 * Additional $config params include:
58 * - backends : Array of backend config and multi-backend settings.
59 * Each value is the config used in the constructor of a
60 * FileBackendStore class, but with these additional settings:
61 * - class : The name of the backend class
62 * - isMultiMaster : This must be set for one backend.
63 * - template: : If given a backend name, this will use
64 * the config of that backend as a template.
65 * Values specified here take precedence.
66 * - syncChecks : Integer bitfield of internal backend sync checks to perform.
67 * Possible bits include the FileBackendMultiWrite::CHECK_* constants.
68 * There are constants for SIZE, TIME, and SHA1.
69 * The checks are done before allowing any file operations.
70 * @param $config Array
71 * @throws MWException
72 */
73 public function __construct( array $config ) {
74 parent::__construct( $config );
75 $this->syncChecks = isset( $config['syncChecks'] )
76 ? $config['syncChecks']
77 : self::CHECK_SIZE;
78 // Construct backends here rather than via registration
79 // to keep these backends hidden from outside the proxy.
80 $namesUsed = array();
81 foreach ( $config['backends'] as $index => $config ) {
82 if ( isset( $config['template'] ) ) {
83 // Config is just a modified version of a registered backend's.
84 // This should only be used when that config is used only by this backend.
85 $config = $config + FileBackendGroup::singleton()->config( $config['template'] );
86 }
87 $name = $config['name'];
88 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
89 throw new MWException( "Two or more backends defined with the name $name." );
90 }
91 $namesUsed[$name] = 1;
92 // Alter certain sub-backend settings for sanity
93 unset( $config['readOnly'] ); // use proxy backend setting
94 unset( $config['fileJournal'] ); // use proxy backend journal
95 $config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID
96 $config['lockManager'] = 'nullLockManager'; // lock under proxy backend
97 if ( !empty( $config['isMultiMaster'] ) ) {
98 if ( $this->masterIndex >= 0 ) {
99 throw new MWException( 'More than one master backend defined.' );
100 }
101 $this->masterIndex = $index; // this is the "master"
102 $config['fileJournal'] = $this->fileJournal; // log under proxy backend
103 }
104 // Create sub-backend object
105 if ( !isset( $config['class'] ) ) {
106 throw new MWException( 'No class given for a backend config.' );
107 }
108 $class = $config['class'];
109 $this->backends[$index] = new $class( $config );
110 }
111 if ( $this->masterIndex < 0 ) { // need backends and must have a master
112 throw new MWException( 'No master backend defined.' );
113 }
114 }
115
116 /**
117 * @see FileBackend::doOperationsInternal()
118 * @return Status
119 */
120 final protected function doOperationsInternal( array $ops, array $opts ) {
121 $status = Status::newGood();
122
123 $mbe = $this->backends[$this->masterIndex]; // convenience
124
125 // Get the paths to lock from the master backend
126 $realOps = $this->substOpBatchPaths( $ops, $mbe );
127 $paths = $mbe->getPathsToLockForOpsInternal( $mbe->getOperationsInternal( $realOps ) );
128 // Get the paths under the proxy backend's name
129 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
130 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
131 // Try to lock those files for the scope of this function...
132 if ( empty( $opts['nonLocking'] ) ) {
133 // Try to lock those files for the scope of this function...
134 $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
135 $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
136 if ( !$status->isOK() ) {
137 return $status; // abort
138 }
139 }
140 // Clear any cache entries (after locks acquired)
141 $this->clearCache();
142 // Do a consistency check to see if the backends agree
143 $status->merge( $this->consistencyCheck( $this->fileStoragePathsForOps( $ops ) ) );
144 if ( !$status->isOK() ) {
145 return $status; // abort
146 }
147 // Actually attempt the operation batch on the master backend...
148 $masterStatus = $mbe->doOperations( $realOps, $opts );
149 $status->merge( $masterStatus );
150 // Propagate the operations to the clone backends...
151 foreach ( $this->backends as $index => $backend ) {
152 if ( $index !== $this->masterIndex ) { // not done already
153 $realOps = $this->substOpBatchPaths( $ops, $backend );
154 $status->merge( $backend->doOperations( $realOps, $opts ) );
155 }
156 }
157 // Make 'success', 'successCount', and 'failCount' fields reflect
158 // the overall operation, rather than all the batches for each backend.
159 // Do this by only using success values from the master backend's batch.
160 $status->success = $masterStatus->success;
161 $status->successCount = $masterStatus->successCount;
162 $status->failCount = $masterStatus->failCount;
163
164 return $status;
165 }
166
167 /**
168 * Check that a set of files are consistent across all internal backends
169 *
170 * @param $paths Array List of storage paths
171 * @return Status
172 */
173 public function consistencyCheck( array $paths ) {
174 $status = Status::newGood();
175 if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) {
176 return $status; // skip checks
177 }
178
179 $mBackend = $this->backends[$this->masterIndex];
180 foreach ( array_unique( $paths ) as $path ) {
181 $params = array( 'src' => $path, 'latest' => true );
182 $mParams = $this->substOpPaths( $params, $mBackend );
183 // Stat the file on the 'master' backend
184 $mStat = $mBackend->getFileStat( $mParams );
185 if ( $this->syncChecks & self::CHECK_SHA1 ) {
186 $mSha1 = $mBackend->getFileSha1Base36( $mParams );
187 } else {
188 $mSha1 = false;
189 }
190 $mUsable = $mBackend->isPathUsableInternal( $mParams['src'] );
191 // Check of all clone backends agree with the master...
192 foreach ( $this->backends as $index => $cBackend ) {
193 if ( $index === $this->masterIndex ) {
194 continue; // master
195 }
196 $cParams = $this->substOpPaths( $params, $cBackend );
197 $cStat = $cBackend->getFileStat( $cParams );
198 if ( $mStat ) { // file is in master
199 if ( !$cStat ) { // file should exist
200 $status->fatal( 'backend-fail-synced', $path );
201 continue;
202 }
203 if ( $this->syncChecks & self::CHECK_SIZE ) {
204 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
205 $status->fatal( 'backend-fail-synced', $path );
206 continue;
207 }
208 }
209 if ( $this->syncChecks & self::CHECK_TIME ) {
210 $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
211 $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
212 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
213 $status->fatal( 'backend-fail-synced', $path );
214 continue;
215 }
216 }
217 if ( $this->syncChecks & self::CHECK_SHA1 ) {
218 if ( $cBackend->getFileSha1Base36( $cParams ) !== $mSha1 ) { // wrong SHA1
219 $status->fatal( 'backend-fail-synced', $path );
220 continue;
221 }
222 }
223 } else { // file is not in master
224 if ( $cStat ) { // file should not exist
225 $status->fatal( 'backend-fail-synced', $path );
226 }
227 }
228 if ( $mUsable !== $cBackend->isPathUsableInternal( $cParams['src'] ) ) {
229 $status->fatal( 'backend-fail-synced', $path );
230 }
231 }
232 }
233
234 return $status;
235 }
236
237 /**
238 * Check that a set of files are consistent across all internal backends
239 * and re-synchronize those files againt the "multi master" if needed.
240 *
241 * @param $paths Array List of storage paths
242 * @return Status
243 */
244 public function resyncFiles( array $paths ) {
245 $status = Status::newGood();
246
247 $mBackend = $this->backends[$this->masterIndex];
248 foreach ( $paths as $path ) {
249 $mPath = $this->substPaths( $path, $mBackend );
250 $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) );
251 $mExist = $mBackend->fileExists( array( 'src' => $mPath ) );
252 // Check of all clone backends agree with the master...
253 foreach ( $this->backends as $index => $cBackend ) {
254 if ( $index === $this->masterIndex ) {
255 continue; // master
256 }
257 $cPath = $this->substPaths( $path, $cBackend );
258 $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath ) );
259 if ( $mSha1 === $cSha1 ) {
260 // already synced; nothing to do
261 } elseif ( $mSha1 ) { // file is in master
262 $fsFile = $mBackend->getLocalReference( array( 'src' => $mPath ) );
263 $status->merge( $cBackend->quickStore(
264 array( 'src' => $fsFile->getPath(), 'dst' => $cPath )
265 ) );
266 } elseif ( $mExist === false ) { // file is not in master
267 $status->merge( $cBackend->quickDelete( array( 'src' => $cPath ) ) );
268 }
269 }
270 }
271
272 return $status;
273 }
274
275 /**
276 * Get a list of file storage paths to read or write for a list of operations
277 *
278 * @param $ops Array Same format as doOperations()
279 * @return Array List of storage paths to files (does not include directories)
280 */
281 protected function fileStoragePathsForOps( array $ops ) {
282 $paths = array();
283 foreach ( $ops as $op ) {
284 if ( isset( $op['src'] ) ) {
285 $paths[] = $op['src'];
286 }
287 if ( isset( $op['srcs'] ) ) {
288 $paths = array_merge( $paths, $op['srcs'] );
289 }
290 if ( isset( $op['dst'] ) ) {
291 $paths[] = $op['dst'];
292 }
293 }
294 return array_unique( $paths );
295 }
296
297 /**
298 * Substitute the backend name in storage path parameters
299 * for a set of operations with that of a given internal backend.
300 *
301 * @param $ops Array List of file operation arrays
302 * @param $backend FileBackendStore
303 * @return Array
304 */
305 protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
306 $newOps = array(); // operations
307 foreach ( $ops as $op ) {
308 $newOp = $op; // operation
309 foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
310 if ( isset( $newOp[$par] ) ) { // string or array
311 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
312 }
313 }
314 $newOps[] = $newOp;
315 }
316 return $newOps;
317 }
318
319 /**
320 * Same as substOpBatchPaths() but for a single operation
321 *
322 * @param $ops array File operation array
323 * @param $backend FileBackendStore
324 * @return Array
325 */
326 protected function substOpPaths( array $ops, FileBackendStore $backend ) {
327 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
328 return $newOps[0];
329 }
330
331 /**
332 * Substitute the backend of storage paths with an internal backend's name
333 *
334 * @param $paths Array|string List of paths or single string path
335 * @param $backend FileBackendStore
336 * @return Array|string
337 */
338 protected function substPaths( $paths, FileBackendStore $backend ) {
339 return preg_replace(
340 '!^mwstore://' . preg_quote( $this->name ) . '/!',
341 StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
342 $paths // string or array
343 );
344 }
345
346 /**
347 * Substitute the backend of internal storage paths with the proxy backend's name
348 *
349 * @param $paths Array|string List of paths or single string path
350 * @return Array|string
351 */
352 protected function unsubstPaths( $paths ) {
353 return preg_replace(
354 '!^mwstore://([^/]+)!',
355 StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ),
356 $paths // string or array
357 );
358 }
359
360 /**
361 * @see FileBackend::doQuickOperationsInternal()
362 * @return Status
363 */
364 protected function doQuickOperationsInternal( array $ops ) {
365 $status = Status::newGood();
366 // Do the operations on the master backend; setting Status fields...
367 $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
368 $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
369 $status->merge( $masterStatus );
370 // Propagate the operations to the clone backends...
371 foreach ( $this->backends as $index => $backend ) {
372 if ( $index !== $this->masterIndex ) { // not done already
373 $realOps = $this->substOpBatchPaths( $ops, $backend );
374 $status->merge( $backend->doQuickOperations( $realOps ) );
375 }
376 }
377 // Make 'success', 'successCount', and 'failCount' fields reflect
378 // the overall operation, rather than all the batches for each backend.
379 // Do this by only using success values from the master backend's batch.
380 $status->success = $masterStatus->success;
381 $status->successCount = $masterStatus->successCount;
382 $status->failCount = $masterStatus->failCount;
383 return $status;
384 }
385
386 /**
387 * @see FileBackend::doPrepare()
388 * @return Status
389 */
390 protected function doPrepare( array $params ) {
391 $status = Status::newGood();
392 foreach ( $this->backends as $backend ) {
393 $realParams = $this->substOpPaths( $params, $backend );
394 $status->merge( $backend->doPrepare( $realParams ) );
395 }
396 return $status;
397 }
398
399 /**
400 * @see FileBackend::doSecure()
401 * @param $params array
402 * @return Status
403 */
404 protected function doSecure( array $params ) {
405 $status = Status::newGood();
406 foreach ( $this->backends as $backend ) {
407 $realParams = $this->substOpPaths( $params, $backend );
408 $status->merge( $backend->doSecure( $realParams ) );
409 }
410 return $status;
411 }
412
413 /**
414 * @see FileBackend::doPublish()
415 * @param $params array
416 * @return Status
417 */
418 protected function doPublish( array $params ) {
419 $status = Status::newGood();
420 foreach ( $this->backends as $backend ) {
421 $realParams = $this->substOpPaths( $params, $backend );
422 $status->merge( $backend->doPublish( $realParams ) );
423 }
424 return $status;
425 }
426
427 /**
428 * @see FileBackend::doClean()
429 * @param $params array
430 * @return Status
431 */
432 protected function doClean( array $params ) {
433 $status = Status::newGood();
434 foreach ( $this->backends as $backend ) {
435 $realParams = $this->substOpPaths( $params, $backend );
436 $status->merge( $backend->doClean( $realParams ) );
437 }
438 return $status;
439 }
440
441 /**
442 * @see FileBackend::concatenate()
443 * @param $params array
444 * @return Status
445 */
446 public function concatenate( array $params ) {
447 // We are writing to an FS file, so we don't need to do this per-backend
448 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
449 return $this->backends[$this->masterIndex]->concatenate( $realParams );
450 }
451
452 /**
453 * @see FileBackend::fileExists()
454 * @param $params array
455 */
456 public function fileExists( array $params ) {
457 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
458 return $this->backends[$this->masterIndex]->fileExists( $realParams );
459 }
460
461 /**
462 * @see FileBackend::getFileTimestamp()
463 * @param $params array
464 * @return bool|string
465 */
466 public function getFileTimestamp( array $params ) {
467 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
468 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
469 }
470
471 /**
472 * @see FileBackend::getFileSize()
473 * @param $params array
474 * @return bool|int
475 */
476 public function getFileSize( array $params ) {
477 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
478 return $this->backends[$this->masterIndex]->getFileSize( $realParams );
479 }
480
481 /**
482 * @see FileBackend::getFileStat()
483 * @param $params array
484 * @return Array|bool|null
485 */
486 public function getFileStat( array $params ) {
487 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
488 return $this->backends[$this->masterIndex]->getFileStat( $realParams );
489 }
490
491 /**
492 * @see FileBackend::getFileContents()
493 * @param $params array
494 * @return bool|string
495 */
496 public function getFileContents( array $params ) {
497 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
498 return $this->backends[$this->masterIndex]->getFileContents( $realParams );
499 }
500
501 /**
502 * @see FileBackend::getFileSha1Base36()
503 * @param $params array
504 * @return bool|string
505 */
506 public function getFileSha1Base36( array $params ) {
507 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
508 return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
509 }
510
511 /**
512 * @see FileBackend::getFileProps()
513 * @param $params array
514 * @return Array
515 */
516 public function getFileProps( array $params ) {
517 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
518 return $this->backends[$this->masterIndex]->getFileProps( $realParams );
519 }
520
521 /**
522 * @see FileBackend::streamFile()
523 * @param $params array
524 * @return \Status
525 */
526 public function streamFile( array $params ) {
527 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
528 return $this->backends[$this->masterIndex]->streamFile( $realParams );
529 }
530
531 /**
532 * @see FileBackend::getLocalReference()
533 * @param $params array
534 * @return FSFile|null
535 */
536 public function getLocalReference( array $params ) {
537 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
538 return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
539 }
540
541 /**
542 * @see FileBackend::getLocalCopy()
543 * @param $params array
544 * @return null|TempFSFile
545 */
546 public function getLocalCopy( array $params ) {
547 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
548 return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
549 }
550
551 /**
552 * @see FileBackend::directoryExists()
553 * @param $params array
554 * @return bool|null
555 */
556 public function directoryExists( array $params ) {
557 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
558 return $this->backends[$this->masterIndex]->directoryExists( $realParams );
559 }
560
561 /**
562 * @see FileBackend::getSubdirectoryList()
563 * @param $params array
564 * @return Array|null|Traversable
565 */
566 public function getDirectoryList( array $params ) {
567 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
568 return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
569 }
570
571 /**
572 * @see FileBackend::getFileList()
573 * @param $params array
574 * @return Array|null|\Traversable
575 */
576 public function getFileList( array $params ) {
577 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
578 return $this->backends[$this->masterIndex]->getFileList( $realParams );
579 }
580
581 /**
582 * @see FileBackend::clearCache()
583 */
584 public function clearCache( array $paths = null ) {
585 foreach ( $this->backends as $backend ) {
586 $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
587 $backend->clearCache( $realPaths );
588 }
589 }
590
591 /**
592 * @see FileBackend::getScopedLocksForOps()
593 */
594 public function getScopedLocksForOps( array $ops, Status $status ) {
595 $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $ops );
596 // Get the paths to lock from the master backend
597 $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
598 // Get the paths under the proxy backend's name
599 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
600 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
601 return array(
602 $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
603 $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
604 );
605 }
606 }