9 * This class defines a multi-write backend. Multiple backends can be
10 * registered to this proxy backend and it will act as a single backend.
11 * Use this when all access to those backends is through this proxy backend.
12 * At least one of the backends must be declared the "master" backend.
14 * Only use this class when transitioning from one storage system to another.
16 * Read operations are only done on the 'master' backend for consistency.
17 * Write operations are performed on all backends, in the order defined.
18 * If an operation fails on one backend it will be rolled back from the others.
20 * @ingroup FileBackend
23 class FileBackendMultiWrite
extends FileBackendBase
{
24 /** @var Array Prioritized list of FileBackend objects */
25 protected $backends = array(); // array of (backend index => backends)
26 protected $masterIndex = -1; // index of master backend
29 * Construct a proxy backend that consists of several internal backends.
30 * Additional $config params include:
31 * 'backends' : Array of backend config and multi-backend settings.
32 * Each value is the config used in the constructor of a
33 * FileBackend class, but with these additional settings:
34 * 'class' : The name of the backend class
35 * 'isMultiMaster' : This must be set for one backend.
36 * @param $config Array
38 public function __construct( array $config ) {
39 parent
::__construct( $config );
41 // Construct backends here rather than via registration
42 // to keep these backends hidden from outside the proxy.
43 foreach ( $config['backends'] as $index => $config ) {
44 $name = $config['name'];
45 if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
46 throw new MWException( "Two or more backends defined with the name $name." );
48 $namesUsed[$name] = 1;
49 if ( !isset( $config['class'] ) ) {
50 throw new MWException( 'No class given for a backend config.' );
52 $class = $config['class'];
53 $this->backends
[$index] = new $class( $config );
54 if ( !empty( $config['isMultiMaster'] ) ) {
55 if ( $this->masterIndex
>= 0 ) {
56 throw new MWException( 'More than one master backend defined.' );
58 $this->masterIndex
= $index;
61 if ( $this->masterIndex
< 0 ) { // need backends and must have a master
62 throw new MWException( 'No master backend defined.' );
67 * @see FileBackendBase::doOperationsInternal()
69 final protected function doOperationsInternal( array $ops, array $opts ) {
70 $status = Status
::newGood();
72 $performOps = array(); // list of FileOp objects
73 $filesRead = $filesChanged = array(); // storage paths used
74 // Build up a list of FileOps. The list will have all the ops
75 // for one backend, then all the ops for the next, and so on.
76 // These batches of ops are all part of a continuous array.
77 // Also build up a list of files read/changed...
78 foreach ( $this->backends
as $index => $backend ) {
79 $backendOps = $this->substOpBatchPaths( $ops, $backend );
80 // Add on the operation batch for this backend
81 $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
82 if ( $index == 0 ) { // first batch
83 // Get the files used for these operations. Each backend has a batch of
84 // the same operations, so we only need to get them from the first batch.
85 foreach ( $performOps as $fileOp ) {
86 $filesRead = array_merge( $filesRead, $fileOp->storagePathsRead() );
87 $filesChanged = array_merge( $filesChanged, $fileOp->storagePathsChanged() );
89 // Get the paths under the proxy backend's name
90 $filesRead = $this->unsubstPaths( $filesRead );
91 $filesChanged = $this->unsubstPaths( $filesChanged );
95 // Try to lock those files for the scope of this function...
96 if ( empty( $opts['nonLocking'] ) ) {
97 $filesLockSh = array_diff( $filesRead, $filesChanged ); // optimization
98 $filesLockEx = $filesChanged;
99 $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager
::LOCK_UW
, $status );
100 $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager
::LOCK_EX
, $status );
101 if ( !$status->isOK() ) {
102 return $status; // abort
106 // Clear any cache entries (after locks acquired)
109 // Do a consistency check to see if the backends agree
110 if ( count( $this->backends
) > 1 ) {
111 $status->merge( $this->consistencyCheck( array_merge( $filesRead, $filesChanged ) ) );
112 if ( !$status->isOK() ) {
113 return $status; // abort
117 // Actually attempt the operation batch...
118 $subStatus = FileOp
::attemptBatch( $performOps, $opts );
121 $failCount = $successCount = 0;
122 // Make 'success', 'successCount', and 'failCount' fields reflect
123 // the overall operation, rather than all the batches for each backend.
124 // Do this by only using success values from the master backend's batch.
125 $batchStart = $this->masterIndex
* count( $ops );
126 $batchEnd = $batchStart +
count( $ops ) - 1;
127 for ( $i = $batchStart; $i <= $batchEnd; $i++
) {
128 if ( !isset( $subStatus->success
[$i] ) ) {
129 break; // failed out before trying this op
130 } elseif ( $subStatus->success
[$i] ) {
135 $success[] = $subStatus->success
[$i];
137 $subStatus->success
= $success;
138 $subStatus->successCount
= $successCount;
139 $subStatus->failCount
= $failCount;
141 // Merge errors into status fields
142 $status->merge( $subStatus );
143 $status->success
= $subStatus->success
; // not done in merge()
149 * Check that a set of files are consistent across all internal backends
151 * @param $paths Array
154 public function consistencyCheck( array $paths ) {
155 $status = Status
::newGood();
157 $mBackend = $this->backends
[$this->masterIndex
];
158 foreach ( array_unique( $paths ) as $path ) {
159 $params = array( 'src' => $path );
160 // Stat the file on the 'master' backend
161 $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
162 // Check of all clone backends agree with the master...
163 foreach ( $this->backends
as $index => $cBackend ) {
164 if ( $index === $this->masterIndex
) {
167 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
168 if ( $mStat ) { // file is in master
169 if ( !$cStat ) { // file should exist
170 $status->fatal( 'backend-fail-synced', $path );
171 } elseif ( $cStat['size'] != $mStat['size'] ) { // wrong size
172 $status->fatal( 'backend-fail-synced', $path );
174 $mTs = wfTimestamp( TS_UNIX
, $mStat['mtime'] );
175 $cTs = wfTimestamp( TS_UNIX
, $cStat['mtime'] );
176 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
177 $status->fatal( 'backend-fail-synced', $path );
180 } else { // file is not in master
181 if ( $cStat ) { // file should not exist
182 $status->fatal( 'backend-fail-synced', $path );
192 * Substitute the backend name in storage path parameters
193 * for a set of operations with that of a given internal backend.
195 * @param $ops Array List of file operation arrays
196 * @param $backend FileBackend
199 protected function substOpBatchPaths( array $ops, FileBackend
$backend ) {
200 $newOps = array(); // operations
201 foreach ( $ops as $op ) {
202 $newOp = $op; // operation
203 foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
204 if ( isset( $newOp[$par] ) ) { // string or array
205 $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
214 * Same as substOpBatchPaths() but for a single operation
216 * @param $op File operation array
217 * @param $backend FileBackend
220 protected function substOpPaths( array $ops, FileBackend
$backend ) {
221 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
226 * Substitute the backend of storage paths with an internal backend's name
228 * @param $paths Array|string List of paths or single string path
229 * @param $backend FileBackend
230 * @return Array|string
232 protected function substPaths( $paths, FileBackend
$backend ) {
234 '!^mwstore://' . preg_quote( $this->name
) . '/!',
235 'mwstore://' . $backend->getName() . '/',
236 $paths // string or array
241 * Substitute the backend of internal storage paths with the proxy backend's name
243 * @param $paths Array|string List of paths or single string path
244 * @return Array|string
246 protected function unsubstPaths( $paths ) {
248 '!^mwstore://([^/]+)!',
249 "mwstore://{$this->name}",
250 $paths // string or array
255 * @see FileBackendBase::doPrepare()
257 public function doPrepare( array $params ) {
258 $status = Status
::newGood();
259 foreach ( $this->backends
as $backend ) {
260 $realParams = $this->substOpPaths( $params, $backend );
261 $status->merge( $backend->doPrepare( $realParams ) );
267 * @see FileBackendBase::doSecure()
269 public function doSecure( array $params ) {
270 $status = Status
::newGood();
271 foreach ( $this->backends
as $backend ) {
272 $realParams = $this->substOpPaths( $params, $backend );
273 $status->merge( $backend->doSecure( $realParams ) );
279 * @see FileBackendBase::doClean()
281 public function doClean( array $params ) {
282 $status = Status
::newGood();
283 foreach ( $this->backends
as $backend ) {
284 $realParams = $this->substOpPaths( $params, $backend );
285 $status->merge( $backend->doClean( $realParams ) );
291 * @see FileBackendBase::getFileList()
293 public function concatenate( array $params ) {
294 // We are writing to an FS file, so we don't need to do this per-backend
295 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
296 return $this->backends
[$this->masterIndex
]->concatenate( $realParams );
300 * @see FileBackendBase::fileExists()
302 public function fileExists( array $params ) {
303 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
304 return $this->backends
[$this->masterIndex
]->fileExists( $realParams );
308 * @see FileBackendBase::getFileTimestamp()
310 public function getFileTimestamp( array $params ) {
311 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
312 return $this->backends
[$this->masterIndex
]->getFileTimestamp( $realParams );
316 * @see FileBackendBase::getFileSize()
318 public function getFileSize( array $params ) {
319 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
320 return $this->backends
[$this->masterIndex
]->getFileSize( $realParams );
324 * @see FileBackendBase::getFileStat()
326 public function getFileStat( array $params ) {
327 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
328 return $this->backends
[$this->masterIndex
]->getFileStat( $realParams );
332 * @see FileBackendBase::getFileContents()
334 public function getFileContents( array $params ) {
335 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
336 return $this->backends
[$this->masterIndex
]->getFileContents( $realParams );
340 * @see FileBackendBase::getFileSha1Base36()
342 public function getFileSha1Base36( array $params ) {
343 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
344 return $this->backends
[$this->masterIndex
]->getFileSha1Base36( $realParams );
348 * @see FileBackendBase::getFileProps()
350 public function getFileProps( array $params ) {
351 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
352 return $this->backends
[$this->masterIndex
]->getFileProps( $realParams );
356 * @see FileBackendBase::streamFile()
358 public function streamFile( array $params ) {
359 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
360 return $this->backends
[$this->masterIndex
]->streamFile( $realParams );
364 * @see FileBackendBase::getLocalReference()
366 public function getLocalReference( array $params ) {
367 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
368 return $this->backends
[$this->masterIndex
]->getLocalReference( $realParams );
372 * @see FileBackendBase::getLocalCopy()
374 public function getLocalCopy( array $params ) {
375 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
376 return $this->backends
[$this->masterIndex
]->getLocalCopy( $realParams );
380 * @see FileBackendBase::getFileList()
382 public function getFileList( array $params ) {
383 $realParams = $this->substOpPaths( $params, $this->backends
[$this->masterIndex
] );
384 return $this->backends
[$this->masterIndex
]->getFileList( $realParams );
388 * @see FileBackendBase::clearCache()
390 public function clearCache( array $paths = null ) {
391 foreach ( $this->backends
as $backend ) {
392 $realPaths = is_array( $paths ) ?
$this->substPaths( $paths ) : null;
393 $backend->clearCache( $realPaths );