Merge "Removed a useless exception catch statement in SwiftFileBackend::getFileListPa...
[lhc/web/wiklou.git] / includes / filerepo / backend / FileOp.php
1 <?php
2 /**
3 * @file
4 * @ingroup FileBackend
5 * @author Aaron Schulz
6 */
7
8 /**
9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
11 *
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
14 *
15 * @ingroup FileBackend
16 * @since 1.19
17 */
18 abstract class FileOp {
19 /** @var Array */
20 protected $params = array();
21 /** @var FileBackendStore */
22 protected $backend;
23
24 protected $state = self::STATE_NEW; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
27 protected $batchId; // string
28
29 protected $sourceSha1; // string
30 protected $destSameAsSource; // boolean
31
32 /* Object life-cycle */
33 const STATE_NEW = 1;
34 const STATE_CHECKED = 2;
35 const STATE_ATTEMPTED = 3;
36
37 /* Timeout related parameters */
38 const MAX_BATCH_SIZE = 1000;
39 const TIME_LIMIT_SEC = 300; // 5 minutes
40
41 /**
42 * Build a new file operation transaction
43 *
44 * @param $backend FileBackendStore
45 * @param $params Array
46 * @throws MWException
47 */
48 final public function __construct( FileBackendStore $backend, array $params ) {
49 $this->backend = $backend;
50 list( $required, $optional ) = $this->allowedParams();
51 foreach ( $required as $name ) {
52 if ( isset( $params[$name] ) ) {
53 $this->params[$name] = $params[$name];
54 } else {
55 throw new MWException( "File operation missing parameter '$name'." );
56 }
57 }
58 foreach ( $optional as $name ) {
59 if ( isset( $params[$name] ) ) {
60 $this->params[$name] = $params[$name];
61 }
62 }
63 $this->params = $params;
64 }
65
66 /**
67 * Set the batch UUID this operation belongs to
68 *
69 * @param $batchId string
70 * @return void
71 */
72 final protected function setBatchId( $batchId ) {
73 $this->batchId = $batchId;
74 }
75
76 /**
77 * Whether to allow stale data for file reads and stat checks
78 *
79 * @param $allowStale bool
80 * @return void
81 */
82 final protected function allowStaleReads( $allowStale ) {
83 $this->useLatest = !$allowStale;
84 }
85
86 /**
87 * Attempt to perform a series of file operations.
88 * Callers are responsible for handling file locking.
89 *
90 * $opts is an array of options, including:
91 * 'force' : Errors that would normally cause a rollback do not.
92 * The remaining operations are still attempted if any fail.
93 * 'allowStale' : Don't require the latest available data.
94 * This can increase performance for non-critical writes.
95 * This has no effect unless the 'force' flag is set.
96 * 'nonJournaled' : Don't log this operation batch in the file journal.
97 *
98 * The resulting Status will be "OK" unless:
99 * a) unexpected operation errors occurred (network partitions, disk full...)
100 * b) significant operation errors occured and 'force' was not set
101 *
102 * @param $performOps Array List of FileOp operations
103 * @param $opts Array Batch operation options
104 * @param $journal FileJournal Journal to log operations to
105 * @return Status
106 */
107 final public static function attemptBatch(
108 array $performOps, array $opts, FileJournal $journal
109 ) {
110 $status = Status::newGood();
111
112 $n = count( $performOps );
113 if ( $n > self::MAX_BATCH_SIZE ) {
114 $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
115 return $status;
116 }
117
118 $batchId = $journal->getTimestampedUUID();
119 $allowStale = !empty( $opts['allowStale'] );
120 $ignoreErrors = !empty( $opts['force'] );
121 $journaled = empty( $opts['nonJournaled'] );
122
123 $entries = array(); // file journal entries
124 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
125 // Do pre-checks for each operation; abort on failure...
126 foreach ( $performOps as $index => $fileOp ) {
127 $fileOp->setBatchId( $batchId );
128 $fileOp->allowStaleReads( $allowStale );
129 $oldPredicates = $predicates;
130 $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
131 $status->merge( $subStatus );
132 if ( $subStatus->isOK() ) {
133 if ( $journaled ) { // journal log entry
134 $entries = array_merge( $entries,
135 self::getJournalEntries( $fileOp, $oldPredicates, $predicates ) );
136 }
137 } else { // operation failed?
138 $status->success[$index] = false;
139 ++$status->failCount;
140 if ( !$ignoreErrors ) {
141 return $status; // abort
142 }
143 }
144 }
145
146 // Log the operations in file journal...
147 if ( count( $entries ) ) {
148 $subStatus = $journal->logChangeBatch( $entries, $batchId );
149 if ( !$subStatus->isOK() ) {
150 return $subStatus; // abort
151 }
152 }
153
154 if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
155 $status->setResult( true, $status->value );
156 }
157
158 // Attempt each operation...
159 foreach ( $performOps as $index => $fileOp ) {
160 if ( $fileOp->failed() ) {
161 continue; // nothing to do
162 }
163 $subStatus = $fileOp->attempt();
164 $status->merge( $subStatus );
165 if ( $subStatus->isOK() ) {
166 $status->success[$index] = true;
167 ++$status->successCount;
168 } else {
169 $status->success[$index] = false;
170 ++$status->failCount;
171 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
172 // Log the remaining ops as failed for recovery...
173 for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
174 $performOps[$i]->logFailure( 'attempt_aborted' );
175 }
176 return $status; // bail out
177 }
178 }
179
180 return $status;
181 }
182
183 /**
184 * Get the file journal entries for a single file operation
185 *
186 * @param $fileOp FileOp
187 * @param $oPredicates Array Pre-op information about files
188 * @param $nPredicates Array Post-op information about files
189 * @return Array
190 */
191 final protected static function getJournalEntries(
192 FileOp $fileOp, array $oPredicates, array $nPredicates
193 ) {
194 $nullEntries = array();
195 $updateEntries = array();
196 $deleteEntries = array();
197 $pathsUsed = array_merge( $fileOp->storagePathsRead(), $fileOp->storagePathsChanged() );
198 foreach ( $pathsUsed as $path ) {
199 $nullEntries[] = array( // assertion for recovery
200 'op' => 'null',
201 'path' => $path,
202 'newSha1' => $fileOp->fileSha1( $path, $oPredicates )
203 );
204 }
205 foreach ( $fileOp->storagePathsChanged() as $path ) {
206 if ( $nPredicates['sha1'][$path] === false ) { // deleted
207 $deleteEntries[] = array(
208 'op' => 'delete',
209 'path' => $path,
210 'newSha1' => ''
211 );
212 } else { // created/updated
213 $updateEntries[] = array(
214 'op' => $fileOp->fileExists( $path, $oPredicates ) ? 'update' : 'create',
215 'path' => $path,
216 'newSha1' => $nPredicates['sha1'][$path]
217 );
218 }
219 }
220 return array_merge( $nullEntries, $updateEntries, $deleteEntries );
221 }
222
223 /**
224 * Get the value of the parameter with the given name
225 *
226 * @param $name string
227 * @return mixed Returns null if the parameter is not set
228 */
229 final public function getParam( $name ) {
230 return isset( $this->params[$name] ) ? $this->params[$name] : null;
231 }
232
233 /**
234 * Check if this operation failed precheck() or attempt()
235 *
236 * @return bool
237 */
238 final public function failed() {
239 return $this->failed;
240 }
241
242 /**
243 * Get a new empty predicates array for precheck()
244 *
245 * @return Array
246 */
247 final public static function newPredicates() {
248 return array( 'exists' => array(), 'sha1' => array() );
249 }
250
251 /**
252 * Check preconditions of the operation without writing anything
253 *
254 * @param $predicates Array
255 * @return Status
256 */
257 final public function precheck( array &$predicates ) {
258 if ( $this->state !== self::STATE_NEW ) {
259 return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
260 }
261 $this->state = self::STATE_CHECKED;
262 $status = $this->doPrecheck( $predicates );
263 if ( !$status->isOK() ) {
264 $this->failed = true;
265 }
266 return $status;
267 }
268
269 /**
270 * Attempt the operation, backing up files as needed; this must be reversible
271 *
272 * @return Status
273 */
274 final public function attempt() {
275 if ( $this->state !== self::STATE_CHECKED ) {
276 return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
277 } elseif ( $this->failed ) { // failed precheck
278 return Status::newFatal( 'fileop-fail-attempt-precheck' );
279 }
280 $this->state = self::STATE_ATTEMPTED;
281 $status = $this->doAttempt();
282 if ( !$status->isOK() ) {
283 $this->failed = true;
284 $this->logFailure( 'attempt' );
285 }
286 return $status;
287 }
288
289 /**
290 * Get the file operation parameters
291 *
292 * @return Array (required params list, optional params list)
293 */
294 protected function allowedParams() {
295 return array( array(), array() );
296 }
297
298 /**
299 * Get a list of storage paths read from for this operation
300 *
301 * @return Array
302 */
303 public function storagePathsRead() {
304 return array();
305 }
306
307 /**
308 * Get a list of storage paths written to for this operation
309 *
310 * @return Array
311 */
312 public function storagePathsChanged() {
313 return array();
314 }
315
316 /**
317 * @return Status
318 */
319 protected function doPrecheck( array &$predicates ) {
320 return Status::newGood();
321 }
322
323 /**
324 * @return Status
325 */
326 protected function doAttempt() {
327 return Status::newGood();
328 }
329
330 /**
331 * Check for errors with regards to the destination file already existing.
332 * This also updates the destSameAsSource and sourceSha1 member variables.
333 * A bad status will be returned if there is no chance it can be overwritten.
334 *
335 * @param $predicates Array
336 * @return Status
337 */
338 protected function precheckDestExistence( array $predicates ) {
339 $status = Status::newGood();
340 // Get hash of source file/string and the destination file
341 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
342 if ( $this->sourceSha1 === null ) { // file in storage?
343 $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
344 }
345 $this->destSameAsSource = false;
346 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
347 if ( $this->getParam( 'overwrite' ) ) {
348 return $status; // OK
349 } elseif ( $this->getParam( 'overwriteSame' ) ) {
350 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
351 // Check if hashes are valid and match each other...
352 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
353 $status->fatal( 'backend-fail-hashes' );
354 } elseif ( $this->sourceSha1 !== $dhash ) {
355 // Give an error if the files are not identical
356 $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
357 } else {
358 $this->destSameAsSource = true; // OK
359 }
360 return $status; // do nothing; either OK or bad status
361 } else {
362 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
363 return $status;
364 }
365 }
366 return $status;
367 }
368
369 /**
370 * precheckDestExistence() helper function to get the source file SHA-1.
371 * Subclasses should overwride this iff the source is not in storage.
372 *
373 * @return string|bool Returns false on failure
374 */
375 protected function getSourceSha1Base36() {
376 return null; // N/A
377 }
378
379 /**
380 * Check if a file will exist in storage when this operation is attempted
381 *
382 * @param $source string Storage path
383 * @param $predicates Array
384 * @return bool
385 */
386 final protected function fileExists( $source, array $predicates ) {
387 if ( isset( $predicates['exists'][$source] ) ) {
388 return $predicates['exists'][$source]; // previous op assures this
389 } else {
390 $params = array( 'src' => $source, 'latest' => $this->useLatest );
391 return $this->backend->fileExists( $params );
392 }
393 }
394
395 /**
396 * Get the SHA-1 of a file in storage when this operation is attempted
397 *
398 * @param $source string Storage path
399 * @param $predicates Array
400 * @return string|bool False on failure
401 */
402 final protected function fileSha1( $source, array $predicates ) {
403 if ( isset( $predicates['sha1'][$source] ) ) {
404 return $predicates['sha1'][$source]; // previous op assures this
405 } else {
406 $params = array( 'src' => $source, 'latest' => $this->useLatest );
407 return $this->backend->getFileSha1Base36( $params );
408 }
409 }
410
411 /**
412 * Log a file operation failure and preserve any temp files
413 *
414 * @param $action string
415 * @return void
416 */
417 final protected function logFailure( $action ) {
418 $params = $this->params;
419 $params['failedAction'] = $action;
420 try {
421 wfDebugLog( 'FileOperation', get_class( $this ) .
422 " failed (batch #{$this->batchId}): " . FormatJson::encode( $params ) );
423 } catch ( Exception $e ) {
424 // bad config? debug log error?
425 }
426 }
427 }
428
429 /**
430 * Store a file into the backend from a file on the file system.
431 * Parameters similar to FileBackendStore::storeInternal(), which include:
432 * src : source path on file system
433 * dst : destination storage path
434 * overwrite : do nothing and pass if an identical file exists at destination
435 * overwriteSame : override any existing file at destination
436 */
437 class StoreFileOp extends FileOp {
438 protected function allowedParams() {
439 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
440 }
441
442 protected function doPrecheck( array &$predicates ) {
443 $status = Status::newGood();
444 // Check if the source file exists on the file system
445 if ( !is_file( $this->params['src'] ) ) {
446 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
447 return $status;
448 // Check if the source file is too big
449 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
450 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
451 return $status;
452 // Check if a file can be placed at the destination
453 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
454 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
455 return $status;
456 }
457 // Check if destination file exists
458 $status->merge( $this->precheckDestExistence( $predicates ) );
459 if ( $status->isOK() ) {
460 // Update file existence predicates
461 $predicates['exists'][$this->params['dst']] = true;
462 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
463 }
464 return $status; // safe to call attempt()
465 }
466
467 protected function doAttempt() {
468 $status = Status::newGood();
469 // Store the file at the destination
470 if ( !$this->destSameAsSource ) {
471 $status->merge( $this->backend->storeInternal( $this->params ) );
472 }
473 return $status;
474 }
475
476 protected function getSourceSha1Base36() {
477 wfSuppressWarnings();
478 $hash = sha1_file( $this->params['src'] );
479 wfRestoreWarnings();
480 if ( $hash !== false ) {
481 $hash = wfBaseConvert( $hash, 16, 36, 31 );
482 }
483 return $hash;
484 }
485
486 public function storagePathsChanged() {
487 return array( $this->params['dst'] );
488 }
489 }
490
491 /**
492 * Create a file in the backend with the given content.
493 * Parameters similar to FileBackendStore::createInternal(), which include:
494 * content : the raw file contents
495 * dst : destination storage path
496 * overwrite : do nothing and pass if an identical file exists at destination
497 * overwriteSame : override any existing file at destination
498 */
499 class CreateFileOp extends FileOp {
500 protected function allowedParams() {
501 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
502 }
503
504 protected function doPrecheck( array &$predicates ) {
505 $status = Status::newGood();
506 // Check if the source data is too big
507 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
508 $status->fatal( 'backend-fail-create', $this->params['dst'] );
509 return $status;
510 // Check if a file can be placed at the destination
511 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
512 $status->fatal( 'backend-fail-create', $this->params['dst'] );
513 return $status;
514 }
515 // Check if destination file exists
516 $status->merge( $this->precheckDestExistence( $predicates ) );
517 if ( $status->isOK() ) {
518 // Update file existence predicates
519 $predicates['exists'][$this->params['dst']] = true;
520 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
521 }
522 return $status; // safe to call attempt()
523 }
524
525 protected function doAttempt() {
526 $status = Status::newGood();
527 // Create the file at the destination
528 if ( !$this->destSameAsSource ) {
529 $status->merge( $this->backend->createInternal( $this->params ) );
530 }
531 return $status;
532 }
533
534 protected function getSourceSha1Base36() {
535 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
536 }
537
538 public function storagePathsChanged() {
539 return array( $this->params['dst'] );
540 }
541 }
542
543 /**
544 * Copy a file from one storage path to another in the backend.
545 * Parameters similar to FileBackendStore::copyInternal(), which include:
546 * src : source storage path
547 * dst : destination storage path
548 * overwrite : do nothing and pass if an identical file exists at destination
549 * overwriteSame : override any existing file at destination
550 */
551 class CopyFileOp extends FileOp {
552 protected function allowedParams() {
553 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
554 }
555
556 protected function doPrecheck( array &$predicates ) {
557 $status = Status::newGood();
558 // Check if the source file exists
559 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
560 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
561 return $status;
562 // Check if a file can be placed at the destination
563 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
564 $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
565 return $status;
566 }
567 // Check if destination file exists
568 $status->merge( $this->precheckDestExistence( $predicates ) );
569 if ( $status->isOK() ) {
570 // Update file existence predicates
571 $predicates['exists'][$this->params['dst']] = true;
572 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
573 }
574 return $status; // safe to call attempt()
575 }
576
577 protected function doAttempt() {
578 $status = Status::newGood();
579 // Do nothing if the src/dst paths are the same
580 if ( $this->params['src'] !== $this->params['dst'] ) {
581 // Copy the file into the destination
582 if ( !$this->destSameAsSource ) {
583 $status->merge( $this->backend->copyInternal( $this->params ) );
584 }
585 }
586 return $status;
587 }
588
589 public function storagePathsRead() {
590 return array( $this->params['src'] );
591 }
592
593 public function storagePathsChanged() {
594 return array( $this->params['dst'] );
595 }
596 }
597
598 /**
599 * Move a file from one storage path to another in the backend.
600 * Parameters similar to FileBackendStore::moveInternal(), which include:
601 * src : source storage path
602 * dst : destination storage path
603 * overwrite : do nothing and pass if an identical file exists at destination
604 * overwriteSame : override any existing file at destination
605 */
606 class MoveFileOp extends FileOp {
607 protected function allowedParams() {
608 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
609 }
610
611 protected function doPrecheck( array &$predicates ) {
612 $status = Status::newGood();
613 // Check if the source file exists
614 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
615 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
616 return $status;
617 // Check if a file can be placed at the destination
618 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
619 $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
620 return $status;
621 }
622 // Check if destination file exists
623 $status->merge( $this->precheckDestExistence( $predicates ) );
624 if ( $status->isOK() ) {
625 // Update file existence predicates
626 $predicates['exists'][$this->params['src']] = false;
627 $predicates['sha1'][$this->params['src']] = false;
628 $predicates['exists'][$this->params['dst']] = true;
629 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
630 }
631 return $status; // safe to call attempt()
632 }
633
634 protected function doAttempt() {
635 $status = Status::newGood();
636 // Do nothing if the src/dst paths are the same
637 if ( $this->params['src'] !== $this->params['dst'] ) {
638 if ( !$this->destSameAsSource ) {
639 // Move the file into the destination
640 $status->merge( $this->backend->moveInternal( $this->params ) );
641 } else {
642 // Just delete source as the destination needs no changes
643 $params = array( 'src' => $this->params['src'] );
644 $status->merge( $this->backend->deleteInternal( $params ) );
645 }
646 }
647 return $status;
648 }
649
650 public function storagePathsRead() {
651 return array( $this->params['src'] );
652 }
653
654 public function storagePathsChanged() {
655 return array( $this->params['src'], $this->params['dst'] );
656 }
657 }
658
659 /**
660 * Delete a file at the given storage path from the backend.
661 * Parameters similar to FileBackendStore::deleteInternal(), which include:
662 * src : source storage path
663 * ignoreMissingSource : don't return an error if the file does not exist
664 */
665 class DeleteFileOp extends FileOp {
666 protected function allowedParams() {
667 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
668 }
669
670 protected $needsDelete = true;
671
672 protected function doPrecheck( array &$predicates ) {
673 $status = Status::newGood();
674 // Check if the source file exists
675 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
676 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
677 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
678 return $status;
679 }
680 $this->needsDelete = false;
681 }
682 // Update file existence predicates
683 $predicates['exists'][$this->params['src']] = false;
684 $predicates['sha1'][$this->params['src']] = false;
685 return $status; // safe to call attempt()
686 }
687
688 protected function doAttempt() {
689 $status = Status::newGood();
690 if ( $this->needsDelete ) {
691 // Delete the source file
692 $status->merge( $this->backend->deleteInternal( $this->params ) );
693 }
694 return $status;
695 }
696
697 public function storagePathsChanged() {
698 return array( $this->params['src'] );
699 }
700 }
701
702 /**
703 * Placeholder operation that has no params and does nothing
704 */
705 class NullFileOp extends FileOp {}