Merge "add Article::setParserOptions" into Wikidata
[lhc/web/wiklou.git] / includes / filerepo / backend / FileOp.php
1 <?php
2 /**
3 * Helper class for representing operations with transaction support.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup FileBackend
22 * @author Aaron Schulz
23 */
24
25 /**
26 * FileBackend helper class for representing operations.
27 * Do not use this class from places outside FileBackend.
28 *
29 * Methods called from FileOpBatch::attempt() should avoid throwing
30 * exceptions at all costs. FileOp objects should be lightweight in order
31 * to support large arrays in memory and serialization.
32 *
33 * @ingroup FileBackend
34 * @since 1.19
35 */
36 abstract class FileOp {
37 /** @var Array */
38 protected $params = array();
39 /** @var FileBackendStore */
40 protected $backend;
41
42 protected $state = self::STATE_NEW; // integer
43 protected $failed = false; // boolean
44 protected $async = false; // boolean
45 protected $useLatest = true; // boolean
46 protected $batchId; // string
47
48 protected $sourceSha1; // string
49 protected $destSameAsSource; // boolean
50
51 /* Object life-cycle */
52 const STATE_NEW = 1;
53 const STATE_CHECKED = 2;
54 const STATE_ATTEMPTED = 3;
55
56 /**
57 * Build a new file operation transaction
58 *
59 * @param $backend FileBackendStore
60 * @param $params Array
61 * @throws MWException
62 */
63 final public function __construct( FileBackendStore $backend, array $params ) {
64 $this->backend = $backend;
65 list( $required, $optional ) = $this->allowedParams();
66 foreach ( $required as $name ) {
67 if ( isset( $params[$name] ) ) {
68 $this->params[$name] = $params[$name];
69 } else {
70 throw new MWException( "File operation missing parameter '$name'." );
71 }
72 }
73 foreach ( $optional as $name ) {
74 if ( isset( $params[$name] ) ) {
75 $this->params[$name] = $params[$name];
76 }
77 }
78 $this->params = $params;
79 }
80
81 /**
82 * Set the batch UUID this operation belongs to
83 *
84 * @param $batchId string
85 * @return void
86 */
87 final public function setBatchId( $batchId ) {
88 $this->batchId = $batchId;
89 }
90
91 /**
92 * Whether to allow stale data for file reads and stat checks
93 *
94 * @param $allowStale bool
95 * @return void
96 */
97 final public function allowStaleReads( $allowStale ) {
98 $this->useLatest = !$allowStale;
99 }
100
101 /**
102 * Get the value of the parameter with the given name
103 *
104 * @param $name string
105 * @return mixed Returns null if the parameter is not set
106 */
107 final public function getParam( $name ) {
108 return isset( $this->params[$name] ) ? $this->params[$name] : null;
109 }
110
111 /**
112 * Check if this operation failed precheck() or attempt()
113 *
114 * @return bool
115 */
116 final public function failed() {
117 return $this->failed;
118 }
119
120 /**
121 * Get a new empty predicates array for precheck()
122 *
123 * @return Array
124 */
125 final public static function newPredicates() {
126 return array( 'exists' => array(), 'sha1' => array() );
127 }
128
129 /**
130 * Get a new empty dependency tracking array for paths read/written to
131 *
132 * @return Array
133 */
134 final public static function newDependencies() {
135 return array( 'read' => array(), 'write' => array() );
136 }
137
138 /**
139 * Update a dependency tracking array to account for this operation
140 *
141 * @param $deps Array Prior path reads/writes; format of FileOp::newPredicates()
142 * @return Array
143 */
144 final public function applyDependencies( array $deps ) {
145 $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 );
146 $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 );
147 return $deps;
148 }
149
150 /**
151 * Check if this operation changes files listed in $paths
152 *
153 * @param $paths Array Prior path reads/writes; format of FileOp::newPredicates()
154 * @return boolean
155 */
156 final public function dependsOn( array $deps ) {
157 foreach ( $this->storagePathsChanged() as $path ) {
158 if ( isset( $deps['read'][$path] ) || isset( $deps['write'][$path] ) ) {
159 return true; // "output" or "anti" dependency
160 }
161 }
162 foreach ( $this->storagePathsRead() as $path ) {
163 if ( isset( $deps['write'][$path] ) ) {
164 return true; // "flow" dependency
165 }
166 }
167 return false;
168 }
169
170 /**
171 * Get the file journal entries for this file operation
172 *
173 * @param $oPredicates Array Pre-op info about files (format of FileOp::newPredicates)
174 * @param $nPredicates Array Post-op info about files (format of FileOp::newPredicates)
175 * @return Array
176 */
177 final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
178 $nullEntries = array();
179 $updateEntries = array();
180 $deleteEntries = array();
181 $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
182 foreach ( $pathsUsed as $path ) {
183 $nullEntries[] = array( // assertion for recovery
184 'op' => 'null',
185 'path' => $path,
186 'newSha1' => $this->fileSha1( $path, $oPredicates )
187 );
188 }
189 foreach ( $this->storagePathsChanged() as $path ) {
190 if ( $nPredicates['sha1'][$path] === false ) { // deleted
191 $deleteEntries[] = array(
192 'op' => 'delete',
193 'path' => $path,
194 'newSha1' => ''
195 );
196 } else { // created/updated
197 $updateEntries[] = array(
198 'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create',
199 'path' => $path,
200 'newSha1' => $nPredicates['sha1'][$path]
201 );
202 }
203 }
204 return array_merge( $nullEntries, $updateEntries, $deleteEntries );
205 }
206
207 /**
208 * Check preconditions of the operation without writing anything
209 *
210 * @param $predicates Array
211 * @return Status
212 */
213 final public function precheck( array &$predicates ) {
214 if ( $this->state !== self::STATE_NEW ) {
215 return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
216 }
217 $this->state = self::STATE_CHECKED;
218 $status = $this->doPrecheck( $predicates );
219 if ( !$status->isOK() ) {
220 $this->failed = true;
221 }
222 return $status;
223 }
224
225 /**
226 * @return Status
227 */
228 protected function doPrecheck( array &$predicates ) {
229 return Status::newGood();
230 }
231
232 /**
233 * Attempt the operation
234 *
235 * @return Status
236 */
237 final public function attempt() {
238 if ( $this->state !== self::STATE_CHECKED ) {
239 return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
240 } elseif ( $this->failed ) { // failed precheck
241 return Status::newFatal( 'fileop-fail-attempt-precheck' );
242 }
243 $this->state = self::STATE_ATTEMPTED;
244 $status = $this->doAttempt();
245 if ( !$status->isOK() ) {
246 $this->failed = true;
247 $this->logFailure( 'attempt' );
248 }
249 return $status;
250 }
251
252 /**
253 * @return Status
254 */
255 protected function doAttempt() {
256 return Status::newGood();
257 }
258
259 /**
260 * Attempt the operation in the background
261 *
262 * @return Status
263 */
264 final public function attemptAsync() {
265 $this->async = true;
266 $result = $this->attempt();
267 $this->async = false;
268 return $result;
269 }
270
271 /**
272 * Get the file operation parameters
273 *
274 * @return Array (required params list, optional params list)
275 */
276 protected function allowedParams() {
277 return array( array(), array() );
278 }
279
280 /**
281 * Adjust params to FileBackendStore internal file calls
282 *
283 * @param $params Array
284 * @return Array (required params list, optional params list)
285 */
286 protected function setFlags( array $params ) {
287 return array( 'async' => $this->async ) + $params;
288 }
289
290 /**
291 * Get a list of storage paths read from for this operation
292 *
293 * @return Array
294 */
295 final public function storagePathsRead() {
296 return array_map( 'FileBackend::normalizeStoragePath', $this->doStoragePathsRead() );
297 }
298
299 /**
300 * @see FileOp::storagePathsRead()
301 * @return Array
302 */
303 protected function doStoragePathsRead() {
304 return array();
305 }
306
307 /**
308 * Get a list of storage paths written to for this operation
309 *
310 * @return Array
311 */
312 final public function storagePathsChanged() {
313 return array_map( 'FileBackend::normalizeStoragePath', $this->doStoragePathsChanged() );
314 }
315
316 /**
317 * @see FileOp::storagePathsChanged()
318 * @return Array
319 */
320 protected function doStoragePathsChanged() {
321 return array();
322 }
323
324 /**
325 * Check for errors with regards to the destination file already existing.
326 * This also updates the destSameAsSource and sourceSha1 member variables.
327 * A bad status will be returned if there is no chance it can be overwritten.
328 *
329 * @param $predicates Array
330 * @return Status
331 */
332 protected function precheckDestExistence( array $predicates ) {
333 $status = Status::newGood();
334 // Get hash of source file/string and the destination file
335 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
336 if ( $this->sourceSha1 === null ) { // file in storage?
337 $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
338 }
339 $this->destSameAsSource = false;
340 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
341 if ( $this->getParam( 'overwrite' ) ) {
342 return $status; // OK
343 } elseif ( $this->getParam( 'overwriteSame' ) ) {
344 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
345 // Check if hashes are valid and match each other...
346 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
347 $status->fatal( 'backend-fail-hashes' );
348 } elseif ( $this->sourceSha1 !== $dhash ) {
349 // Give an error if the files are not identical
350 $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
351 } else {
352 $this->destSameAsSource = true; // OK
353 }
354 return $status; // do nothing; either OK or bad status
355 } else {
356 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
357 return $status;
358 }
359 }
360 return $status;
361 }
362
363 /**
364 * precheckDestExistence() helper function to get the source file SHA-1.
365 * Subclasses should overwride this iff the source is not in storage.
366 *
367 * @return string|bool Returns false on failure
368 */
369 protected function getSourceSha1Base36() {
370 return null; // N/A
371 }
372
373 /**
374 * Check if a file will exist in storage when this operation is attempted
375 *
376 * @param $source string Storage path
377 * @param $predicates Array
378 * @return bool
379 */
380 final protected function fileExists( $source, array $predicates ) {
381 if ( isset( $predicates['exists'][$source] ) ) {
382 return $predicates['exists'][$source]; // previous op assures this
383 } else {
384 $params = array( 'src' => $source, 'latest' => $this->useLatest );
385 return $this->backend->fileExists( $params );
386 }
387 }
388
389 /**
390 * Get the SHA-1 of a file in storage when this operation is attempted
391 *
392 * @param $source string Storage path
393 * @param $predicates Array
394 * @return string|bool False on failure
395 */
396 final protected function fileSha1( $source, array $predicates ) {
397 if ( isset( $predicates['sha1'][$source] ) ) {
398 return $predicates['sha1'][$source]; // previous op assures this
399 } else {
400 $params = array( 'src' => $source, 'latest' => $this->useLatest );
401 return $this->backend->getFileSha1Base36( $params );
402 }
403 }
404
405 /**
406 * Get the backend this operation is for
407 *
408 * @return FileBackendStore
409 */
410 public function getBackend() {
411 return $this->backend;
412 }
413
414 /**
415 * Log a file operation failure and preserve any temp files
416 *
417 * @param $action string
418 * @return void
419 */
420 final public function logFailure( $action ) {
421 $params = $this->params;
422 $params['failedAction'] = $action;
423 try {
424 wfDebugLog( 'FileOperation', get_class( $this ) .
425 " failed (batch #{$this->batchId}): " . FormatJson::encode( $params ) );
426 } catch ( Exception $e ) {
427 // bad config? debug log error?
428 }
429 }
430 }
431
432 /**
433 * Store a file into the backend from a file on the file system.
434 * Parameters similar to FileBackendStore::storeInternal(), which include:
435 * - src : source path on file system
436 * - dst : destination storage path
437 * - overwrite : do nothing and pass if an identical file exists at destination
438 * - overwriteSame : override any existing file at destination
439 */
440 class StoreFileOp extends FileOp {
441 /**
442 * @return array
443 */
444 protected function allowedParams() {
445 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
446 }
447
448 /**
449 * @param $predicates array
450 * @return Status
451 */
452 protected function doPrecheck( array &$predicates ) {
453 $status = Status::newGood();
454 // Check if the source file exists on the file system
455 if ( !is_file( $this->params['src'] ) ) {
456 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
457 return $status;
458 // Check if the source file is too big
459 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
460 $status->fatal( 'backend-fail-maxsize',
461 $this->params['dst'], $this->backend->maxFileSizeInternal() );
462 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
463 return $status;
464 // Check if a file can be placed at the destination
465 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
466 $status->fatal( 'backend-fail-usable', $this->params['dst'] );
467 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
468 return $status;
469 }
470 // Check if destination file exists
471 $status->merge( $this->precheckDestExistence( $predicates ) );
472 if ( $status->isOK() ) {
473 // Update file existence predicates
474 $predicates['exists'][$this->params['dst']] = true;
475 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
476 }
477 return $status; // safe to call attempt()
478 }
479
480 /**
481 * @return Status
482 */
483 protected function doAttempt() {
484 // Store the file at the destination
485 if ( !$this->destSameAsSource ) {
486 return $this->backend->storeInternal( $this->setFlags( $this->params ) );
487 }
488 return Status::newGood();
489 }
490
491 /**
492 * @return bool|string
493 */
494 protected function getSourceSha1Base36() {
495 wfSuppressWarnings();
496 $hash = sha1_file( $this->params['src'] );
497 wfRestoreWarnings();
498 if ( $hash !== false ) {
499 $hash = wfBaseConvert( $hash, 16, 36, 31 );
500 }
501 return $hash;
502 }
503
504 protected function doStoragePathsChanged() {
505 return array( $this->params['dst'] );
506 }
507 }
508
509 /**
510 * Create a file in the backend with the given content.
511 * Parameters similar to FileBackendStore::createInternal(), which include:
512 * - content : the raw file contents
513 * - dst : destination storage path
514 * - overwrite : do nothing and pass if an identical file exists at destination
515 * - overwriteSame : override any existing file at destination
516 */
517 class CreateFileOp extends FileOp {
518 protected function allowedParams() {
519 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
520 }
521
522 protected function doPrecheck( array &$predicates ) {
523 $status = Status::newGood();
524 // Check if the source data is too big
525 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
526 $status->fatal( 'backend-fail-maxsize',
527 $this->params['dst'], $this->backend->maxFileSizeInternal() );
528 $status->fatal( 'backend-fail-create', $this->params['dst'] );
529 return $status;
530 // Check if a file can be placed at the destination
531 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
532 $status->fatal( 'backend-fail-usable', $this->params['dst'] );
533 $status->fatal( 'backend-fail-create', $this->params['dst'] );
534 return $status;
535 }
536 // Check if destination file exists
537 $status->merge( $this->precheckDestExistence( $predicates ) );
538 if ( $status->isOK() ) {
539 // Update file existence predicates
540 $predicates['exists'][$this->params['dst']] = true;
541 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
542 }
543 return $status; // safe to call attempt()
544 }
545
546 /**
547 * @return Status
548 */
549 protected function doAttempt() {
550 if ( !$this->destSameAsSource ) {
551 // Create the file at the destination
552 return $this->backend->createInternal( $this->setFlags( $this->params ) );
553 }
554 return Status::newGood();
555 }
556
557 /**
558 * @return bool|String
559 */
560 protected function getSourceSha1Base36() {
561 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
562 }
563
564 /**
565 * @return array
566 */
567 protected function doStoragePathsChanged() {
568 return array( $this->params['dst'] );
569 }
570 }
571
572 /**
573 * Copy a file from one storage path to another in the backend.
574 * Parameters similar to FileBackendStore::copyInternal(), which include:
575 * - src : source storage path
576 * - dst : destination storage path
577 * - overwrite : do nothing and pass if an identical file exists at destination
578 * - overwriteSame : override any existing file at destination
579 */
580 class CopyFileOp extends FileOp {
581 /**
582 * @return array
583 */
584 protected function allowedParams() {
585 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
586 }
587
588 /**
589 * @param $predicates array
590 * @return Status
591 */
592 protected function doPrecheck( array &$predicates ) {
593 $status = Status::newGood();
594 // Check if the source file exists
595 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
596 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
597 return $status;
598 // Check if a file can be placed at the destination
599 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
600 $status->fatal( 'backend-fail-usable', $this->params['dst'] );
601 $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
602 return $status;
603 }
604 // Check if destination file exists
605 $status->merge( $this->precheckDestExistence( $predicates ) );
606 if ( $status->isOK() ) {
607 // Update file existence predicates
608 $predicates['exists'][$this->params['dst']] = true;
609 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
610 }
611 return $status; // safe to call attempt()
612 }
613
614 /**
615 * @return Status
616 */
617 protected function doAttempt() {
618 // Do nothing if the src/dst paths are the same
619 if ( $this->params['src'] !== $this->params['dst'] ) {
620 // Copy the file into the destination
621 if ( !$this->destSameAsSource ) {
622 return $this->backend->copyInternal( $this->setFlags( $this->params ) );
623 }
624 }
625 return Status::newGood();
626 }
627
628 /**
629 * @return array
630 */
631 protected function doStoragePathsRead() {
632 return array( $this->params['src'] );
633 }
634
635 /**
636 * @return array
637 */
638 protected function doStoragePathsChanged() {
639 return array( $this->params['dst'] );
640 }
641 }
642
643 /**
644 * Move a file from one storage path to another in the backend.
645 * Parameters similar to FileBackendStore::moveInternal(), which include:
646 * - src : source storage path
647 * - dst : destination storage path
648 * - overwrite : do nothing and pass if an identical file exists at destination
649 * - overwriteSame : override any existing file at destination
650 */
651 class MoveFileOp extends FileOp {
652 /**
653 * @return array
654 */
655 protected function allowedParams() {
656 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
657 }
658
659 /**
660 * @param $predicates array
661 * @return Status
662 */
663 protected function doPrecheck( array &$predicates ) {
664 $status = Status::newGood();
665 // Check if the source file exists
666 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
667 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
668 return $status;
669 // Check if a file can be placed at the destination
670 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
671 $status->fatal( 'backend-fail-usable', $this->params['dst'] );
672 $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
673 return $status;
674 }
675 // Check if destination file exists
676 $status->merge( $this->precheckDestExistence( $predicates ) );
677 if ( $status->isOK() ) {
678 // Update file existence predicates
679 $predicates['exists'][$this->params['src']] = false;
680 $predicates['sha1'][$this->params['src']] = false;
681 $predicates['exists'][$this->params['dst']] = true;
682 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
683 }
684 return $status; // safe to call attempt()
685 }
686
687 /**
688 * @return Status
689 */
690 protected function doAttempt() {
691 // Do nothing if the src/dst paths are the same
692 if ( $this->params['src'] !== $this->params['dst'] ) {
693 if ( !$this->destSameAsSource ) {
694 // Move the file into the destination
695 return $this->backend->moveInternal( $this->setFlags( $this->params ) );
696 } else {
697 // Just delete source as the destination needs no changes
698 $params = array( 'src' => $this->params['src'] );
699 return $this->backend->deleteInternal( $this->setFlags( $params ) );
700 }
701 }
702 return Status::newGood();
703 }
704
705 /**
706 * @return array
707 */
708 protected function doStoragePathsRead() {
709 return array( $this->params['src'] );
710 }
711
712 /**
713 * @return array
714 */
715 protected function doStoragePathsChanged() {
716 return array( $this->params['src'], $this->params['dst'] );
717 }
718 }
719
720 /**
721 * Delete a file at the given storage path from the backend.
722 * Parameters similar to FileBackendStore::deleteInternal(), which include:
723 * - src : source storage path
724 * - ignoreMissingSource : don't return an error if the file does not exist
725 */
726 class DeleteFileOp extends FileOp {
727 /**
728 * @return array
729 */
730 protected function allowedParams() {
731 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
732 }
733
734 protected $needsDelete = true;
735
736 /**
737 * @param array $predicates
738 * @return Status
739 */
740 protected function doPrecheck( array &$predicates ) {
741 $status = Status::newGood();
742 // Check if the source file exists
743 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
744 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
745 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
746 return $status;
747 }
748 $this->needsDelete = false;
749 }
750 // Update file existence predicates
751 $predicates['exists'][$this->params['src']] = false;
752 $predicates['sha1'][$this->params['src']] = false;
753 return $status; // safe to call attempt()
754 }
755
756 /**
757 * @return Status
758 */
759 protected function doAttempt() {
760 if ( $this->needsDelete ) {
761 // Delete the source file
762 return $this->backend->deleteInternal( $this->setFlags( $this->params ) );
763 }
764 return Status::newGood();
765 }
766
767 /**
768 * @return array
769 */
770 protected function doStoragePathsChanged() {
771 return array( $this->params['src'] );
772 }
773 }
774
775 /**
776 * Placeholder operation that has no params and does nothing
777 */
778 class NullFileOp extends FileOp {}