Fixed $headers check in streamFile() to use isset()
[lhc/web/wiklou.git] / includes / filerepo / backend / FileOp.php
1 <?php
2 /**
3 * @file
4 * @ingroup FileBackend
5 * @author Aaron Schulz
6 */
7
8 /**
9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
11 *
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
14 *
15 * @ingroup FileBackend
16 * @since 1.19
17 */
18 abstract class FileOp {
19 /** @var Array */
20 protected $params = array();
21 /** @var FileBackendStore */
22 protected $backend;
23
24 protected $state = self::STATE_NEW; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
27
28 protected $sourceSha1; // string
29 protected $destSameAsSource; // boolean
30
31 /* Object life-cycle */
32 const STATE_NEW = 1;
33 const STATE_CHECKED = 2;
34 const STATE_ATTEMPTED = 3;
35
36 /* Timeout related parameters */
37 const MAX_BATCH_SIZE = 1000;
38 const TIME_LIMIT_SEC = 300; // 5 minutes
39
40 /**
41 * Build a new file operation transaction
42 *
43 * @param $backend FileBackendStore
44 * @param $params Array
45 * @throws MWException
46 */
47 final public function __construct( FileBackendStore $backend, array $params ) {
48 $this->backend = $backend;
49 list( $required, $optional ) = $this->allowedParams();
50 foreach ( $required as $name ) {
51 if ( isset( $params[$name] ) ) {
52 $this->params[$name] = $params[$name];
53 } else {
54 throw new MWException( "File operation missing parameter '$name'." );
55 }
56 }
57 foreach ( $optional as $name ) {
58 if ( isset( $params[$name] ) ) {
59 $this->params[$name] = $params[$name];
60 }
61 }
62 $this->params = $params;
63 }
64
65 /**
66 * Allow stale data for file reads and existence checks
67 *
68 * @return void
69 */
70 final protected function allowStaleReads() {
71 $this->useLatest = false;
72 }
73
74 /**
75 * Attempt a series of file operations.
76 * Callers are responsible for handling file locking.
77 *
78 * $opts is an array of options, including:
79 * 'force' : Errors that would normally cause a rollback do not.
80 * The remaining operations are still attempted if any fail.
81 * 'allowStale' : Don't require the latest available data.
82 * This can increase performance for non-critical writes.
83 * This has no effect unless the 'force' flag is set.
84 *
85 * @param $performOps Array List of FileOp operations
86 * @param $opts Array Batch operation options
87 * @return Status
88 */
89 final public static function attemptBatch( array $performOps, array $opts ) {
90 $status = Status::newGood();
91
92 $allowStale = !empty( $opts['allowStale'] );
93 $ignoreErrors = !empty( $opts['force'] );
94
95 $n = count( $performOps );
96 if ( $n > self::MAX_BATCH_SIZE ) {
97 $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
98 return $status;
99 }
100
101 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
102 // Do pre-checks for each operation; abort on failure...
103 foreach ( $performOps as $index => $fileOp ) {
104 if ( $allowStale ) {
105 $fileOp->allowStaleReads(); // allow potentially stale reads
106 }
107 $subStatus = $fileOp->precheck( $predicates );
108 $status->merge( $subStatus );
109 if ( !$subStatus->isOK() ) { // operation failed?
110 $status->success[$index] = false;
111 ++$status->failCount;
112 if ( !$ignoreErrors ) {
113 return $status; // abort
114 }
115 }
116 }
117
118 // Restart PHP's execution timer and set the timeout to safe amount.
119 // This handles cases where the operations take a long time or where we are
120 // already running low on time left. The old timeout is restored afterwards.
121 # @TODO: re-enable this for when the number of batches is high.
122 #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
123
124 // Attempt each operation...
125 foreach ( $performOps as $index => $fileOp ) {
126 if ( $fileOp->failed() ) {
127 continue; // nothing to do
128 }
129 $subStatus = $fileOp->attempt();
130 $status->merge( $subStatus );
131 if ( $subStatus->isOK() ) {
132 $status->success[$index] = true;
133 ++$status->successCount;
134 } else {
135 $status->success[$index] = false;
136 ++$status->failCount;
137 if ( !$ignoreErrors ) {
138 // Log remaining ops as failed for recovery...
139 for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
140 $performOps[$i]->logFailure( 'attempt_aborted' );
141 }
142 return $status; // bail out
143 }
144 }
145 }
146
147 return $status;
148 }
149
150 /**
151 * Get the value of the parameter with the given name
152 *
153 * @param $name string
154 * @return mixed Returns null if the parameter is not set
155 */
156 final public function getParam( $name ) {
157 return isset( $this->params[$name] ) ? $this->params[$name] : null;
158 }
159
160 /**
161 * Check if this operation failed precheck() or attempt()
162 *
163 * @return bool
164 */
165 final public function failed() {
166 return $this->failed;
167 }
168
169 /**
170 * Get a new empty predicates array for precheck()
171 *
172 * @return Array
173 */
174 final public static function newPredicates() {
175 return array( 'exists' => array(), 'sha1' => array() );
176 }
177
178 /**
179 * Check preconditions of the operation without writing anything
180 *
181 * @param $predicates Array
182 * @return Status
183 */
184 final public function precheck( array &$predicates ) {
185 if ( $this->state !== self::STATE_NEW ) {
186 return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
187 }
188 $this->state = self::STATE_CHECKED;
189 $status = $this->doPrecheck( $predicates );
190 if ( !$status->isOK() ) {
191 $this->failed = true;
192 }
193 return $status;
194 }
195
196 /**
197 * Attempt the operation, backing up files as needed; this must be reversible
198 *
199 * @return Status
200 */
201 final public function attempt() {
202 if ( $this->state !== self::STATE_CHECKED ) {
203 return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
204 } elseif ( $this->failed ) { // failed precheck
205 return Status::newFatal( 'fileop-fail-attempt-precheck' );
206 }
207 $this->state = self::STATE_ATTEMPTED;
208 $status = $this->doAttempt();
209 if ( !$status->isOK() ) {
210 $this->failed = true;
211 $this->logFailure( 'attempt' );
212 }
213 return $status;
214 }
215
216 /**
217 * Get the file operation parameters
218 *
219 * @return Array (required params list, optional params list)
220 */
221 protected function allowedParams() {
222 return array( array(), array() );
223 }
224
225 /**
226 * Get a list of storage paths read from for this operation
227 *
228 * @return Array
229 */
230 public function storagePathsRead() {
231 return array();
232 }
233
234 /**
235 * Get a list of storage paths written to for this operation
236 *
237 * @return Array
238 */
239 public function storagePathsChanged() {
240 return array();
241 }
242
243 /**
244 * @return Status
245 */
246 protected function doPrecheck( array &$predicates ) {
247 return Status::newGood();
248 }
249
250 /**
251 * @return Status
252 */
253 protected function doAttempt() {
254 return Status::newGood();
255 }
256
257 /**
258 * Check for errors with regards to the destination file already existing.
259 * This also updates the destSameAsSource and sourceSha1 member variables.
260 * A bad status will be returned if there is no chance it can be overwritten.
261 *
262 * @param $predicates Array
263 * @return Status
264 */
265 protected function precheckDestExistence( array $predicates ) {
266 $status = Status::newGood();
267 // Get hash of source file/string and the destination file
268 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
269 if ( $this->sourceSha1 === null ) { // file in storage?
270 $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
271 }
272 $this->destSameAsSource = false;
273 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
274 if ( $this->getParam( 'overwrite' ) ) {
275 return $status; // OK
276 } elseif ( $this->getParam( 'overwriteSame' ) ) {
277 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
278 // Check if hashes are valid and match each other...
279 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
280 $status->fatal( 'backend-fail-hashes' );
281 } elseif ( $this->sourceSha1 !== $dhash ) {
282 // Give an error if the files are not identical
283 $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
284 } else {
285 $this->destSameAsSource = true; // OK
286 }
287 return $status; // do nothing; either OK or bad status
288 } else {
289 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
290 return $status;
291 }
292 }
293 return $status;
294 }
295
296 /**
297 * precheckDestExistence() helper function to get the source file SHA-1.
298 * Subclasses should overwride this iff the source is not in storage.
299 *
300 * @return string|bool Returns false on failure
301 */
302 protected function getSourceSha1Base36() {
303 return null; // N/A
304 }
305
306 /**
307 * Check if a file will exist in storage when this operation is attempted
308 *
309 * @param $source string Storage path
310 * @param $predicates Array
311 * @return bool
312 */
313 final protected function fileExists( $source, array $predicates ) {
314 if ( isset( $predicates['exists'][$source] ) ) {
315 return $predicates['exists'][$source]; // previous op assures this
316 } else {
317 $params = array( 'src' => $source, 'latest' => $this->useLatest );
318 return $this->backend->fileExists( $params );
319 }
320 }
321
322 /**
323 * Get the SHA-1 of a file in storage when this operation is attempted
324 *
325 * @param $source string Storage path
326 * @param $predicates Array
327 * @return string|bool False on failure
328 */
329 final protected function fileSha1( $source, array $predicates ) {
330 if ( isset( $predicates['sha1'][$source] ) ) {
331 return $predicates['sha1'][$source]; // previous op assures this
332 } else {
333 $params = array( 'src' => $source, 'latest' => $this->useLatest );
334 return $this->backend->getFileSha1Base36( $params );
335 }
336 }
337
338 /**
339 * Log a file operation failure and preserve any temp files
340 *
341 * @param $action string
342 * @return void
343 */
344 final protected function logFailure( $action ) {
345 $params = $this->params;
346 $params['failedAction'] = $action;
347 try {
348 wfDebugLog( 'FileOperation',
349 get_class( $this ) . ' failed:' . serialize( $params ) );
350 } catch ( Exception $e ) {
351 // bad config? debug log error?
352 }
353 }
354 }
355
356 /**
357 * FileOp helper class to expand PHP execution time for a function.
358 * On construction, set_time_limit() is called and set to $seconds.
359 * When the object goes out of scope, the timer is restarted, with
360 * the original time limit minus the time the object existed.
361 */
362 class FileOpScopedPHPTimeout {
363 protected $startTime; // float; seconds
364 protected $oldTimeout; // integer; seconds
365
366 protected static $stackDepth = 0; // integer
367 protected static $totalCalls = 0; // integer
368 protected static $totalElapsed = 0; // float; seconds
369
370 /* Prevent callers in infinite loops from running forever */
371 const MAX_TOTAL_CALLS = 1000000;
372 const MAX_TOTAL_TIME = 300; // seconds
373
374 /**
375 * @param $seconds integer
376 */
377 public function __construct( $seconds ) {
378 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
379 if ( self::$totalCalls >= self::MAX_TOTAL_CALLS ) {
380 trigger_error( "Maximum invocations of " . __CLASS__ . " exceeded." );
381 } elseif ( self::$totalElapsed >= self::MAX_TOTAL_TIME ) {
382 trigger_error( "Time limit within invocations of " . __CLASS__ . " exceeded." );
383 } elseif ( self::$stackDepth > 0 ) { // recursion guard
384 trigger_error( "Resursive invocation of " . __CLASS__ . " attempted." );
385 } else {
386 $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
387 $this->startTime = microtime( true );
388 ++self::$stackDepth;
389 ++self::$totalCalls; // proof against < 1us scopes
390 }
391 }
392 }
393
394 /**
395 * Restore the original timeout.
396 * This does not account for the timer value on __construct().
397 */
398 public function __destruct() {
399 if ( $this->oldTimeout ) {
400 $elapsed = microtime( true ) - $this->startTime;
401 // Note: a limit of 0 is treated as "forever"
402 set_time_limit( max( 1, $this->oldTimeout - (int)$elapsed ) );
403 // If each scoped timeout is for less than one second, we end up
404 // restoring the original timeout without any decrease in value.
405 // Thus web scripts in an infinite loop can run forever unless we
406 // take some measures to prevent this. Track total time and calls.
407 self::$totalElapsed += $elapsed;
408 --self::$stackDepth;
409 }
410 }
411 }
412
413 /**
414 * Store a file into the backend from a file on the file system.
415 * Parameters similar to FileBackendStore::storeInternal(), which include:
416 * src : source path on file system
417 * dst : destination storage path
418 * overwrite : do nothing and pass if an identical file exists at destination
419 * overwriteSame : override any existing file at destination
420 */
421 class StoreFileOp extends FileOp {
422 protected function allowedParams() {
423 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
424 }
425
426 protected function doPrecheck( array &$predicates ) {
427 $status = Status::newGood();
428 // Check if the source file exists on the file system
429 if ( !is_file( $this->params['src'] ) ) {
430 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
431 return $status;
432 // Check if the source file is too big
433 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
434 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
435 return $status;
436 // Check if a file can be placed at the destination
437 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
438 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
439 return $status;
440 }
441 // Check if destination file exists
442 $status->merge( $this->precheckDestExistence( $predicates ) );
443 if ( $status->isOK() ) {
444 // Update file existence predicates
445 $predicates['exists'][$this->params['dst']] = true;
446 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
447 }
448 return $status; // safe to call attempt()
449 }
450
451 protected function doAttempt() {
452 $status = Status::newGood();
453 // Store the file at the destination
454 if ( !$this->destSameAsSource ) {
455 $status->merge( $this->backend->storeInternal( $this->params ) );
456 }
457 return $status;
458 }
459
460 protected function getSourceSha1Base36() {
461 wfSuppressWarnings();
462 $hash = sha1_file( $this->params['src'] );
463 wfRestoreWarnings();
464 if ( $hash !== false ) {
465 $hash = wfBaseConvert( $hash, 16, 36, 31 );
466 }
467 return $hash;
468 }
469
470 public function storagePathsChanged() {
471 return array( $this->params['dst'] );
472 }
473 }
474
475 /**
476 * Create a file in the backend with the given content.
477 * Parameters similar to FileBackendStore::createInternal(), which include:
478 * content : the raw file contents
479 * dst : destination storage path
480 * overwrite : do nothing and pass if an identical file exists at destination
481 * overwriteSame : override any existing file at destination
482 */
483 class CreateFileOp extends FileOp {
484 protected function allowedParams() {
485 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
486 }
487
488 protected function doPrecheck( array &$predicates ) {
489 $status = Status::newGood();
490 // Check if the source data is too big
491 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
492 $status->fatal( 'backend-fail-create', $this->params['dst'] );
493 return $status;
494 // Check if a file can be placed at the destination
495 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
496 $status->fatal( 'backend-fail-create', $this->params['dst'] );
497 return $status;
498 }
499 // Check if destination file exists
500 $status->merge( $this->precheckDestExistence( $predicates ) );
501 if ( $status->isOK() ) {
502 // Update file existence predicates
503 $predicates['exists'][$this->params['dst']] = true;
504 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
505 }
506 return $status; // safe to call attempt()
507 }
508
509 protected function doAttempt() {
510 $status = Status::newGood();
511 // Create the file at the destination
512 if ( !$this->destSameAsSource ) {
513 $status->merge( $this->backend->createInternal( $this->params ) );
514 }
515 return $status;
516 }
517
518 protected function getSourceSha1Base36() {
519 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
520 }
521
522 public function storagePathsChanged() {
523 return array( $this->params['dst'] );
524 }
525 }
526
527 /**
528 * Copy a file from one storage path to another in the backend.
529 * Parameters similar to FileBackendStore::copyInternal(), which include:
530 * src : source storage path
531 * dst : destination storage path
532 * overwrite : do nothing and pass if an identical file exists at destination
533 * overwriteSame : override any existing file at destination
534 */
535 class CopyFileOp extends FileOp {
536 protected function allowedParams() {
537 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
538 }
539
540 protected function doPrecheck( array &$predicates ) {
541 $status = Status::newGood();
542 // Check if the source file exists
543 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
544 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
545 return $status;
546 // Check if a file can be placed at the destination
547 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
548 $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
549 return $status;
550 }
551 // Check if destination file exists
552 $status->merge( $this->precheckDestExistence( $predicates ) );
553 if ( $status->isOK() ) {
554 // Update file existence predicates
555 $predicates['exists'][$this->params['dst']] = true;
556 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
557 }
558 return $status; // safe to call attempt()
559 }
560
561 protected function doAttempt() {
562 $status = Status::newGood();
563 // Do nothing if the src/dst paths are the same
564 if ( $this->params['src'] !== $this->params['dst'] ) {
565 // Copy the file into the destination
566 if ( !$this->destSameAsSource ) {
567 $status->merge( $this->backend->copyInternal( $this->params ) );
568 }
569 }
570 return $status;
571 }
572
573 public function storagePathsRead() {
574 return array( $this->params['src'] );
575 }
576
577 public function storagePathsChanged() {
578 return array( $this->params['dst'] );
579 }
580 }
581
582 /**
583 * Move a file from one storage path to another in the backend.
584 * Parameters similar to FileBackendStore::moveInternal(), which include:
585 * src : source storage path
586 * dst : destination storage path
587 * overwrite : do nothing and pass if an identical file exists at destination
588 * overwriteSame : override any existing file at destination
589 */
590 class MoveFileOp extends FileOp {
591 protected function allowedParams() {
592 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
593 }
594
595 protected function doPrecheck( array &$predicates ) {
596 $status = Status::newGood();
597 // Check if the source file exists
598 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
599 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
600 return $status;
601 // Check if a file can be placed at the destination
602 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
603 $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
604 return $status;
605 }
606 // Check if destination file exists
607 $status->merge( $this->precheckDestExistence( $predicates ) );
608 if ( $status->isOK() ) {
609 // Update file existence predicates
610 $predicates['exists'][$this->params['src']] = false;
611 $predicates['sha1'][$this->params['src']] = false;
612 $predicates['exists'][$this->params['dst']] = true;
613 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
614 }
615 return $status; // safe to call attempt()
616 }
617
618 protected function doAttempt() {
619 $status = Status::newGood();
620 // Do nothing if the src/dst paths are the same
621 if ( $this->params['src'] !== $this->params['dst'] ) {
622 if ( !$this->destSameAsSource ) {
623 // Move the file into the destination
624 $status->merge( $this->backend->moveInternal( $this->params ) );
625 } else {
626 // Just delete source as the destination needs no changes
627 $params = array( 'src' => $this->params['src'] );
628 $status->merge( $this->backend->deleteInternal( $params ) );
629 }
630 }
631 return $status;
632 }
633
634 public function storagePathsRead() {
635 return array( $this->params['src'] );
636 }
637
638 public function storagePathsChanged() {
639 return array( $this->params['dst'] );
640 }
641 }
642
643 /**
644 * Delete a file at the given storage path from the backend.
645 * Parameters similar to FileBackendStore::deleteInternal(), which include:
646 * src : source storage path
647 * ignoreMissingSource : don't return an error if the file does not exist
648 */
649 class DeleteFileOp extends FileOp {
650 protected function allowedParams() {
651 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
652 }
653
654 protected $needsDelete = true;
655
656 protected function doPrecheck( array &$predicates ) {
657 $status = Status::newGood();
658 // Check if the source file exists
659 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
660 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
661 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
662 return $status;
663 }
664 $this->needsDelete = false;
665 }
666 // Update file existence predicates
667 $predicates['exists'][$this->params['src']] = false;
668 $predicates['sha1'][$this->params['src']] = false;
669 return $status; // safe to call attempt()
670 }
671
672 protected function doAttempt() {
673 $status = Status::newGood();
674 if ( $this->needsDelete ) {
675 // Delete the source file
676 $status->merge( $this->backend->deleteInternal( $this->params ) );
677 }
678 return $status;
679 }
680
681 public function storagePathsChanged() {
682 return array( $this->params['src'] );
683 }
684 }
685
686 /**
687 * Placeholder operation that has no params and does nothing
688 */
689 class NullFileOp extends FileOp {}