includes/filerepo/backend/FileOp.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * Helper class for representing operations with transaction support.
  10  * FileBackend::doOperations() will require these classes for supported operations.
  11  * Do not use this class from places outside FileBackend.
  12  *
  13  * Use of large fields should be avoided as we want to support
  14  * potentially many FileOp classes in large arrays in memory.
  15  *
  16  * @ingroup FileBackend
  17  * @since 1.19
  18  */
  19 abstract class FileOp {
  20         /** @var Array */
  21         protected $params = array();
  22         /** @var FileBackendBase */
  23         protected $backend;
  24
  25         protected $state = self::STATE_NEW; // integer
  26         protected $failed = false; // boolean
  27         protected $useLatest = true; // boolean
  28
  29         protected $sourceSha1; // string
  30         protected $destSameAsSource; // boolean
  31
  32         /* Object life-cycle */
  33         const STATE_NEW = 1;
  34         const STATE_CHECKED = 2;
  35         const STATE_ATTEMPTED = 3;
  36
  37         /* Timeout related parameters */
  38         const MAX_BATCH_SIZE = 1000;
  39         const TIME_LIMIT_SEC = 300; // 5 minutes
  40
  41         /**
  42          * Build a new file operation transaction
  43          *
  44          * @params $backend FileBackend
  45          * @params $params Array
  46          */
  47         final public function __construct( FileBackendBase $backend, array $params ) {
  48                 $this->backend = $backend;
  49                 foreach ( $this->allowedParams() as $name ) {
  50                         if ( isset( $params[$name] ) ) {
  51                                 $this->params[$name] = $params[$name];
  52                         }
  53                 }
  54                 $this->params = $params;
  55         }
  56
  57         /**
  58          * Allow stale data for file reads and existence checks
  59          *
  60          * @return void
  61          */
  62         final protected function allowStaleReads() {
  63                 $this->useLatest = false;
  64         }
  65
  66         /**
  67          * Attempt a series of file operations.
  68          * Callers are responsible for handling file locking.
  69          *
  70          * $opts is an array of options, including:
  71          * 'force'      : Errors that would normally cause a rollback do not.
  72          *                The remaining operations are still attempted if any fail.
  73          * 'allowStale' : Don't require the latest available data.
  74          *                This can increase performance for non-critical writes.
  75          *                This has no effect unless the 'force' flag is set.
  76          *
  77          * @param $performOps Array List of FileOp operations
  78          * @param $opts Array Batch operation options
  79          * @return Status
  80          */
  81         final public static function attemptBatch( array $performOps, array $opts ) {
  82                 $status = Status::newGood();
  83
  84                 $allowStale = !empty( $opts['allowStale'] );
  85                 $ignoreErrors = !empty( $opts['force'] );
  86
  87                 $n = count( $performOps );
  88                 if ( $n > self::MAX_BATCH_SIZE ) {
  89                         $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
  90                         return $status;
  91                 }
  92
  93                 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
  94                 // Do pre-checks for each operation; abort on failure...
  95                 foreach ( $performOps as $index => $fileOp ) {
  96                         if ( $allowStale ) {
  97                                 $fileOp->allowStaleReads(); // allow potentially stale reads
  98                         }
  99                         $subStatus = $fileOp->precheck( $predicates );
 100                         $status->merge( $subStatus );
 101                         if ( !$subStatus->isOK() ) { // operation failed?
 102                                 $status->success[$index] = false;
 103                                 ++$status->failCount;
 104                                 if ( !$ignoreErrors ) {
 105                                         return $status; // abort
 106                                 }
 107                         }
 108                 }
 109
 110                 // Restart PHP's execution timer and set the timeout to safe amount.
 111                 // This handles cases where the operations take a long time or where we are
 112                 // already running low on time left. The old timeout is restored afterwards.
 113                 $scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
 114
 115                 // Attempt each operation...
 116                 foreach ( $performOps as $index => $fileOp ) {
 117                         if ( $fileOp->failed() ) {
 118                                 continue; // nothing to do
 119                         }
 120                         $subStatus = $fileOp->attempt();
 121                         $status->merge( $subStatus );
 122                         if ( $subStatus->isOK() ) {
 123                                 $status->success[$index] = true;
 124                                 ++$status->successCount;
 125                         } else {
 126                                 $status->success[$index] = false;
 127                                 ++$status->failCount;
 128                                 if ( !$ignoreErrors ) {
 129                                         // Log remaining ops as failed for recovery...
 130                                         for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
 131                                                 $performOps[$i]->logFailure( 'attempt_aborted' );
 132                                         }
 133                                         return $status; // bail out
 134                                 }
 135                         }
 136                 }
 137
 138                 return $status;
 139         }
 140
 141         /**
 142          * Get the value of the parameter with the given name
 143          *
 144          * @param $name string
 145          * @return mixed Returns null if the parameter is not set
 146          */
 147         final public function getParam( $name ) {
 148                 return isset( $this->params[$name] ) ? $this->params[$name] : null;
 149         }
 150
 151         /**
 152          * Check if this operation failed precheck() or attempt()
 153          *
 154          * @return bool
 155          */
 156         final public function failed() {
 157                 return $this->failed;
 158         }
 159
 160         /**
 161          * Get a new empty predicates array for precheck()
 162          *
 163          * @return Array
 164          */
 165         final public static function newPredicates() {
 166                 return array( 'exists' => array(), 'sha1' => array() );
 167         }
 168
 169         /**
 170          * Check preconditions of the operation without writing anything
 171          *
 172          * @param $predicates Array
 173          * @return Status
 174          */
 175         final public function precheck( array &$predicates ) {
 176                 if ( $this->state !== self::STATE_NEW ) {
 177                         return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
 178                 }
 179                 $this->state = self::STATE_CHECKED;
 180                 $status = $this->doPrecheck( $predicates );
 181                 if ( !$status->isOK() ) {
 182                         $this->failed = true;
 183                 }
 184                 return $status;
 185         }
 186
 187         /**
 188          * Attempt the operation, backing up files as needed; this must be reversible
 189          *
 190          * @return Status
 191          */
 192         final public function attempt() {
 193                 if ( $this->state !== self::STATE_CHECKED ) {
 194                         return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
 195                 } elseif ( $this->failed ) { // failed precheck
 196                         return Status::newFatal( 'fileop-fail-attempt-precheck' );
 197                 }
 198                 $this->state = self::STATE_ATTEMPTED;
 199                 $status = $this->doAttempt();
 200                 if ( !$status->isOK() ) {
 201                         $this->failed = true;
 202                         $this->logFailure( 'attempt' );
 203                 }
 204                 return $status;
 205         }
 206
 207         /**
 208          * Get a list of storage paths read from for this operation
 209          *
 210          * @return Array
 211          */
 212         public function storagePathsRead() {
 213                 return array();
 214         }
 215
 216         /**
 217          * Get a list of storage paths written to for this operation
 218          *
 219          * @return Array
 220          */
 221         public function storagePathsChanged() {
 222                 return array();
 223         }
 224
 225         /**
 226          * @return Array List of allowed parameters
 227          */
 228         protected function allowedParams() {
 229                 return array();
 230         }
 231
 232         /**
 233          * @return Status
 234          */
 235         protected function doPrecheck( array &$predicates ) {
 236                 return Status::newGood();
 237         }
 238
 239         /**
 240          * @return Status
 241          */
 242         abstract protected function doAttempt();
 243
 244         /**
 245          * Check for errors with regards to the destination file already existing.
 246          * This also updates the destSameAsSource and sourceSha1 member variables.
 247          * A bad status will be returned if there is no chance it can be overwritten.
 248          *
 249          * @param $predicates Array
 250          * @return Status
 251          */
 252         protected function precheckDestExistence( array $predicates ) {
 253                 $status = Status::newGood();
 254                 // Get hash of source file/string and the destination file
 255                 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
 256                 if ( $this->sourceSha1 === null ) { // file in storage?
 257                         $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
 258                 }
 259                 $this->destSameAsSource = false;
 260                 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
 261                         if ( $this->getParam( 'overwrite' ) ) {
 262                                 return $status; // OK
 263                         } elseif ( $this->getParam( 'overwriteSame' ) ) {
 264                                 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
 265                                 // Check if hashes are valid and match each other...
 266                                 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
 267                                         $status->fatal( 'backend-fail-hashes' );
 268                                 } elseif ( $this->sourceSha1 !== $dhash ) {
 269                                         // Give an error if the files are not identical
 270                                         $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
 271                                 } else {
 272                                         $this->destSameAsSource = true; // OK
 273                                 }
 274                                 return $status; // do nothing; either OK or bad status
 275                         } else {
 276                                 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
 277                                 return $status;
 278                         }
 279                 }
 280                 return $status;
 281         }
 282
 283         /**
 284          * precheckDestExistence() helper function to get the source file SHA-1.
 285          * Subclasses should overwride this iff the source is not in storage.
 286          *
 287          * @return string|false Returns false on failure
 288          */
 289         protected function getSourceSha1Base36() {
 290                 return null; // N/A
 291         }
 292
 293         /**
 294          * Check if a file will exist in storage when this operation is attempted
 295          *
 296          * @param $source string Storage path
 297          * @param $predicates Array
 298          * @return bool
 299          */
 300         final protected function fileExists( $source, array $predicates ) {
 301                 if ( isset( $predicates['exists'][$source] ) ) {
 302                         return $predicates['exists'][$source]; // previous op assures this
 303                 } else {
 304                         $params = array( 'src' => $source, 'latest' => $this->useLatest );
 305                         return $this->backend->fileExists( $params );
 306                 }
 307         }
 308
 309         /**
 310          * Get the SHA-1 of a file in storage when this operation is attempted
 311          *
 312          * @param $source string Storage path
 313          * @param $predicates Array
 314          * @return string|false
 315          */
 316         final protected function fileSha1( $source, array $predicates ) {
 317                 if ( isset( $predicates['sha1'][$source] ) ) {
 318                         return $predicates['sha1'][$source]; // previous op assures this
 319                 } else {
 320                         $params = array( 'src' => $source, 'latest' => $this->useLatest );
 321                         return $this->backend->getFileSha1Base36( $params );
 322                 }
 323         }
 324
 325         /**
 326          * Log a file operation failure and preserve any temp files
 327          *
 328          * @param $action string
 329          * @return void
 330          */
 331         final protected function logFailure( $action ) {
 332                 $params = $this->params;
 333                 $params['failedAction'] = $action;
 334                 try {
 335                         wfDebugLog( 'FileOperation',
 336                                 get_class( $this ) . ' failed:' . serialize( $params ) );
 337                 } catch ( Exception $e ) {
 338                         // bad config? debug log error?
 339                 }
 340         }
 341 }
 342
 343 /**
 344  * FileOp helper class to expand PHP execution time for a function.
 345  * On construction, set_time_limit() is called and set to $seconds.
 346  * When the object goes out of scope, the timer is restarted, with
 347  * the original time limit minus the time the object existed.
 348  */
 349 class FileOpScopedPHPTimeout {
 350         protected $startTime; // integer seconds
 351         protected $oldTimeout; // integer seconds
 352
 353         /**
 354          * @param $seconds integer
 355          */
 356         public function __construct( $seconds ) {
 357                 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
 358                         $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
 359                 }
 360                 $this->startTime = time();
 361         }
 362
 363         /*
 364          * Restore the original timeout.
 365          * This does not account for the timer value on __construct().
 366          */
 367         public function __destruct() {
 368                 if ( $this->oldTimeout ) {
 369                         $elapsed = time() - $this->startTime;
 370                         // Note: a limit of 0 is treated as "forever"
 371                         set_time_limit( max( 1, $this->oldTimeout - $elapsed ) );
 372                 }
 373         }
 374 }
 375
 376 /**
 377  * Store a file into the backend from a file on the file system.
 378  * Parameters similar to FileBackend::storeInternal(), which include:
 379  *     src           : source path on file system
 380  *     dst           : destination storage path
 381  *     overwrite     : do nothing and pass if an identical file exists at destination
 382  *     overwriteSame : override any existing file at destination
 383  */
 384 class StoreFileOp extends FileOp {
 385         protected function allowedParams() {
 386                 return array( 'src', 'dst', 'overwrite', 'overwriteSame' );
 387         }
 388
 389         protected function doPrecheck( array &$predicates ) {
 390                 $status = Status::newGood();
 391                 // Check if the source file exists on the file system
 392                 if ( !is_file( $this->params['src'] ) ) {
 393                         $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 394                         return $status;
 395                 // Check if the source file is too big
 396                 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
 397                         $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
 398                         return $status;
 399                 // Check if a file can be placed at the destination
 400                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 401                         $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
 402                         return $status;
 403                 }
 404                 // Check if destination file exists
 405                 $status->merge( $this->precheckDestExistence( $predicates ) );
 406                 if ( $status->isOK() ) {
 407                         // Update file existence predicates
 408                         $predicates['exists'][$this->params['dst']] = true;
 409                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 410                 }
 411                 return $status; // safe to call attempt()
 412         }
 413
 414         protected function doAttempt() {
 415                 $status = Status::newGood();
 416                 // Store the file at the destination
 417                 if ( !$this->destSameAsSource ) {
 418                         $status->merge( $this->backend->storeInternal( $this->params ) );
 419                 }
 420                 return $status;
 421         }
 422
 423         protected function getSourceSha1Base36() {
 424                 wfSuppressWarnings();
 425                 $hash = sha1_file( $this->params['src'] );
 426                 wfRestoreWarnings();
 427                 if ( $hash !== false ) {
 428                         $hash = wfBaseConvert( $hash, 16, 36, 31 );
 429                 }
 430                 return $hash;
 431         }
 432
 433         public function storagePathsChanged() {
 434                 return array( $this->params['dst'] );
 435         }
 436 }
 437
 438 /**
 439  * Create a file in the backend with the given content.
 440  * Parameters similar to FileBackend::createInternal(), which include:
 441  *     content       : a string of raw file content. Let unset to create an empty file.
 442  *     dst           : destination storage path
 443  *     overwrite     : do nothing and pass if an identical file exists at destination
 444  *     overwriteSame : override any existing file at destination
 445  */
 446 class CreateFileOp extends FileOp {
 447         protected function allowedParams() {
 448                 return array( 'content', 'dst', 'overwrite', 'overwriteSame' );
 449         }
 450
 451         protected function doPrecheck( array &$predicates ) {
 452                 $status = Status::newGood();
 453                 // Check if the source data is too big
 454                 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
 455                         $status->fatal( 'backend-fail-create', $this->params['dst'] );
 456                         return $status;
 457                 // Check if a file can be placed at the destination
 458                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 459                         $status->fatal( 'backend-fail-create', $this->params['dst'] );
 460                         return $status;
 461                 }
 462                 // Check if destination file exists
 463                 $status->merge( $this->precheckDestExistence( $predicates ) );
 464                 if ( $status->isOK() ) {
 465                         // Update file existence predicates
 466                         $predicates['exists'][$this->params['dst']] = true;
 467                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 468                 }
 469                 return $status; // safe to call attempt()
 470         }
 471
 472         protected function doAttempt() {
 473                 $status = Status::newGood();
 474                 // Create the file at the destination
 475                 if ( !$this->destSameAsSource ) {
 476                         $status->merge( $this->backend->createInternal( $this->params ) );
 477                 }
 478                 return $status;
 479         }
 480
 481         protected function getSourceSha1Base36() {
 482                 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
 483         }
 484
 485         public function storagePathsChanged() {
 486                 return array( $this->params['dst'] );
 487         }
 488 }
 489
 490 /**
 491  * Copy a file from one storage path to another in the backend.
 492  * Parameters similar to FileBackend::copyInternal(), which include:
 493  *     src           : source storage path
 494  *     dst           : destination storage path
 495  *     overwrite     : do nothing and pass if an identical file exists at destination
 496  *     overwriteSame : override any existing file at destination
 497  */
 498 class CopyFileOp extends FileOp {
 499         protected function allowedParams() {
 500                 return array( 'src', 'dst', 'overwrite', 'overwriteSame' );
 501         }
 502
 503         protected function doPrecheck( array &$predicates ) {
 504                 $status = Status::newGood();
 505                 // Check if the source file exists
 506                 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
 507                         $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 508                         return $status;
 509                 // Check if a file can be placed at the destination
 510                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 511                         $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
 512                         return $status;
 513                 }
 514                 // Check if destination file exists
 515                 $status->merge( $this->precheckDestExistence( $predicates ) );
 516                 if ( $status->isOK() ) {
 517                         // Update file existence predicates
 518                         $predicates['exists'][$this->params['dst']] = true;
 519                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 520                 }
 521                 return $status; // safe to call attempt()
 522         }
 523
 524         protected function doAttempt() {
 525                 $status = Status::newGood();
 526                 // Do nothing if the src/dst paths are the same
 527                 if ( $this->params['src'] !== $this->params['dst'] ) {
 528                         // Copy the file into the destination
 529                         if ( !$this->destSameAsSource ) {
 530                                 $status->merge( $this->backend->copyInternal( $this->params ) );
 531                         }
 532                 }
 533                 return $status;
 534         }
 535
 536         public function storagePathsRead() {
 537                 return array( $this->params['src'] );
 538         }
 539
 540         public function storagePathsChanged() {
 541                 return array( $this->params['dst'] );
 542         }
 543 }
 544
 545 /**
 546  * Move a file from one storage path to another in the backend.
 547  * Parameters similar to FileBackend::moveInternal(), which include:
 548  *     src           : source storage path
 549  *     dst           : destination storage path
 550  *     overwrite     : do nothing and pass if an identical file exists at destination
 551  *     overwriteSame : override any existing file at destination
 552  */
 553 class MoveFileOp extends FileOp {
 554         protected function allowedParams() {
 555                 return array( 'src', 'dst', 'overwrite', 'overwriteSame' );
 556         }
 557
 558         protected function doPrecheck( array &$predicates ) {
 559                 $status = Status::newGood();
 560                 // Check if the source file exists
 561                 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
 562                         $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 563                         return $status;
 564                 // Check if a file can be placed at the destination
 565                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 566                         $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
 567                         return $status;
 568                 }
 569                 // Check if destination file exists
 570                 $status->merge( $this->precheckDestExistence( $predicates ) );
 571                 if ( $status->isOK() ) {
 572                         // Update file existence predicates
 573                         $predicates['exists'][$this->params['src']] = false;
 574                         $predicates['sha1'][$this->params['src']] = false;
 575                         $predicates['exists'][$this->params['dst']] = true;
 576                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 577                 }
 578                 return $status; // safe to call attempt()
 579         }
 580
 581         protected function doAttempt() {
 582                 $status = Status::newGood();
 583                 // Do nothing if the src/dst paths are the same
 584                 if ( $this->params['src'] !== $this->params['dst'] ) {
 585                         if ( !$this->destSameAsSource ) {
 586                                 // Move the file into the destination
 587                                 $status->merge( $this->backend->moveInternal( $this->params ) );
 588                         } else {
 589                                 // Just delete source as the destination needs no changes
 590                                 $params = array( 'src' => $this->params['src'] );
 591                                 $status->merge( $this->backend->deleteInternal( $params ) );
 592                         }
 593                 }
 594                 return $status;
 595         }
 596
 597         public function storagePathsRead() {
 598                 return array( $this->params['src'] );
 599         }
 600
 601         public function storagePathsChanged() {
 602                 return array( $this->params['dst'] );
 603         }
 604 }
 605
 606 /**
 607  * Delete a file at the storage path.
 608  * Parameters similar to FileBackend::deleteInternal(), which include:
 609  *     src                 : source storage path
 610  *     ignoreMissingSource : don't return an error if the file does not exist
 611  */
 612 class DeleteFileOp extends FileOp {
 613         protected $needsDelete = true;
 614
 615         protected function allowedParams() {
 616                 return array( 'src', 'ignoreMissingSource' );
 617         }
 618
 619         protected function doPrecheck( array &$predicates ) {
 620                 $status = Status::newGood();
 621                 // Check if the source file exists
 622                 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
 623                         if ( !$this->getParam( 'ignoreMissingSource' ) ) {
 624                                 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 625                                 return $status;
 626                         }
 627                         $this->needsDelete = false;
 628                 }
 629                 // Update file existence predicates
 630                 $predicates['exists'][$this->params['src']] = false;
 631                 $predicates['sha1'][$this->params['src']] = false;
 632                 return $status; // safe to call attempt()
 633         }
 634
 635         protected function doAttempt() {
 636                 $status = Status::newGood();
 637                 if ( $this->needsDelete ) {
 638                         // Delete the source file
 639                         $status->merge( $this->backend->deleteInternal( $this->params ) );
 640                 }
 641                 return $status;
 642         }
 643
 644         public function storagePathsChanged() {
 645                 return array( $this->params['src'] );
 646         }
 647 }
 648
 649 /**
 650  * Placeholder operation that has no params and does nothing
 651  */
 652 class NullFileOp extends FileOp {
 653         protected function doAttempt() {
 654                 return Status::newGood();
 655         }
 656 }