includes/filerepo/backend/FileOp.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * Helper class for representing operations with transaction support.
  10  * Do not use this class from places outside FileBackend.
  11  *
  12  * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
  13  * FileOp objects should be lightweight in order to support large arrays in memory.
  14  *
  15  * @ingroup FileBackend
  16  * @since 1.19
  17  */
  18 abstract class FileOp {
  19         /** @var Array */
  20         protected $params = array();
  21         /** @var FileBackendBase */
  22         protected $backend;
  23
  24         protected $state = self::STATE_NEW; // integer
  25         protected $failed = false; // boolean
  26         protected $useLatest = true; // boolean
  27
  28         protected $sourceSha1; // string
  29         protected $destSameAsSource; // boolean
  30
  31         /* Operation parameters */
  32         protected static $requiredParams = array();
  33         protected static $optionalParams = array();
  34
  35         /* Object life-cycle */
  36         const STATE_NEW = 1;
  37         const STATE_CHECKED = 2;
  38         const STATE_ATTEMPTED = 3;
  39
  40         /* Timeout related parameters */
  41         const MAX_BATCH_SIZE = 1000;
  42         const TIME_LIMIT_SEC = 300; // 5 minutes
  43
  44         /**
  45          * Build a new file operation transaction
  46          *
  47          * @params $backend FileBackend
  48          * @params $params Array
  49          * @throws MWException
  50          */
  51         final public function __construct( FileBackendBase $backend, array $params ) {
  52                 $this->backend = $backend;
  53                 $class = get_class( $this ); // simulate LSB
  54                 foreach ( $class::$requiredParams as $name ) {
  55                         if ( isset( $params[$name] ) ) {
  56                                 $this->params[$name] = $params[$name];
  57                         } else {
  58                                 throw new MWException( "File operation missing parameter '$name'." );
  59                         }
  60                 }
  61                 foreach ( $class::$optionalParams as $name ) {
  62                         if ( isset( $params[$name] ) ) {
  63                                 $this->params[$name] = $params[$name];
  64                         }
  65                 }
  66                 $this->params = $params;
  67         }
  68
  69         /**
  70          * Allow stale data for file reads and existence checks
  71          *
  72          * @return void
  73          */
  74         final protected function allowStaleReads() {
  75                 $this->useLatest = false;
  76         }
  77
  78         /**
  79          * Attempt a series of file operations.
  80          * Callers are responsible for handling file locking.
  81          *
  82          * $opts is an array of options, including:
  83          * 'force'      : Errors that would normally cause a rollback do not.
  84          *                The remaining operations are still attempted if any fail.
  85          * 'allowStale' : Don't require the latest available data.
  86          *                This can increase performance for non-critical writes.
  87          *                This has no effect unless the 'force' flag is set.
  88          *
  89          * @param $performOps Array List of FileOp operations
  90          * @param $opts Array Batch operation options
  91          * @return Status
  92          */
  93         final public static function attemptBatch( array $performOps, array $opts ) {
  94                 $status = Status::newGood();
  95
  96                 $allowStale = !empty( $opts['allowStale'] );
  97                 $ignoreErrors = !empty( $opts['force'] );
  98
  99                 $n = count( $performOps );
 100                 if ( $n > self::MAX_BATCH_SIZE ) {
 101                         $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
 102                         return $status;
 103                 }
 104
 105                 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
 106                 // Do pre-checks for each operation; abort on failure...
 107                 foreach ( $performOps as $index => $fileOp ) {
 108                         if ( $allowStale ) {
 109                                 $fileOp->allowStaleReads(); // allow potentially stale reads
 110                         }
 111                         $subStatus = $fileOp->precheck( $predicates );
 112                         $status->merge( $subStatus );
 113                         if ( !$subStatus->isOK() ) { // operation failed?
 114                                 $status->success[$index] = false;
 115                                 ++$status->failCount;
 116                                 if ( !$ignoreErrors ) {
 117                                         return $status; // abort
 118                                 }
 119                         }
 120                 }
 121
 122                 // Restart PHP's execution timer and set the timeout to safe amount.
 123                 // This handles cases where the operations take a long time or where we are
 124                 // already running low on time left. The old timeout is restored afterwards.
 125                 $scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
 126
 127                 // Attempt each operation...
 128                 foreach ( $performOps as $index => $fileOp ) {
 129                         if ( $fileOp->failed() ) {
 130                                 continue; // nothing to do
 131                         }
 132                         $subStatus = $fileOp->attempt();
 133                         $status->merge( $subStatus );
 134                         if ( $subStatus->isOK() ) {
 135                                 $status->success[$index] = true;
 136                                 ++$status->successCount;
 137                         } else {
 138                                 $status->success[$index] = false;
 139                                 ++$status->failCount;
 140                                 if ( !$ignoreErrors ) {
 141                                         // Log remaining ops as failed for recovery...
 142                                         for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
 143                                                 $performOps[$i]->logFailure( 'attempt_aborted' );
 144                                         }
 145                                         return $status; // bail out
 146                                 }
 147                         }
 148                 }
 149
 150                 return $status;
 151         }
 152
 153         /**
 154          * Get the value of the parameter with the given name
 155          *
 156          * @param $name string
 157          * @return mixed Returns null if the parameter is not set
 158          */
 159         final public function getParam( $name ) {
 160                 return isset( $this->params[$name] ) ? $this->params[$name] : null;
 161         }
 162
 163         /**
 164          * Check if this operation failed precheck() or attempt()
 165          *
 166          * @return bool
 167          */
 168         final public function failed() {
 169                 return $this->failed;
 170         }
 171
 172         /**
 173          * Get a new empty predicates array for precheck()
 174          *
 175          * @return Array
 176          */
 177         final public static function newPredicates() {
 178                 return array( 'exists' => array(), 'sha1' => array() );
 179         }
 180
 181         /**
 182          * Check preconditions of the operation without writing anything
 183          *
 184          * @param $predicates Array
 185          * @return Status
 186          */
 187         final public function precheck( array &$predicates ) {
 188                 if ( $this->state !== self::STATE_NEW ) {
 189                         return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
 190                 }
 191                 $this->state = self::STATE_CHECKED;
 192                 $status = $this->doPrecheck( $predicates );
 193                 if ( !$status->isOK() ) {
 194                         $this->failed = true;
 195                 }
 196                 return $status;
 197         }
 198
 199         /**
 200          * Attempt the operation, backing up files as needed; this must be reversible
 201          *
 202          * @return Status
 203          */
 204         final public function attempt() {
 205                 if ( $this->state !== self::STATE_CHECKED ) {
 206                         return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
 207                 } elseif ( $this->failed ) { // failed precheck
 208                         return Status::newFatal( 'fileop-fail-attempt-precheck' );
 209                 }
 210                 $this->state = self::STATE_ATTEMPTED;
 211                 $status = $this->doAttempt();
 212                 if ( !$status->isOK() ) {
 213                         $this->failed = true;
 214                         $this->logFailure( 'attempt' );
 215                 }
 216                 return $status;
 217         }
 218
 219         /**
 220          * Get a list of storage paths read from for this operation
 221          *
 222          * @return Array
 223          */
 224         public function storagePathsRead() {
 225                 return array();
 226         }
 227
 228         /**
 229          * Get a list of storage paths written to for this operation
 230          *
 231          * @return Array
 232          */
 233         public function storagePathsChanged() {
 234                 return array();
 235         }
 236
 237         /**
 238          * @return Status
 239          */
 240         protected function doPrecheck( array &$predicates ) {
 241                 return Status::newGood();
 242         }
 243
 244         /**
 245          * @return Status
 246          */
 247         abstract protected function doAttempt();
 248
 249         /**
 250          * Check for errors with regards to the destination file already existing.
 251          * This also updates the destSameAsSource and sourceSha1 member variables.
 252          * A bad status will be returned if there is no chance it can be overwritten.
 253          *
 254          * @param $predicates Array
 255          * @return Status
 256          */
 257         protected function precheckDestExistence( array $predicates ) {
 258                 $status = Status::newGood();
 259                 // Get hash of source file/string and the destination file
 260                 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
 261                 if ( $this->sourceSha1 === null ) { // file in storage?
 262                         $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
 263                 }
 264                 $this->destSameAsSource = false;
 265                 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
 266                         if ( $this->getParam( 'overwrite' ) ) {
 267                                 return $status; // OK
 268                         } elseif ( $this->getParam( 'overwriteSame' ) ) {
 269                                 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
 270                                 // Check if hashes are valid and match each other...
 271                                 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
 272                                         $status->fatal( 'backend-fail-hashes' );
 273                                 } elseif ( $this->sourceSha1 !== $dhash ) {
 274                                         // Give an error if the files are not identical
 275                                         $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
 276                                 } else {
 277                                         $this->destSameAsSource = true; // OK
 278                                 }
 279                                 return $status; // do nothing; either OK or bad status
 280                         } else {
 281                                 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
 282                                 return $status;
 283                         }
 284                 }
 285                 return $status;
 286         }
 287
 288         /**
 289          * precheckDestExistence() helper function to get the source file SHA-1.
 290          * Subclasses should overwride this iff the source is not in storage.
 291          *
 292          * @return string|false Returns false on failure
 293          */
 294         protected function getSourceSha1Base36() {
 295                 return null; // N/A
 296         }
 297
 298         /**
 299          * Check if a file will exist in storage when this operation is attempted
 300          *
 301          * @param $source string Storage path
 302          * @param $predicates Array
 303          * @return bool
 304          */
 305         final protected function fileExists( $source, array $predicates ) {
 306                 if ( isset( $predicates['exists'][$source] ) ) {
 307                         return $predicates['exists'][$source]; // previous op assures this
 308                 } else {
 309                         $params = array( 'src' => $source, 'latest' => $this->useLatest );
 310                         return $this->backend->fileExists( $params );
 311                 }
 312         }
 313
 314         /**
 315          * Get the SHA-1 of a file in storage when this operation is attempted
 316          *
 317          * @param $source string Storage path
 318          * @param $predicates Array
 319          * @return string|false
 320          */
 321         final protected function fileSha1( $source, array $predicates ) {
 322                 if ( isset( $predicates['sha1'][$source] ) ) {
 323                         return $predicates['sha1'][$source]; // previous op assures this
 324                 } else {
 325                         $params = array( 'src' => $source, 'latest' => $this->useLatest );
 326                         return $this->backend->getFileSha1Base36( $params );
 327                 }
 328         }
 329
 330         /**
 331          * Log a file operation failure and preserve any temp files
 332          *
 333          * @param $action string
 334          * @return void
 335          */
 336         final protected function logFailure( $action ) {
 337                 $params = $this->params;
 338                 $params['failedAction'] = $action;
 339                 try {
 340                         wfDebugLog( 'FileOperation',
 341                                 get_class( $this ) . ' failed:' . serialize( $params ) );
 342                 } catch ( Exception $e ) {
 343                         // bad config? debug log error?
 344                 }
 345         }
 346 }
 347
 348 /**
 349  * FileOp helper class to expand PHP execution time for a function.
 350  * On construction, set_time_limit() is called and set to $seconds.
 351  * When the object goes out of scope, the timer is restarted, with
 352  * the original time limit minus the time the object existed.
 353  */
 354 class FileOpScopedPHPTimeout {
 355         protected $startTime; // integer seconds
 356         protected $oldTimeout; // integer seconds
 357
 358         /**
 359          * @param $seconds integer
 360          */
 361         public function __construct( $seconds ) {
 362                 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
 363                         $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
 364                 }
 365                 $this->startTime = time();
 366         }
 367
 368         /*
 369          * Restore the original timeout.
 370          * This does not account for the timer value on __construct().
 371          */
 372         public function __destruct() {
 373                 if ( $this->oldTimeout ) {
 374                         $elapsed = time() - $this->startTime;
 375                         // Note: a limit of 0 is treated as "forever"
 376                         set_time_limit( max( 1, $this->oldTimeout - $elapsed ) );
 377                 }
 378         }
 379 }
 380
 381 /**
 382  * Store a file into the backend from a file on the file system.
 383  * Parameters similar to FileBackend::storeInternal(), which include:
 384  *     src           : source path on file system
 385  *     dst           : destination storage path
 386  *     overwrite     : do nothing and pass if an identical file exists at destination
 387  *     overwriteSame : override any existing file at destination
 388  */
 389 class StoreFileOp extends FileOp {
 390         protected static $requiredParams = array( 'src', 'dst' );
 391         protected static $optionalParams = array( 'overwrite', 'overwriteSame' );
 392
 393         protected function doPrecheck( array &$predicates ) {
 394                 $status = Status::newGood();
 395                 // Check if the source file exists on the file system
 396                 if ( !is_file( $this->params['src'] ) ) {
 397                         $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 398                         return $status;
 399                 // Check if the source file is too big
 400                 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
 401                         $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
 402                         return $status;
 403                 // Check if a file can be placed at the destination
 404                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 405                         $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
 406                         return $status;
 407                 }
 408                 // Check if destination file exists
 409                 $status->merge( $this->precheckDestExistence( $predicates ) );
 410                 if ( $status->isOK() ) {
 411                         // Update file existence predicates
 412                         $predicates['exists'][$this->params['dst']] = true;
 413                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 414                 }
 415                 return $status; // safe to call attempt()
 416         }
 417
 418         protected function doAttempt() {
 419                 $status = Status::newGood();
 420                 // Store the file at the destination
 421                 if ( !$this->destSameAsSource ) {
 422                         $status->merge( $this->backend->storeInternal( $this->params ) );
 423                 }
 424                 return $status;
 425         }
 426
 427         protected function getSourceSha1Base36() {
 428                 wfSuppressWarnings();
 429                 $hash = sha1_file( $this->params['src'] );
 430                 wfRestoreWarnings();
 431                 if ( $hash !== false ) {
 432                         $hash = wfBaseConvert( $hash, 16, 36, 31 );
 433                 }
 434                 return $hash;
 435         }
 436
 437         public function storagePathsChanged() {
 438                 return array( $this->params['dst'] );
 439         }
 440 }
 441
 442 /**
 443  * Create a file in the backend with the given content.
 444  * Parameters similar to FileBackend::createInternal(), which include:
 445  *     content       : the raw file contents
 446  *     dst           : destination storage path
 447  *     overwrite     : do nothing and pass if an identical file exists at destination
 448  *     overwriteSame : override any existing file at destination
 449  */
 450 class CreateFileOp extends FileOp {
 451         protected static $requiredParams = array( 'content', 'dst' );
 452         protected static $optionalParams = array( 'overwrite', 'overwriteSame' );
 453
 454         protected function doPrecheck( array &$predicates ) {
 455                 $status = Status::newGood();
 456                 // Check if the source data is too big
 457                 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
 458                         $status->fatal( 'backend-fail-create', $this->params['dst'] );
 459                         return $status;
 460                 // Check if a file can be placed at the destination
 461                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 462                         $status->fatal( 'backend-fail-create', $this->params['dst'] );
 463                         return $status;
 464                 }
 465                 // Check if destination file exists
 466                 $status->merge( $this->precheckDestExistence( $predicates ) );
 467                 if ( $status->isOK() ) {
 468                         // Update file existence predicates
 469                         $predicates['exists'][$this->params['dst']] = true;
 470                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 471                 }
 472                 return $status; // safe to call attempt()
 473         }
 474
 475         protected function doAttempt() {
 476                 $status = Status::newGood();
 477                 // Create the file at the destination
 478                 if ( !$this->destSameAsSource ) {
 479                         $status->merge( $this->backend->createInternal( $this->params ) );
 480                 }
 481                 return $status;
 482         }
 483
 484         protected function getSourceSha1Base36() {
 485                 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
 486         }
 487
 488         public function storagePathsChanged() {
 489                 return array( $this->params['dst'] );
 490         }
 491 }
 492
 493 /**
 494  * Copy a file from one storage path to another in the backend.
 495  * Parameters similar to FileBackend::copyInternal(), which include:
 496  *     src           : source storage path
 497  *     dst           : destination storage path
 498  *     overwrite     : do nothing and pass if an identical file exists at destination
 499  *     overwriteSame : override any existing file at destination
 500  */
 501 class CopyFileOp extends FileOp {
 502         protected static $requiredParams = array( 'src', 'dst' );
 503         protected static $optionalParams = array( 'overwrite', 'overwriteSame' );
 504
 505         protected function doPrecheck( array &$predicates ) {
 506                 $status = Status::newGood();
 507                 // Check if the source file exists
 508                 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
 509                         $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 510                         return $status;
 511                 // Check if a file can be placed at the destination
 512                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 513                         $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
 514                         return $status;
 515                 }
 516                 // Check if destination file exists
 517                 $status->merge( $this->precheckDestExistence( $predicates ) );
 518                 if ( $status->isOK() ) {
 519                         // Update file existence predicates
 520                         $predicates['exists'][$this->params['dst']] = true;
 521                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 522                 }
 523                 return $status; // safe to call attempt()
 524         }
 525
 526         protected function doAttempt() {
 527                 $status = Status::newGood();
 528                 // Do nothing if the src/dst paths are the same
 529                 if ( $this->params['src'] !== $this->params['dst'] ) {
 530                         // Copy the file into the destination
 531                         if ( !$this->destSameAsSource ) {
 532                                 $status->merge( $this->backend->copyInternal( $this->params ) );
 533                         }
 534                 }
 535                 return $status;
 536         }
 537
 538         public function storagePathsRead() {
 539                 return array( $this->params['src'] );
 540         }
 541
 542         public function storagePathsChanged() {
 543                 return array( $this->params['dst'] );
 544         }
 545 }
 546
 547 /**
 548  * Move a file from one storage path to another in the backend.
 549  * Parameters similar to FileBackend::moveInternal(), which include:
 550  *     src           : source storage path
 551  *     dst           : destination storage path
 552  *     overwrite     : do nothing and pass if an identical file exists at destination
 553  *     overwriteSame : override any existing file at destination
 554  */
 555 class MoveFileOp extends FileOp {
 556         protected static $requiredParams = array( 'src', 'dst' );
 557         protected static $optionalParams = array( 'overwrite', 'overwriteSame' );
 558
 559         protected function doPrecheck( array &$predicates ) {
 560                 $status = Status::newGood();
 561                 // Check if the source file exists
 562                 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
 563                         $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 564                         return $status;
 565                 // Check if a file can be placed at the destination
 566                 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
 567                         $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
 568                         return $status;
 569                 }
 570                 // Check if destination file exists
 571                 $status->merge( $this->precheckDestExistence( $predicates ) );
 572                 if ( $status->isOK() ) {
 573                         // Update file existence predicates
 574                         $predicates['exists'][$this->params['src']] = false;
 575                         $predicates['sha1'][$this->params['src']] = false;
 576                         $predicates['exists'][$this->params['dst']] = true;
 577                         $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
 578                 }
 579                 return $status; // safe to call attempt()
 580         }
 581
 582         protected function doAttempt() {
 583                 $status = Status::newGood();
 584                 // Do nothing if the src/dst paths are the same
 585                 if ( $this->params['src'] !== $this->params['dst'] ) {
 586                         if ( !$this->destSameAsSource ) {
 587                                 // Move the file into the destination
 588                                 $status->merge( $this->backend->moveInternal( $this->params ) );
 589                         } else {
 590                                 // Just delete source as the destination needs no changes
 591                                 $params = array( 'src' => $this->params['src'] );
 592                                 $status->merge( $this->backend->deleteInternal( $params ) );
 593                         }
 594                 }
 595                 return $status;
 596         }
 597
 598         public function storagePathsRead() {
 599                 return array( $this->params['src'] );
 600         }
 601
 602         public function storagePathsChanged() {
 603                 return array( $this->params['dst'] );
 604         }
 605 }
 606
 607 /**
 608  * Delete a file at the storage path.
 609  * Parameters similar to FileBackend::deleteInternal(), which include:
 610  *     src                 : source storage path
 611  *     ignoreMissingSource : don't return an error if the file does not exist
 612  */
 613 class DeleteFileOp extends FileOp {
 614         protected static $requiredParams = array( 'src' );
 615         protected static $optionalParams = array( 'ignoreMissingSource' );
 616
 617         protected $needsDelete = true;
 618
 619         protected function doPrecheck( array &$predicates ) {
 620                 $status = Status::newGood();
 621                 // Check if the source file exists
 622                 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
 623                         if ( !$this->getParam( 'ignoreMissingSource' ) ) {
 624                                 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
 625                                 return $status;
 626                         }
 627                         $this->needsDelete = false;
 628                 }
 629                 // Update file existence predicates
 630                 $predicates['exists'][$this->params['src']] = false;
 631                 $predicates['sha1'][$this->params['src']] = false;
 632                 return $status; // safe to call attempt()
 633         }
 634
 635         protected function doAttempt() {
 636                 $status = Status::newGood();
 637                 if ( $this->needsDelete ) {
 638                         // Delete the source file
 639                         $status->merge( $this->backend->deleteInternal( $this->params ) );
 640                 }
 641                 return $status;
 642         }
 643
 644         public function storagePathsChanged() {
 645                 return array( $this->params['src'] );
 646         }
 647 }
 648
 649 /**
 650  * Placeholder operation that has no params and does nothing
 651  */
 652 class NullFileOp extends FileOp {
 653         protected function doAttempt() {
 654                 return Status::newGood();
 655         }
 656 }