includes/libs/filebackend/fileop/FileOp.php

   1 <?php
   2 /**
   3  * Helper class for representing operations with transaction support.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  */
  23 use Psr\Log\LoggerInterface;
  24
  25 /**
  26  * FileBackend helper class for representing operations.
  27  * Do not use this class from places outside FileBackend.
  28  *
  29  * Methods called from FileOpBatch::attempt() should avoid throwing
  30  * exceptions at all costs. FileOp objects should be lightweight in order
  31  * to support large arrays in memory and serialization.
  32  *
  33  * @ingroup FileBackend
  34  * @since 1.19
  35  */
  36 abstract class FileOp {
  37         /** @var array */
  38         protected $params = [];
  39
  40         /** @var FileBackendStore */
  41         protected $backend;
  42         /** @var LoggerInterface */
  43         protected $logger;
  44
  45         /** @var int */
  46         protected $state = self::STATE_NEW;
  47
  48         /** @var bool */
  49         protected $failed = false;
  50
  51         /** @var bool */
  52         protected $async = false;
  53
  54         /** @var string */
  55         protected $batchId;
  56
  57         /** @var bool Operation is not a no-op */
  58         protected $doOperation = true;
  59
  60         /** @var string */
  61         protected $sourceSha1;
  62
  63         /** @var bool */
  64         protected $overwriteSameCase;
  65
  66         /** @var bool */
  67         protected $destExists;
  68
  69         /* Object life-cycle */
  70         const STATE_NEW = 1;
  71         const STATE_CHECKED = 2;
  72         const STATE_ATTEMPTED = 3;
  73
  74         /**
  75          * Build a new batch file operation transaction
  76          *
  77          * @param FileBackendStore $backend
  78          * @param array $params
  79          * @param LoggerInterface $logger PSR logger instance
  80          * @throws InvalidArgumentException
  81          */
  82         final public function __construct(
  83                 FileBackendStore $backend, array $params, LoggerInterface $logger
  84         ) {
  85                 $this->backend = $backend;
  86                 $this->logger = $logger;
  87                 list( $required, $optional, $paths ) = $this->allowedParams();
  88                 foreach ( $required as $name ) {
  89                         if ( isset( $params[$name] ) ) {
  90                                 $this->params[$name] = $params[$name];
  91                         } else {
  92                                 throw new InvalidArgumentException( "File operation missing parameter '$name'." );
  93                         }
  94                 }
  95                 foreach ( $optional as $name ) {
  96                         if ( isset( $params[$name] ) ) {
  97                                 $this->params[$name] = $params[$name];
  98                         }
  99                 }
 100                 foreach ( $paths as $name ) {
 101                         if ( isset( $this->params[$name] ) ) {
 102                                 // Normalize paths so the paths to the same file have the same string
 103                                 $this->params[$name] = self::normalizeIfValidStoragePath( $this->params[$name] );
 104                         }
 105                 }
 106         }
 107
 108         /**
 109          * Normalize a string if it is a valid storage path
 110          *
 111          * @param string $path
 112          * @return string
 113          */
 114         protected static function normalizeIfValidStoragePath( $path ) {
 115                 if ( FileBackend::isStoragePath( $path ) ) {
 116                         $res = FileBackend::normalizeStoragePath( $path );
 117
 118                         return $res ?? $path;
 119                 }
 120
 121                 return $path;
 122         }
 123
 124         /**
 125          * Set the batch UUID this operation belongs to
 126          *
 127          * @param string $batchId
 128          */
 129         final public function setBatchId( $batchId ) {
 130                 $this->batchId = $batchId;
 131         }
 132
 133         /**
 134          * Get the value of the parameter with the given name
 135          *
 136          * @param string $name
 137          * @return mixed Returns null if the parameter is not set
 138          */
 139         final public function getParam( $name ) {
 140                 return $this->params[$name] ?? null;
 141         }
 142
 143         /**
 144          * Check if this operation failed precheck() or attempt()
 145          *
 146          * @return bool
 147          */
 148         final public function failed() {
 149                 return $this->failed;
 150         }
 151
 152         /**
 153          * Get a new empty predicates array for precheck()
 154          *
 155          * @return array
 156          */
 157         final public static function newPredicates() {
 158                 return [ 'exists' => [], 'sha1' => [] ];
 159         }
 160
 161         /**
 162          * Get a new empty dependency tracking array for paths read/written to
 163          *
 164          * @return array
 165          */
 166         final public static function newDependencies() {
 167                 return [ 'read' => [], 'write' => [] ];
 168         }
 169
 170         /**
 171          * Update a dependency tracking array to account for this operation
 172          *
 173          * @param array $deps Prior path reads/writes; format of FileOp::newPredicates()
 174          * @return array
 175          */
 176         final public function applyDependencies( array $deps ) {
 177                 $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 );
 178                 $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 );
 179
 180                 return $deps;
 181         }
 182
 183         /**
 184          * Check if this operation changes files listed in $paths
 185          *
 186          * @param array $deps Prior path reads/writes; format of FileOp::newPredicates()
 187          * @return bool
 188          */
 189         final public function dependsOn( array $deps ) {
 190                 foreach ( $this->storagePathsChanged() as $path ) {
 191                         if ( isset( $deps['read'][$path] ) || isset( $deps['write'][$path] ) ) {
 192                                 return true; // "output" or "anti" dependency
 193                         }
 194                 }
 195                 foreach ( $this->storagePathsRead() as $path ) {
 196                         if ( isset( $deps['write'][$path] ) ) {
 197                                 return true; // "flow" dependency
 198                         }
 199                 }
 200
 201                 return false;
 202         }
 203
 204         /**
 205          * Get the file journal entries for this file operation
 206          *
 207          * @param array $oPredicates Pre-op info about files (format of FileOp::newPredicates)
 208          * @param array $nPredicates Post-op info about files (format of FileOp::newPredicates)
 209          * @return array
 210          */
 211         final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
 212                 if ( !$this->doOperation ) {
 213                         return []; // this is a no-op
 214                 }
 215                 $nullEntries = [];
 216                 $updateEntries = [];
 217                 $deleteEntries = [];
 218                 $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
 219                 foreach ( array_unique( $pathsUsed ) as $path ) {
 220                         $nullEntries[] = [ // assertion for recovery
 221                                 'op' => 'null',
 222                                 'path' => $path,
 223                                 'newSha1' => $this->fileSha1( $path, $oPredicates )
 224                         ];
 225                 }
 226                 foreach ( $this->storagePathsChanged() as $path ) {
 227                         if ( $nPredicates['sha1'][$path] === false ) { // deleted
 228                                 $deleteEntries[] = [
 229                                         'op' => 'delete',
 230                                         'path' => $path,
 231                                         'newSha1' => ''
 232                                 ];
 233                         } else { // created/updated
 234                                 $updateEntries[] = [
 235                                         'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create',
 236                                         'path' => $path,
 237                                         'newSha1' => $nPredicates['sha1'][$path]
 238                                 ];
 239                         }
 240                 }
 241
 242                 return array_merge( $nullEntries, $updateEntries, $deleteEntries );
 243         }
 244
 245         /**
 246          * Check preconditions of the operation without writing anything.
 247          * This must update $predicates for each path that the op can change
 248          * except when a failing StatusValue object is returned.
 249          *
 250          * @param array &$predicates
 251          * @return StatusValue
 252          */
 253         final public function precheck( array &$predicates ) {
 254                 if ( $this->state !== self::STATE_NEW ) {
 255                         return StatusValue::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
 256                 }
 257                 $this->state = self::STATE_CHECKED;
 258
 259                 $status = StatusValue::newGood();
 260                 $storagePaths = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
 261                 foreach ( array_unique( $storagePaths ) as $storagePath ) {
 262                         if ( !$this->backend->isPathUsableInternal( $storagePath ) ) {
 263                                 $status->fatal( 'backend-fail-usable', $storagePath );
 264                         }
 265                 }
 266                 if ( !$status->isOK() ) {
 267                         return $status;
 268                 }
 269
 270                 $status = $this->doPrecheck( $predicates );
 271                 if ( !$status->isOK() ) {
 272                         $this->failed = true;
 273                 }
 274
 275                 return $status;
 276         }
 277
 278         /**
 279          * @param array &$predicates
 280          * @return StatusValue
 281          */
 282         protected function doPrecheck( array &$predicates ) {
 283                 return StatusValue::newGood();
 284         }
 285
 286         /**
 287          * Attempt the operation
 288          *
 289          * @return StatusValue
 290          */
 291         final public function attempt() {
 292                 if ( $this->state !== self::STATE_CHECKED ) {
 293                         return StatusValue::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
 294                 } elseif ( $this->failed ) { // failed precheck
 295                         return StatusValue::newFatal( 'fileop-fail-attempt-precheck' );
 296                 }
 297                 $this->state = self::STATE_ATTEMPTED;
 298                 if ( $this->doOperation ) {
 299                         $status = $this->doAttempt();
 300                         if ( !$status->isOK() ) {
 301                                 $this->failed = true;
 302                                 $this->logFailure( 'attempt' );
 303                         }
 304                 } else { // no-op
 305                         $status = StatusValue::newGood();
 306                 }
 307
 308                 return $status;
 309         }
 310
 311         /**
 312          * @return StatusValue
 313          */
 314         protected function doAttempt() {
 315                 return StatusValue::newGood();
 316         }
 317
 318         /**
 319          * Attempt the operation in the background
 320          *
 321          * @return StatusValue
 322          */
 323         final public function attemptAsync() {
 324                 $this->async = true;
 325                 $result = $this->attempt();
 326                 $this->async = false;
 327
 328                 return $result;
 329         }
 330
 331         /**
 332          * Get the file operation parameters
 333          *
 334          * @return array (required params list, optional params list, list of params that are paths)
 335          */
 336         protected function allowedParams() {
 337                 return [ [], [], [] ];
 338         }
 339
 340         /**
 341          * Adjust params to FileBackendStore internal file calls
 342          *
 343          * @param array $params
 344          * @return array (required params list, optional params list)
 345          */
 346         protected function setFlags( array $params ) {
 347                 return [ 'async' => $this->async ] + $params;
 348         }
 349
 350         /**
 351          * Get a list of storage paths read from for this operation
 352          *
 353          * @return array
 354          */
 355         public function storagePathsRead() {
 356                 return [];
 357         }
 358
 359         /**
 360          * Get a list of storage paths written to for this operation
 361          *
 362          * @return array
 363          */
 364         public function storagePathsChanged() {
 365                 return [];
 366         }
 367
 368         /**
 369          * Check for errors with regards to the destination file already existing.
 370          * Also set the destExists, overwriteSameCase and sourceSha1 member variables.
 371          * A bad StatusValue will be returned if there is no chance it can be overwritten.
 372          *
 373          * @param array $predicates
 374          * @return StatusValue
 375          */
 376         protected function precheckDestExistence( array $predicates ) {
 377                 $status = StatusValue::newGood();
 378                 // Get hash of source file/string and the destination file
 379                 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
 380                 if ( $this->sourceSha1 === null ) { // file in storage?
 381                         $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
 382                 }
 383                 $this->overwriteSameCase = false;
 384                 $this->destExists = $this->fileExists( $this->params['dst'], $predicates );
 385                 if ( $this->destExists ) {
 386                         if ( $this->getParam( 'overwrite' ) ) {
 387                                 return $status; // OK
 388                         } elseif ( $this->getParam( 'overwriteSame' ) ) {
 389                                 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
 390                                 // Check if hashes are valid and match each other...
 391                                 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
 392                                         $status->fatal( 'backend-fail-hashes' );
 393                                 } elseif ( $this->sourceSha1 !== $dhash ) {
 394                                         // Give an error if the files are not identical
 395                                         $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
 396                                 } else {
 397                                         $this->overwriteSameCase = true; // OK
 398                                 }
 399
 400                                 return $status; // do nothing; either OK or bad status
 401                         } else {
 402                                 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
 403
 404                                 return $status;
 405                         }
 406                 } elseif ( $this->destExists === FileBackend::EXISTENCE_ERROR ) {
 407                         $status->fatal( 'backend-fail-stat', $this->params['dst'] );
 408                 }
 409
 410                 return $status;
 411         }
 412
 413         /**
 414          * precheckDestExistence() helper function to get the source file SHA-1.
 415          * Subclasses should overwride this if the source is not in storage.
 416          *
 417          * @return string|bool Returns false on failure
 418          */
 419         protected function getSourceSha1Base36() {
 420                 return null; // N/A
 421         }
 422
 423         /**
 424          * Check if a file will exist in storage when this operation is attempted
 425          *
 426          * Ideally, the file stat entry should already be preloaded via preloadFileStat().
 427          * Otherwise, this will query the backend.
 428          *
 429          * @param string $source Storage path
 430          * @param array $predicates
 431          * @return bool|null Whether the file will exist or null on error
 432          */
 433         final protected function fileExists( $source, array $predicates ) {
 434                 if ( isset( $predicates['exists'][$source] ) ) {
 435                         return $predicates['exists'][$source]; // previous op assures this
 436                 } else {
 437                         $params = [ 'src' => $source, 'latest' => true ];
 438
 439                         return $this->backend->fileExists( $params );
 440                 }
 441         }
 442
 443         /**
 444          * Get the SHA-1 hash a file in storage will have when this operation is attempted
 445          *
 446          * Ideally, file the stat entry should already be preloaded via preloadFileStat() and
 447          * the backend tracks hashes as extended attributes. Otherwise, this will query the backend.
 448          *
 449          * @param string $source Storage path
 450          * @param array $predicates
 451          * @return string|bool The SHA-1 hash the file will have or false if non-existent or on error
 452          */
 453         final protected function fileSha1( $source, array $predicates ) {
 454                 if ( isset( $predicates['sha1'][$source] ) ) {
 455                         return $predicates['sha1'][$source]; // previous op assures this
 456                 } elseif ( isset( $predicates['exists'][$source] ) && !$predicates['exists'][$source] ) {
 457                         return false; // previous op assures this
 458                 } else {
 459                         $params = [ 'src' => $source, 'latest' => true ];
 460
 461                         return $this->backend->getFileSha1Base36( $params );
 462                 }
 463         }
 464
 465         /**
 466          * Get the backend this operation is for
 467          *
 468          * @return FileBackendStore
 469          */
 470         public function getBackend() {
 471                 return $this->backend;
 472         }
 473
 474         /**
 475          * Log a file operation failure and preserve any temp files
 476          *
 477          * @param string $action
 478          */
 479         final public function logFailure( $action ) {
 480                 $params = $this->params;
 481                 $params['failedAction'] = $action;
 482                 try {
 483                         $this->logger->error( static::class .
 484                                 " failed (batch #{$this->batchId}): " . FormatJson::encode( $params ) );
 485                 } catch ( Exception $e ) {
 486                         // bad config? debug log error?
 487                 }
 488         }
 489 }