includes/filerepo/backend/FileOpBatch.php

   1 <?php
   2 /**
   3  * Helper class for representing batch file operations.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * Helper class for representing batch file operations.
  27  * Do not use this class from places outside FileBackend.
  28  *
  29  * Methods should avoid throwing exceptions at all costs.
  30  *
  31  * @ingroup FileBackend
  32  * @since 1.20
  33  */
  34 class FileOpBatch {
  35         /* Timeout related parameters */
  36         const MAX_BATCH_SIZE = 1000; // integer
  37
  38         /**
  39          * Attempt to perform a series of file operations.
  40          * Callers are responsible for handling file locking.
  41          *
  42          * $opts is an array of options, including:
  43          *   - force        : Errors that would normally cause a rollback do not.
  44          *                    The remaining operations are still attempted if any fail.
  45          *   - allowStale   : Don't require the latest available data.
  46          *                    This can increase performance for non-critical writes.
  47          *                    This has no effect unless the 'force' flag is set.
  48          *   - nonJournaled : Don't log this operation batch in the file journal.
  49          *   - concurrency  : Try to do this many operations in parallel when possible.
  50          *
  51          * The resulting Status will be "OK" unless:
  52          *   - a) unexpected operation errors occurred (network partitions, disk full...)
  53          *   - b) significant operation errors occured and 'force' was not set
  54          *
  55          * @param $performOps Array List of FileOp operations
  56          * @param $opts Array Batch operation options
  57          * @param $journal FileJournal Journal to log operations to
  58          * @return Status
  59          */
  60         public static function attempt( array $performOps, array $opts, FileJournal $journal ) {
  61                 wfProfileIn( __METHOD__ );
  62                 $status = Status::newGood();
  63
  64                 $n = count( $performOps );
  65                 if ( $n > self::MAX_BATCH_SIZE ) {
  66                         $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
  67                         wfProfileOut( __METHOD__ );
  68                         return $status;
  69                 }
  70
  71                 $batchId = $journal->getTimestampedUUID();
  72                 $allowStale = !empty( $opts['allowStale'] );
  73                 $ignoreErrors = !empty( $opts['force'] );
  74                 $journaled = empty( $opts['nonJournaled'] );
  75                 $maxConcurrency = isset( $opts['concurrency'] ) ? $opts['concurrency'] : 1;
  76
  77                 $entries = array(); // file journal entry list
  78                 $predicates = FileOp::newPredicates(); // account for previous ops in prechecks
  79                 $curBatch = array(); // concurrent FileOp sub-batch accumulation
  80                 $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch
  81                 $pPerformOps = array(); // ordered list of concurrent FileOp sub-batches
  82                 $lastBackend = null; // last op backend name
  83                 // Do pre-checks for each operation; abort on failure...
  84                 foreach ( $performOps as $index => $fileOp ) {
  85                         $backendName = $fileOp->getBackend()->getName();
  86                         $fileOp->setBatchId( $batchId ); // transaction ID
  87                         $fileOp->allowStaleReads( $allowStale ); // consistency level
  88                         // Decide if this op can be done concurrently within this sub-batch
  89                         // or if a new concurrent sub-batch must be started after this one...
  90                         if ( $fileOp->dependsOn( $curBatchDeps )
  91                                 || count( $curBatch ) >= $maxConcurrency
  92                                 || ( $backendName !== $lastBackend && count( $curBatch ) )
  93                         ) {
  94                                 $pPerformOps[] = $curBatch; // push this batch
  95                                 $curBatch = array(); // start a new sub-batch
  96                                 $curBatchDeps = FileOp::newDependencies();
  97                         }
  98                         $lastBackend = $backendName;
  99                         $curBatch[$index] = $fileOp; // keep index
 100                         // Update list of affected paths in this batch
 101                         $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps );
 102                         // Simulate performing the operation...
 103                         $oldPredicates = $predicates;
 104                         $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
 105                         $status->merge( $subStatus );
 106                         if ( $subStatus->isOK() ) {
 107                                 if ( $journaled ) { // journal log entries
 108                                         $entries = array_merge( $entries,
 109                                                 $fileOp->getJournalEntries( $oldPredicates, $predicates ) );
 110                                 }
 111                         } else { // operation failed?
 112                                 $status->success[$index] = false;
 113                                 ++$status->failCount;
 114                                 if ( !$ignoreErrors ) {
 115                                         wfProfileOut( __METHOD__ );
 116                                         return $status; // abort
 117                                 }
 118                         }
 119                 }
 120                 // Push the last sub-batch
 121                 if ( count( $curBatch ) ) {
 122                         $pPerformOps[] = $curBatch;
 123                 }
 124
 125                 // Log the operations in the file journal...
 126                 if ( count( $entries ) ) {
 127                         $subStatus = $journal->logChangeBatch( $entries, $batchId );
 128                         if ( !$subStatus->isOK() ) {
 129                                 wfProfileOut( __METHOD__ );
 130                                 return $subStatus; // abort
 131                         }
 132                 }
 133
 134                 if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
 135                         $status->setResult( true, $status->value );
 136                 }
 137
 138                 // Attempt each operation (in parallel if allowed and possible)...
 139                 if ( count( $pPerformOps ) < count( $performOps ) ) {
 140                         self::runBatchParallel( $pPerformOps, $status );
 141                 } else {
 142                         self::runBatchSeries( $performOps, $status );
 143                 }
 144
 145                 wfProfileOut( __METHOD__ );
 146                 return $status;
 147         }
 148
 149         /**
 150          * Attempt a list of file operations in series.
 151          * This will abort remaining ops on failure.
 152          *
 153          * @param $performOps Array
 154          * @param $status Status
 155          * @return bool Success
 156          */
 157         protected static function runBatchSeries( array $performOps, Status $status ) {
 158                 foreach ( $performOps as $index => $fileOp ) {
 159                         if ( $fileOp->failed() ) {
 160                                 continue; // nothing to do
 161                         }
 162                         $subStatus = $fileOp->attempt();
 163                         $status->merge( $subStatus );
 164                         if ( $subStatus->isOK() ) {
 165                                 $status->success[$index] = true;
 166                                 ++$status->successCount;
 167                         } else {
 168                                 $status->success[$index] = false;
 169                                 ++$status->failCount;
 170                                 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
 171                                 // Log the remaining ops as failed for recovery...
 172                                 for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
 173                                         $performOps[$i]->logFailure( 'attempt_aborted' );
 174                                 }
 175                                 return false; // bail out
 176                         }
 177                 }
 178                 return true;
 179         }
 180
 181         /**
 182          * Attempt a list of file operations sub-batches in series.
 183          *
 184          * The operations *in* each sub-batch will be done in parallel.
 185          * The caller is responsible for making sure the operations
 186          * within any given sub-batch do not depend on each other.
 187          * This will abort remaining ops on failure.
 188          *
 189          * @param $pPerformOps Array
 190          * @param $status Status
 191          * @return bool Success
 192          */
 193         protected static function runBatchParallel( array $pPerformOps, Status $status ) {
 194                 $aborted = false;
 195                 foreach ( $pPerformOps as $performOpsBatch ) {
 196                         if ( $aborted ) { // check batch op abort flag...
 197                                 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
 198                                 // Log the remaining ops as failed for recovery...
 199                                 foreach ( $performOpsBatch as $i => $fileOp ) {
 200                                         $performOpsBatch[$i]->logFailure( 'attempt_aborted' );
 201                                 }
 202                                 continue;
 203                         }
 204                         $statuses = array();
 205                         $opHandles = array();
 206                         // Get the backend; all sub-batch ops belong to a single backend
 207                         $backend = reset( $performOpsBatch )->getBackend();
 208                         // If attemptAsync() returns synchronously, it was either an
 209                         // error Status or the backend just doesn't support async ops.
 210                         foreach ( $performOpsBatch as $i => $fileOp ) {
 211                                 if ( !$fileOp->failed() ) { // failed => already has Status
 212                                         $subStatus = $fileOp->attemptAsync();
 213                                         if ( $subStatus->value instanceof FileBackendStoreOpHandle ) {
 214                                                 $opHandles[$i] = $subStatus->value; // deferred
 215                                         } else {
 216                                                 $statuses[$i] = $subStatus; // done already
 217                                         }
 218                                 }
 219                         }
 220                         // Try to do all the operations concurrently...
 221                         $statuses = $statuses + $backend->executeOpHandlesInternal( $opHandles );
 222                         // Marshall and merge all the responses (blocking)...
 223                         foreach ( $performOpsBatch as $i => $fileOp ) {
 224                                 if ( !$fileOp->failed() ) { // failed => already has Status
 225                                         $subStatus = $statuses[$i];
 226                                         $status->merge( $subStatus );
 227                                         if ( $subStatus->isOK() ) {
 228                                                 $status->success[$i] = true;
 229                                                 ++$status->successCount;
 230                                         } else {
 231                                                 $status->success[$i] = false;
 232                                                 ++$status->failCount;
 233                                                 $aborted = true; // set abort flag; we can't continue
 234                                         }
 235                                 }
 236                         }
 237                 }
 238                 return $status;
 239         }
 240 }