includes/job/JobQueue.php

   1 <?php
   2 /**
   3  * Job queue base code
   4  *
   5  * @file
   6  * @defgroup JobQueue JobQueue
   7  */
   8
   9 if ( !defined( 'MEDIAWIKI' ) ) {
  10         die( "This file is part of MediaWiki, it is not a valid entry point\n" );
  11 }
  12
  13 /**
  14  * Class to both describe a background job and handle jobs.
  15  *
  16  * @ingroup JobQueue
  17  */
  18 abstract class Job {
  19
  20         /**
  21          * @var Title
  22          */
  23         var $title;
  24
  25         var $command,
  26                 $params,
  27                 $id,
  28                 $removeDuplicates,
  29                 $error;
  30
  31         /*-------------------------------------------------------------------------
  32          * Abstract functions
  33          *------------------------------------------------------------------------*/
  34
  35         /**
  36          * Run the job
  37          * @return boolean success
  38          */
  39         abstract function run();
  40
  41         /*-------------------------------------------------------------------------
  42          * Static functions
  43          *------------------------------------------------------------------------*/
  44
  45         /**
  46          * Pop a job of a certain type.  This tries less hard than pop() to
  47          * actually find a job; it may be adversely affected by concurrent job
  48          * runners.
  49          */
  50         static function pop_type( $type ) {
  51                 wfProfilein( __METHOD__ );
  52
  53                 $dbw = wfGetDB( DB_MASTER );
  54
  55                 $row = $dbw->selectRow(
  56                         'job',
  57                         '*',
  58                         array( 'job_cmd' => $type ),
  59                         __METHOD__,
  60                         array( 'LIMIT' => 1 )
  61                 );
  62
  63                 if ( $row === false ) {
  64                         wfProfileOut( __METHOD__ );
  65                         return false;
  66                 }
  67
  68                 /* Ensure we "own" this row */
  69                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  70                 $affected = $dbw->affectedRows();
  71
  72                 if ( $affected == 0 ) {
  73                         wfProfileOut( __METHOD__ );
  74                         return false;
  75                 }
  76
  77                 wfIncrStats( 'job-pop' );
  78                 $namespace = $row->job_namespace;
  79                 $dbkey = $row->job_title;
  80                 $title = Title::makeTitleSafe( $namespace, $dbkey );
  81                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
  82                         $row->job_id );
  83
  84                 $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
  85                 $affected = $dbw->affectedRows();
  86                 $dbw->commit();
  87
  88                 if ( $affected ) {
  89                         wfIncrStats( 'job-dup-delete', $affected );
  90                 }
  91
  92                 wfProfileOut( __METHOD__ );
  93                 return $job;
  94         }
  95
  96         /**
  97          * Pop a job off the front of the queue
  98          *
  99          * @param $offset Integer: Number of jobs to skip
 100          * @return Job or false if there's no jobs
 101          */
 102         static function pop( $offset = 0 ) {
 103                 wfProfileIn( __METHOD__ );
 104
 105                 $dbr = wfGetDB( DB_SLAVE );
 106
 107                 /* Get a job from the slave, start with an offset,
 108                         scan full set afterwards, avoid hitting purged rows
 109
 110                         NB: If random fetch previously was used, offset
 111                                 will always be ahead of few entries
 112                 */
 113
 114                 $row = $dbr->selectRow( 'job', '*', "job_id >= ${offset}", __METHOD__,
 115                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 116
 117                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 118                 // and have lower IDs than jobs already executed, blame concurrency :)
 119                 //
 120                 if ( $row === false ) {
 121                         if ( $offset != 0 ) {
 122                                 $row = $dbr->selectRow( 'job', '*', '', __METHOD__,
 123                                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 124                         }
 125
 126                         if ( $row === false ) {
 127                                 wfProfileOut( __METHOD__ );
 128                                 return false;
 129                         }
 130                 }
 131
 132                 // Try to delete it from the master
 133                 $dbw = wfGetDB( DB_MASTER );
 134                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 135                 $affected = $dbw->affectedRows();
 136                 $dbw->commit();
 137
 138                 if ( !$affected ) {
 139                         // Failed, someone else beat us to it
 140                         // Try getting a random row
 141                         $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
 142                                 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ );
 143                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 144                                 // No jobs to get
 145                                 wfProfileOut( __METHOD__ );
 146                                 return false;
 147                         }
 148                         // Get the random row
 149                         $row = $dbw->selectRow( 'job', '*',
 150                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 151                         if ( $row === false ) {
 152                                 // Random job gone before we got the chance to select it
 153                                 // Give up
 154                                 wfProfileOut( __METHOD__ );
 155                                 return false;
 156                         }
 157                         // Delete the random row
 158                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 159                         $affected = $dbw->affectedRows();
 160                         $dbw->commit();
 161
 162                         if ( !$affected ) {
 163                                 // Random job gone before we exclusively deleted it
 164                                 // Give up
 165                                 wfProfileOut( __METHOD__ );
 166                                 return false;
 167                         }
 168                 }
 169
 170                 // If execution got to here, there's a row in $row that has been deleted from the database
 171                 // by this thread. Hence the concurrent pop was successful.
 172                 wfIncrStats( 'job-pop' );
 173                 $namespace = $row->job_namespace;
 174                 $dbkey = $row->job_title;
 175                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 176                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 177
 178                 // Remove any duplicates it may have later in the queue
 179                 // Deadlock prone section
 180                 $dbw->begin();
 181                 $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
 182                 $affected = $dbw->affectedRows();
 183                 $dbw->commit();
 184
 185                 if ( $affected ) {
 186                         wfIncrStats( 'job-dup-delete', $affected );
 187                 }
 188
 189                 wfProfileOut( __METHOD__ );
 190                 return $job;
 191         }
 192
 193         /**
 194          * Create the appropriate object to handle a specific job
 195          *
 196          * @param $command String: Job command
 197          * @param $title Title: Associated title
 198          * @param $params Array: Job parameters
 199          * @param $id Int: Job identifier
 200          * @return Job
 201          */
 202         static function factory( $command, $title, $params = false, $id = 0 ) {
 203                 global $wgJobClasses;
 204                 if( isset( $wgJobClasses[$command] ) ) {
 205                         $class = $wgJobClasses[$command];
 206                         return new $class( $title, $params, $id );
 207                 }
 208                 throw new MWException( "Invalid job command `{$command}`" );
 209         }
 210
 211         static function makeBlob( $params ) {
 212                 if ( $params !== false ) {
 213                         return serialize( $params );
 214                 } else {
 215                         return '';
 216                 }
 217         }
 218
 219         static function extractBlob( $blob ) {
 220                 if ( (string)$blob !== '' ) {
 221                         return unserialize( $blob );
 222                 } else {
 223                         return false;
 224                 }
 225         }
 226
 227         /**
 228          * Batch-insert a group of jobs into the queue.
 229          * This will be wrapped in a transaction with a forced commit.
 230          *
 231          * This may add duplicate at insert time, but they will be
 232          * removed later on, when the first one is popped.
 233          *
 234          * @param $jobs array of Job objects
 235          */
 236         static function batchInsert( $jobs ) {
 237                 if( !count( $jobs ) ) {
 238                         return;
 239                 }
 240                 $dbw = wfGetDB( DB_MASTER );
 241                 $rows = array();
 242                 foreach( $jobs as $job ) {
 243                         $rows[] = $job->insertFields();
 244                         if ( count( $rows ) >= 50 ) {
 245                                 # Do a small transaction to avoid slave lag
 246                                 $dbw->begin();
 247                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 248                                 $dbw->commit();
 249                                 $rows = array();
 250                         }
 251                 }
 252                 if ( $rows ) {
 253                         wfIncrStats( 'job-insert', count( $rows ) );
 254                         $dbw->begin();
 255                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 256                         $dbw->commit();
 257                 }
 258         }
 259
 260         /*-------------------------------------------------------------------------
 261          * Non-static functions
 262          *------------------------------------------------------------------------*/
 263
 264         /**
 265          * @param $command
 266          * @param $title
 267          * @param $params array
 268          * @param int $id
 269          */
 270         function __construct( $command, $title, $params = false, $id = 0 ) {
 271                 $this->command = $command;
 272                 $this->title = $title;
 273                 $this->params = $params;
 274                 $this->id = $id;
 275
 276                 // A bit of premature generalisation
 277                 // Oh well, the whole class is premature generalisation really
 278                 $this->removeDuplicates = true;
 279         }
 280
 281         /**
 282          * Insert a single job into the queue.
 283          * @return bool true on success
 284          */
 285         function insert() {
 286                 $fields = $this->insertFields();
 287
 288                 $dbw = wfGetDB( DB_MASTER );
 289
 290                 if ( $this->removeDuplicates ) {
 291                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 292                         if ( $dbw->numRows( $res ) ) {
 293                                 return;
 294                         }
 295                 }
 296                 wfIncrStats( 'job-insert' );
 297                 return $dbw->insert( 'job', $fields, __METHOD__ );
 298         }
 299
 300         protected function insertFields() {
 301                 $dbw = wfGetDB( DB_MASTER );
 302                 return array(
 303                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 304                         'job_cmd' => $this->command,
 305                         'job_namespace' => $this->title->getNamespace(),
 306                         'job_title' => $this->title->getDBkey(),
 307                         'job_params' => Job::makeBlob( $this->params )
 308                 );
 309         }
 310
 311         function toString() {
 312                 $paramString = '';
 313                 if ( $this->params ) {
 314                         foreach ( $this->params as $key => $value ) {
 315                                 if ( $paramString != '' ) {
 316                                         $paramString .= ' ';
 317                                 }
 318                                 $paramString .= "$key=$value";
 319                         }
 320                 }
 321
 322                 if ( is_object( $this->title ) ) {
 323                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 324                         if ( $paramString !== '' ) {
 325                                 $s .= ' ' . $paramString;
 326                         }
 327                         return $s;
 328                 } else {
 329                         return "{$this->command} $paramString";
 330                 }
 331         }
 332
 333         protected function setLastError( $error ) {
 334                 $this->error = $error;
 335         }
 336
 337         function getLastError() {
 338                 return $this->error;
 339         }
 340 }