includes/job/JobQueue.php

   1 <?php
   2 /**
   3  * @defgroup JobQueue JobQueue
   4  */
   5
   6 if ( !defined( 'MEDIAWIKI' ) ) {
   7         die( "This file is part of MediaWiki, it is not a valid entry point\n" );
   8 }
   9
  10 /**
  11  * Class to both describe a background job and handle jobs.
  12  *
  13  * @ingroup JobQueue
  14  */
  15 abstract class Job {
  16         var $command,
  17                 $title,
  18                 $params,
  19                 $id,
  20                 $removeDuplicates,
  21                 $error;
  22
  23         /*-------------------------------------------------------------------------
  24          * Abstract functions
  25          *------------------------------------------------------------------------*/
  26
  27         /**
  28          * Run the job
  29          * @return boolean success
  30          */
  31         abstract function run();
  32
  33         /*-------------------------------------------------------------------------
  34          * Static functions
  35          *------------------------------------------------------------------------*/
  36
  37         /**
  38          * @deprecated use LinksUpdate::queueRecursiveJobs()
  39          */
  40         /**
  41          * static function queueLinksJobs( $titles ) {}
  42          */
  43
  44         /**
  45          * Pop a job of a certain type.  This tries less hard than pop() to
  46          * actually find a job; it may be adversely affected by concurrent job
  47          * runners.
  48          */
  49         static function pop_type( $type ) {
  50                 wfProfilein( __METHOD__ );
  51
  52                 $dbw = wfGetDB( DB_MASTER );
  53
  54                 $row = $dbw->selectRow(
  55                         'job',
  56                         '*',
  57                         array( 'job_cmd' => $type ),
  58                         __METHOD__,
  59                         array( 'LIMIT' => 1 )
  60                 );
  61
  62                 if ( $row === false ) {
  63                         wfProfileOut( __METHOD__ );
  64                         return false;
  65                 }
  66
  67                 /* Ensure we "own" this row */
  68                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  69                 $affected = $dbw->affectedRows();
  70
  71                 if ( $affected == 0 ) {
  72                         wfProfileOut( __METHOD__ );
  73                         return false;
  74                 }
  75
  76                 $namespace = $row->job_namespace;
  77                 $dbkey = $row->job_title;
  78                 $title = Title::makeTitleSafe( $namespace, $dbkey );
  79                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
  80                         $row->job_id );
  81
  82                 $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
  83                 $dbw->commit();
  84
  85                 wfProfileOut( __METHOD__ );
  86                 return $job;
  87         }
  88
  89         /**
  90          * Pop a job off the front of the queue
  91          *
  92          * @param $offset Integer: Number of jobs to skip
  93          * @return Job or false if there's no jobs
  94          */
  95         static function pop( $offset = 0 ) {
  96                 wfProfileIn( __METHOD__ );
  97
  98                 $dbr = wfGetDB( DB_SLAVE );
  99
 100                 /* Get a job from the slave, start with an offset,
 101                         scan full set afterwards, avoid hitting purged rows
 102
 103                         NB: If random fetch previously was used, offset
 104                                 will always be ahead of few entries
 105                 */
 106
 107                 $row = $dbr->selectRow( 'job', '*', "job_id >= ${offset}", __METHOD__,
 108                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 109
 110                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 111                 // and have lower IDs than jobs already executed, blame concurrency :)
 112                 //
 113                 if ( $row === false ) {
 114                         if ( $offset != 0 ) {
 115                                 $row = $dbr->selectRow( 'job', '*', '', __METHOD__,
 116                                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 117                         }
 118
 119                         if ( $row === false ) {
 120                                 wfProfileOut( __METHOD__ );
 121                                 return false;
 122                         }
 123                 }
 124                 $offset = $row->job_id;
 125
 126                 // Try to delete it from the master
 127                 $dbw = wfGetDB( DB_MASTER );
 128                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 129                 $affected = $dbw->affectedRows();
 130                 $dbw->commit();
 131
 132                 if ( !$affected ) {
 133                         // Failed, someone else beat us to it
 134                         // Try getting a random row
 135                         $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
 136                                 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ );
 137                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 138                                 // No jobs to get
 139                                 wfProfileOut( __METHOD__ );
 140                                 return false;
 141                         }
 142                         // Get the random row
 143                         $row = $dbw->selectRow( 'job', '*',
 144                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 145                         if ( $row === false ) {
 146                                 // Random job gone before we got the chance to select it
 147                                 // Give up
 148                                 wfProfileOut( __METHOD__ );
 149                                 return false;
 150                         }
 151                         // Delete the random row
 152                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 153                         $affected = $dbw->affectedRows();
 154                         $dbw->commit();
 155
 156                         if ( !$affected ) {
 157                                 // Random job gone before we exclusively deleted it
 158                                 // Give up
 159                                 wfProfileOut( __METHOD__ );
 160                                 return false;
 161                         }
 162                 }
 163
 164                 // If execution got to here, there's a row in $row that has been deleted from the database
 165                 // by this thread. Hence the concurrent pop was successful.
 166                 $namespace = $row->job_namespace;
 167                 $dbkey = $row->job_title;
 168                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 169                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 170
 171                 // Remove any duplicates it may have later in the queue
 172                 // Deadlock prone section
 173                 $dbw->begin();
 174                 $dbw->delete( 'job', $job->insertFields(), __METHOD__ );
 175                 $dbw->commit();
 176
 177                 wfProfileOut( __METHOD__ );
 178                 return $job;
 179         }
 180
 181         /**
 182          * Create the appropriate object to handle a specific job
 183          *
 184          * @param $command String: Job command
 185          * @param $title Title: Associated title
 186          * @param $params Array: Job parameters
 187          * @param $id Int: Job identifier
 188          * @return Job
 189          */
 190         static function factory( $command, $title, $params = false, $id = 0 ) {
 191                 global $wgJobClasses;
 192                 if( isset( $wgJobClasses[$command] ) ) {
 193                         $class = $wgJobClasses[$command];
 194                         return new $class( $title, $params, $id );
 195                 }
 196                 throw new MWException( "Invalid job command `{$command}`" );
 197         }
 198
 199         static function makeBlob( $params ) {
 200                 if ( $params !== false ) {
 201                         return serialize( $params );
 202                 } else {
 203                         return '';
 204                 }
 205         }
 206
 207         static function extractBlob( $blob ) {
 208                 if ( (string)$blob !== '' ) {
 209                         return unserialize( $blob );
 210                 } else {
 211                         return false;
 212                 }
 213         }
 214
 215         /**
 216          * Batch-insert a group of jobs into the queue.
 217          * This will be wrapped in a transaction with a forced commit.
 218          *
 219          * This may add duplicate at insert time, but they will be
 220          * removed later on, when the first one is popped.
 221          *
 222          * @param $jobs array of Job objects
 223          */
 224         static function batchInsert( $jobs ) {
 225                 if( !count( $jobs ) ) {
 226                         return;
 227                 }
 228                 $dbw = wfGetDB( DB_MASTER );
 229                 $rows = array();
 230                 foreach( $jobs as $job ) {
 231                         $rows[] = $job->insertFields();
 232                         if ( count( $rows ) >= 50 ) {
 233                                 # Do a small transaction to avoid slave lag
 234                                 $dbw->begin();
 235                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 236                                 $dbw->commit();
 237                                 $rows = array();
 238                         }
 239                 }
 240                 if ( $rows ) {
 241                         $dbw->begin();
 242                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 243                         $dbw->commit();
 244                 }
 245         }
 246
 247         /*-------------------------------------------------------------------------
 248          * Non-static functions
 249          *------------------------------------------------------------------------*/
 250
 251         function __construct( $command, $title, $params = false, $id = 0 ) {
 252                 $this->command = $command;
 253                 $this->title = $title;
 254                 $this->params = $params;
 255                 $this->id = $id;
 256
 257                 // A bit of premature generalisation
 258                 // Oh well, the whole class is premature generalisation really
 259                 $this->removeDuplicates = true;
 260         }
 261
 262         /**
 263          * Insert a single job into the queue.
 264          * @return bool true on success
 265          */
 266         function insert() {
 267                 $fields = $this->insertFields();
 268
 269                 $dbw = wfGetDB( DB_MASTER );
 270
 271                 if ( $this->removeDuplicates ) {
 272                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 273                         if ( $dbw->numRows( $res ) ) {
 274                                 return;
 275                         }
 276                 }
 277                 return $dbw->insert( 'job', $fields, __METHOD__ );
 278         }
 279
 280         protected function insertFields() {
 281                 $dbw = wfGetDB( DB_MASTER );
 282                 return array(
 283                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 284                         'job_cmd' => $this->command,
 285                         'job_namespace' => $this->title->getNamespace(),
 286                         'job_title' => $this->title->getDBkey(),
 287                         'job_params' => Job::makeBlob( $this->params )
 288                 );
 289         }
 290
 291         function toString() {
 292                 $paramString = '';
 293                 if ( $this->params ) {
 294                         foreach ( $this->params as $key => $value ) {
 295                                 if ( $paramString != '' ) {
 296                                         $paramString .= ' ';
 297                                 }
 298                                 $paramString .= "$key=$value";
 299                         }
 300                 }
 301
 302                 if ( is_object( $this->title ) ) {
 303                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 304                         if ( $paramString !== '' ) {
 305                                 $s .= ' ' . $paramString;
 306                         }
 307                         return $s;
 308                 } else {
 309                         return "{$this->command} $paramString";
 310                 }
 311         }
 312
 313         protected function setLastError( $error ) {
 314                 $this->error = $error;
 315         }
 316
 317         function getLastError() {
 318                 return $this->error;
 319         }
 320 }