includes/job/Job.php

   1 <?php
   2 /**
   3  * Job queue base code.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @defgroup JobQueue JobQueue
  22  */
  23
  24 /**
  25  * Class to both describe a background job and handle jobs.
  26  *
  27  * @ingroup JobQueue
  28  */
  29 abstract class Job {
  30
  31         /**
  32          * @var Title
  33          */
  34         var $title;
  35
  36         var $command,
  37                 $params,
  38                 $id,
  39                 $removeDuplicates,
  40                 $error;
  41
  42         /*-------------------------------------------------------------------------
  43          * Abstract functions
  44          *------------------------------------------------------------------------*/
  45
  46         /**
  47          * Run the job
  48          * @return boolean success
  49          */
  50         abstract function run();
  51
  52         /*-------------------------------------------------------------------------
  53          * Static functions
  54          *------------------------------------------------------------------------*/
  55
  56         /**
  57          * Pop a job of a certain type.  This tries less hard than pop() to
  58          * actually find a job; it may be adversely affected by concurrent job
  59          * runners.
  60          *
  61          * @param $type string
  62          *
  63          * @return Job
  64          */
  65         static function pop_type( $type ) {
  66                 wfProfilein( __METHOD__ );
  67
  68                 $dbw = wfGetDB( DB_MASTER );
  69
  70                 $dbw->begin( __METHOD__ );
  71
  72                 $row = $dbw->selectRow(
  73                         'job',
  74                         '*',
  75                         array( 'job_cmd' => $type ),
  76                         __METHOD__,
  77                         array( 'LIMIT' => 1, 'FOR UPDATE' )
  78                 );
  79
  80                 if ( $row === false ) {
  81                         $dbw->commit( __METHOD__ );
  82                         wfProfileOut( __METHOD__ );
  83                         return false;
  84                 }
  85
  86                 /* Ensure we "own" this row */
  87                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  88                 $affected = $dbw->affectedRows();
  89                 $dbw->commit( __METHOD__ );
  90
  91                 if ( $affected == 0 ) {
  92                         wfProfileOut( __METHOD__ );
  93                         return false;
  94                 }
  95
  96                 wfIncrStats( 'job-pop' );
  97                 $namespace = $row->job_namespace;
  98                 $dbkey = $row->job_title;
  99                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 100                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
 101                         $row->job_id );
 102
 103                 $job->removeDuplicates();
 104
 105                 wfProfileOut( __METHOD__ );
 106                 return $job;
 107         }
 108
 109         /**
 110          * Pop a job off the front of the queue
 111          *
 112          * @param $offset Integer: Number of jobs to skip
 113          * @return Job or false if there's no jobs
 114          */
 115         static function pop( $offset = 0 ) {
 116                 wfProfileIn( __METHOD__ );
 117
 118                 $dbr = wfGetDB( DB_SLAVE );
 119
 120                 /* Get a job from the slave, start with an offset,
 121                         scan full set afterwards, avoid hitting purged rows
 122
 123                         NB: If random fetch previously was used, offset
 124                                 will always be ahead of few entries
 125                 */
 126
 127                 $conditions = self::defaultQueueConditions();
 128
 129                 $offset = intval( $offset );
 130                 $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
 131
 132                 $row = $dbr->selectRow( 'job', '*',
 133                         array_merge( $conditions, array( "job_id >= $offset" ) ),
 134                         __METHOD__,
 135                         $options
 136                 );
 137
 138                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 139                 // and have lower IDs than jobs already executed, blame concurrency :)
 140                 //
 141                 if ( $row === false ) {
 142                         if ( $offset != 0 ) {
 143                                 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__, $options );
 144                         }
 145
 146                         if ( $row === false ) {
 147                                 wfProfileOut( __METHOD__ );
 148                                 return false;
 149                         }
 150                 }
 151
 152                 // Try to delete it from the master
 153                 $dbw = wfGetDB( DB_MASTER );
 154                 $dbw->begin( __METHOD__ );
 155                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 156                 $affected = $dbw->affectedRows();
 157                 $dbw->commit( __METHOD__ );
 158
 159                 if ( !$affected ) {
 160                         $dbw->begin( __METHOD__ );
 161
 162                         // Failed, someone else beat us to it
 163                         // Try getting a random row
 164                         $row = $dbw->selectRow( 'job', array( 'minjob' => 'MIN(job_id)',
 165                                 'maxjob' => 'MAX(job_id)' ), '1=1', __METHOD__ );
 166                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 167                                 // No jobs to get
 168                                 $dbw->rollback( __METHOD__ );
 169                                 wfProfileOut( __METHOD__ );
 170                                 return false;
 171                         }
 172                         // Get the random row
 173                         $row = $dbw->selectRow( 'job', '*',
 174                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 175                         if ( $row === false ) {
 176                                 // Random job gone before we got the chance to select it
 177                                 // Give up
 178                                 $dbw->rollback( __METHOD__ );
 179                                 wfProfileOut( __METHOD__ );
 180                                 return false;
 181                         }
 182                         // Delete the random row
 183                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 184                         $affected = $dbw->affectedRows();
 185                         $dbw->commit( __METHOD__ );
 186
 187                         if ( !$affected ) {
 188                                 // Random job gone before we exclusively deleted it
 189                                 // Give up
 190                                 wfProfileOut( __METHOD__ );
 191                                 return false;
 192                         }
 193                 }
 194
 195                 // If execution got to here, there's a row in $row that has been deleted from the database
 196                 // by this thread. Hence the concurrent pop was successful.
 197                 wfIncrStats( 'job-pop' );
 198                 $namespace = $row->job_namespace;
 199                 $dbkey = $row->job_title;
 200                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 201
 202                 if ( is_null( $title ) ) {
 203                         wfProfileOut( __METHOD__ );
 204                         return false;
 205                 }
 206
 207                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 208
 209                 // Remove any duplicates it may have later in the queue
 210                 $job->removeDuplicates();
 211
 212                 wfProfileOut( __METHOD__ );
 213                 return $job;
 214         }
 215
 216         /**
 217          * Create the appropriate object to handle a specific job
 218          *
 219          * @param $command String: Job command
 220          * @param $title Title: Associated title
 221          * @param $params Array|bool: Job parameters
 222          * @param $id Int: Job identifier
 223          * @throws MWException
 224          * @return Job
 225          */
 226         static function factory( $command, Title $title, $params = false, $id = 0 ) {
 227                 global $wgJobClasses;
 228                 if( isset( $wgJobClasses[$command] ) ) {
 229                         $class = $wgJobClasses[$command];
 230                         return new $class( $title, $params, $id );
 231                 }
 232                 throw new MWException( "Invalid job command `{$command}`" );
 233         }
 234
 235         /**
 236          * @param $params
 237          * @return string
 238          */
 239         static function makeBlob( $params ) {
 240                 if ( $params !== false ) {
 241                         return serialize( $params );
 242                 } else {
 243                         return '';
 244                 }
 245         }
 246
 247         /**
 248          * @param $blob
 249          * @return bool|mixed
 250          */
 251         static function extractBlob( $blob ) {
 252                 if ( (string)$blob !== '' ) {
 253                         return unserialize( $blob );
 254                 } else {
 255                         return false;
 256                 }
 257         }
 258
 259         /**
 260          * Batch-insert a group of jobs into the queue.
 261          * This will be wrapped in a transaction with a forced commit.
 262          *
 263          * This may add duplicate at insert time, but they will be
 264          * removed later on, when the first one is popped.
 265          *
 266          * @param $jobs array of Job objects
 267          */
 268         static function batchInsert( $jobs ) {
 269                 if ( !count( $jobs ) ) {
 270                         return;
 271                 }
 272                 $dbw = wfGetDB( DB_MASTER );
 273                 $rows = array();
 274
 275                 /**
 276                  * @var $job Job
 277                  */
 278                 foreach ( $jobs as $job ) {
 279                         $rows[] = $job->insertFields();
 280                         if ( count( $rows ) >= 50 ) {
 281                                 # Do a small transaction to avoid slave lag
 282                                 $dbw->begin( __METHOD__ );
 283                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 284                                 $dbw->commit( __METHOD__ );
 285                                 $rows = array();
 286                         }
 287                 }
 288                 if ( $rows ) { // last chunk
 289                         $dbw->begin( __METHOD__ );
 290                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 291                         $dbw->commit( __METHOD__ );
 292                 }
 293                 wfIncrStats( 'job-insert', count( $jobs ) );
 294         }
 295
 296         /**
 297          * Insert a group of jobs into the queue.
 298          *
 299          * Same as batchInsert() but does not commit and can thus
 300          * be rolled-back as part of a larger transaction. However,
 301          * large batches of jobs can cause slave lag.
 302          *
 303          * @param $jobs array of Job objects
 304          */
 305         static function safeBatchInsert( $jobs ) {
 306                 if ( !count( $jobs ) ) {
 307                         return;
 308                 }
 309                 $dbw = wfGetDB( DB_MASTER );
 310                 $rows = array();
 311                 foreach ( $jobs as $job ) {
 312                         $rows[] = $job->insertFields();
 313                         if ( count( $rows ) >= 500 ) {
 314                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 315                                 $rows = array();
 316                         }
 317                 }
 318                 if ( $rows ) { // last chunk
 319                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 320                 }
 321                 wfIncrStats( 'job-insert', count( $jobs ) );
 322         }
 323
 324
 325         /**
 326          * SQL conditions to apply on most JobQueue queries
 327          *
 328          * Whenever we exclude jobs types from the default queue, we want to make
 329          * sure that queries to the job queue actually ignore them.
 330          *
 331          * @return array SQL conditions suitable for Database:: methods
 332          */
 333         static function defaultQueueConditions( ) {
 334                 global $wgJobTypesExcludedFromDefaultQueue;
 335                 $conditions = array();
 336                 if ( count( $wgJobTypesExcludedFromDefaultQueue ) > 0 ) {
 337                         $dbr = wfGetDB( DB_SLAVE );
 338                         foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
 339                                 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
 340                         }
 341                 }
 342                 return $conditions;
 343         }
 344
 345         /*-------------------------------------------------------------------------
 346          * Non-static functions
 347          *------------------------------------------------------------------------*/
 348
 349         /**
 350          * @param $command
 351          * @param $title
 352          * @param $params array|bool
 353          * @param $id int
 354          */
 355         function __construct( $command, $title, $params = false, $id = 0 ) {
 356                 $this->command = $command;
 357                 $this->title = $title;
 358                 $this->params = $params;
 359                 $this->id = $id;
 360
 361                 // A bit of premature generalisation
 362                 // Oh well, the whole class is premature generalisation really
 363                 $this->removeDuplicates = true;
 364         }
 365
 366         /**
 367          * Insert a single job into the queue.
 368          * @return bool true on success
 369          */
 370         function insert() {
 371                 $fields = $this->insertFields();
 372
 373                 $dbw = wfGetDB( DB_MASTER );
 374
 375                 if ( $this->removeDuplicates ) {
 376                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 377                         if ( $dbw->numRows( $res ) ) {
 378                                 return true;
 379                         }
 380                 }
 381                 wfIncrStats( 'job-insert' );
 382                 return $dbw->insert( 'job', $fields, __METHOD__ );
 383         }
 384
 385         /**
 386          * @return array
 387          */
 388         protected function insertFields() {
 389                 $dbw = wfGetDB( DB_MASTER );
 390                 return array(
 391                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 392                         'job_cmd' => $this->command,
 393                         'job_namespace' => $this->title->getNamespace(),
 394                         'job_title' => $this->title->getDBkey(),
 395                         'job_timestamp' => $dbw->timestamp(),
 396                         'job_params' => Job::makeBlob( $this->params )
 397                 );
 398         }
 399
 400         /**
 401          * Remove jobs in the job queue which are duplicates of this job.
 402          * This is deadlock-prone and so starts its own transaction.
 403          */
 404         function removeDuplicates() {
 405                 if ( !$this->removeDuplicates ) {
 406                         return;
 407                 }
 408
 409                 $fields = $this->insertFields();
 410                 unset( $fields['job_id'] );
 411                 unset( $fields['job_timestamp'] );
 412                 $dbw = wfGetDB( DB_MASTER );
 413                 $dbw->begin( __METHOD__ );
 414                 $dbw->delete( 'job', $fields, __METHOD__ );
 415                 $affected = $dbw->affectedRows();
 416                 $dbw->commit( __METHOD__ );
 417                 if ( $affected ) {
 418                         wfIncrStats( 'job-dup-delete', $affected );
 419                 }
 420         }
 421
 422         /**
 423          * @return string
 424          */
 425         function toString() {
 426                 $paramString = '';
 427                 if ( $this->params ) {
 428                         foreach ( $this->params as $key => $value ) {
 429                                 if ( $paramString != '' ) {
 430                                         $paramString .= ' ';
 431                                 }
 432                                 $paramString .= "$key=$value";
 433                         }
 434                 }
 435
 436                 if ( is_object( $this->title ) ) {
 437                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 438                         if ( $paramString !== '' ) {
 439                                 $s .= ' ' . $paramString;
 440                         }
 441                         return $s;
 442                 } else {
 443                         return "{$this->command} $paramString";
 444                 }
 445         }
 446
 447         protected function setLastError( $error ) {
 448                 $this->error = $error;
 449         }
 450
 451         function getLastError() {
 452                 return $this->error;
 453         }
 454 }