5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @defgroup JobQueue JobQueue
24 if ( !defined( 'MEDIAWIKI' ) ) {
25 die( "This file is part of MediaWiki, it is not a valid entry point\n" );
29 * Class to both describe a background job and handle jobs.
46 /*-------------------------------------------------------------------------
48 *------------------------------------------------------------------------*/
52 * @return boolean success
54 abstract function run();
56 /*-------------------------------------------------------------------------
58 *------------------------------------------------------------------------*/
61 * Pop a job of a certain type. This tries less hard than pop() to
62 * actually find a job; it may be adversely affected by concurrent job
69 static function pop_type( $type ) {
70 wfProfilein( __METHOD__
);
72 $dbw = wfGetDB( DB_MASTER
);
74 $dbw->begin( __METHOD__
);
76 $row = $dbw->selectRow(
79 array( 'job_cmd' => $type ),
81 array( 'LIMIT' => 1, 'FOR UPDATE' )
84 if ( $row === false ) {
85 $dbw->commit( __METHOD__
);
86 wfProfileOut( __METHOD__
);
90 /* Ensure we "own" this row */
91 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
92 $affected = $dbw->affectedRows();
93 $dbw->commit( __METHOD__
);
95 if ( $affected == 0 ) {
96 wfProfileOut( __METHOD__
);
100 wfIncrStats( 'job-pop' );
101 $namespace = $row->job_namespace
;
102 $dbkey = $row->job_title
;
103 $title = Title
::makeTitleSafe( $namespace, $dbkey );
104 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
),
107 $job->removeDuplicates();
109 wfProfileOut( __METHOD__
);
114 * Pop a job off the front of the queue
116 * @param $offset Integer: Number of jobs to skip
117 * @return Job or false if there's no jobs
119 static function pop( $offset = 0 ) {
120 global $wgJobTypesExcludedFromDefaultQueue;
121 wfProfileIn( __METHOD__
);
123 $dbr = wfGetDB( DB_SLAVE
);
125 /* Get a job from the slave, start with an offset,
126 scan full set afterwards, avoid hitting purged rows
128 NB: If random fetch previously was used, offset
129 will always be ahead of few entries
131 $conditions = array();
132 if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) {
133 foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
134 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
137 $offset = intval( $offset );
138 $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
140 $row = $dbr->selectRow( 'job', '*',
141 array_merge( $conditions, array( "job_id >= $offset" ) ),
146 // Refetching without offset is needed as some of job IDs could have had delayed commits
147 // and have lower IDs than jobs already executed, blame concurrency :)
149 if ( $row === false ) {
150 if ( $offset != 0 ) {
151 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__
, $options );
154 if ( $row === false ) {
155 wfProfileOut( __METHOD__
);
160 // Try to delete it from the master
161 $dbw = wfGetDB( DB_MASTER
);
162 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
163 $affected = $dbw->affectedRows();
164 $dbw->commit( __METHOD__
);
167 // Failed, someone else beat us to it
168 // Try getting a random row
169 $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
170 'MAX(job_id) as maxjob' ), '1=1', __METHOD__
);
171 if ( $row === false ||
is_null( $row->minjob
) ||
is_null( $row->maxjob
) ) {
173 wfProfileOut( __METHOD__
);
176 // Get the random row
177 $row = $dbw->selectRow( 'job', '*',
178 'job_id >= ' . mt_rand( $row->minjob
, $row->maxjob
), __METHOD__
);
179 if ( $row === false ) {
180 // Random job gone before we got the chance to select it
182 wfProfileOut( __METHOD__
);
185 // Delete the random row
186 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
187 $affected = $dbw->affectedRows();
188 $dbw->commit( __METHOD__
);
191 // Random job gone before we exclusively deleted it
193 wfProfileOut( __METHOD__
);
198 // If execution got to here, there's a row in $row that has been deleted from the database
199 // by this thread. Hence the concurrent pop was successful.
200 wfIncrStats( 'job-pop' );
201 $namespace = $row->job_namespace
;
202 $dbkey = $row->job_title
;
203 $title = Title
::makeTitleSafe( $namespace, $dbkey );
204 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
), $row->job_id
);
206 // Remove any duplicates it may have later in the queue
207 $job->removeDuplicates();
209 wfProfileOut( __METHOD__
);
214 * Create the appropriate object to handle a specific job
216 * @param $command String: Job command
217 * @param $title Title: Associated title
218 * @param $params Array: Job parameters
219 * @param $id Int: Job identifier
222 static function factory( $command, $title, $params = false, $id = 0 ) {
223 global $wgJobClasses;
224 if( isset( $wgJobClasses[$command] ) ) {
225 $class = $wgJobClasses[$command];
226 return new $class( $title, $params, $id );
228 throw new MWException( "Invalid job command `{$command}`" );
235 static function makeBlob( $params ) {
236 if ( $params !== false ) {
237 return serialize( $params );
247 static function extractBlob( $blob ) {
248 if ( (string)$blob !== '' ) {
249 return unserialize( $blob );
256 * Batch-insert a group of jobs into the queue.
257 * This will be wrapped in a transaction with a forced commit.
259 * This may add duplicate at insert time, but they will be
260 * removed later on, when the first one is popped.
262 * @param $jobs array of Job objects
264 static function batchInsert( $jobs ) {
265 if ( !count( $jobs ) ) {
268 $dbw = wfGetDB( DB_MASTER
);
274 foreach ( $jobs as $job ) {
275 $rows[] = $job->insertFields();
276 if ( count( $rows ) >= 50 ) {
277 # Do a small transaction to avoid slave lag
278 $dbw->begin( __METHOD__
);
279 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
280 $dbw->commit( __METHOD__
);
284 if ( $rows ) { // last chunk
285 $dbw->begin( __METHOD__
);
286 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
287 $dbw->commit( __METHOD__
);
289 wfIncrStats( 'job-insert', count( $jobs ) );
293 * Insert a group of jobs into the queue.
295 * Same as batchInsert() but does not commit and can thus
296 * be rolled-back as part of a larger transaction. However,
297 * large batches of jobs can cause slave lag.
299 * @param $jobs array of Job objects
301 static function safeBatchInsert( $jobs ) {
302 if ( !count( $jobs ) ) {
305 $dbw = wfGetDB( DB_MASTER
);
307 foreach ( $jobs as $job ) {
308 $rows[] = $job->insertFields();
309 if ( count( $rows ) >= 500 ) {
310 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
314 if ( $rows ) { // last chunk
315 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
317 wfIncrStats( 'job-insert', count( $jobs ) );
320 /*-------------------------------------------------------------------------
321 * Non-static functions
322 *------------------------------------------------------------------------*/
327 * @param $params array
330 function __construct( $command, $title, $params = false, $id = 0 ) {
331 $this->command
= $command;
332 $this->title
= $title;
333 $this->params
= $params;
336 // A bit of premature generalisation
337 // Oh well, the whole class is premature generalisation really
338 $this->removeDuplicates
= true;
342 * Insert a single job into the queue.
343 * @return bool true on success
346 $fields = $this->insertFields();
348 $dbw = wfGetDB( DB_MASTER
);
350 if ( $this->removeDuplicates
) {
351 $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__
);
352 if ( $dbw->numRows( $res ) ) {
356 wfIncrStats( 'job-insert' );
357 return $dbw->insert( 'job', $fields, __METHOD__
);
363 protected function insertFields() {
364 $dbw = wfGetDB( DB_MASTER
);
366 'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
367 'job_cmd' => $this->command
,
368 'job_namespace' => $this->title
->getNamespace(),
369 'job_title' => $this->title
->getDBkey(),
370 'job_timestamp' => $dbw->timestamp(),
371 'job_params' => Job
::makeBlob( $this->params
)
376 * Remove jobs in the job queue which are duplicates of this job.
377 * This is deadlock-prone and so starts its own transaction.
379 function removeDuplicates() {
380 if ( !$this->removeDuplicates
) {
384 $fields = $this->insertFields();
385 unset( $fields['job_id'] );
386 $dbw = wfGetDB( DB_MASTER
);
387 $dbw->begin( __METHOD__
);
388 $dbw->delete( 'job', $fields, __METHOD__
);
389 $affected = $dbw->affectedRows();
390 $dbw->commit( __METHOD__
);
392 wfIncrStats( 'job-dup-delete', $affected );
399 function toString() {
401 if ( $this->params
) {
402 foreach ( $this->params
as $key => $value ) {
403 if ( $paramString != '' ) {
406 $paramString .= "$key=$value";
410 if ( is_object( $this->title
) ) {
411 $s = "{$this->command} " . $this->title
->getPrefixedDBkey();
412 if ( $paramString !== '' ) {
413 $s .= ' ' . $paramString;
417 return "{$this->command} $paramString";
421 protected function setLastError( $error ) {
422 $this->error
= $error;
425 function getLastError() {