X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2FJobQueue.php;h=afa757d752ed84f34e1ee6fe777c4edd6dc6059c;hb=7efc871cba8dc24aff2cd13d3ad4c8a20902bad7;hp=54af5e73a7d676d7a5b627eec58fc25b87014e92;hpb=3a6ac5a3c3c59c15d7034b914f9549ce60fb7712;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/JobQueue.php b/includes/JobQueue.php index 54af5e73a7..afa757d752 100644 --- a/includes/JobQueue.php +++ b/includes/JobQueue.php @@ -1,9 +1,17 @@ selectRow( 'job', '*', array( 'job_cmd' => $type ), __METHOD__, + array( 'LIMIT' => 1 )); + + if ($row === false) { + wfProfileOut( __METHOD__ ); + return false; + } + + /* Ensure we "own" this row */ + $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); + $affected = $dbw->affectedRows(); + + if ($affected == 0) { + wfProfileOut( __METHOD__ ); + return false; + } + + $namespace = $row->job_namespace; + $dbkey = $row->job_title; + $title = Title::makeTitleSafe( $namespace, $dbkey ); + $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id ); + + $dbw->delete( 'job', $job->insertFields(), __METHOD__ ); + $dbw->immediateCommit(); + + wfProfileOut( __METHOD__ ); + return $job; + } + /** * Pop a job off the front of the queue - * @static + * * @param $offset Number of jobs to skip * @return Job or false if there's no jobs */ @@ -34,31 +92,31 @@ abstract class Job { $dbr = wfGetDB( DB_SLAVE ); - /* Get a job from the slave, start with an offset, + /* Get a job from the slave, start with an offset, scan full set afterwards, avoid hitting purged rows - - NB: If random fetch previously was used, offset - will always be ahead of few entries + + NB: If random fetch previously was used, offset + will always be ahead of few entries */ - + $row = $dbr->selectRow( 'job', '*', "job_id >= ${offset}", __METHOD__, array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 )); - + // Refetching without offset is needed as some of job IDs could have had delayed commits // and have lower IDs than jobs already executed, blame concurrency :) - // + // if ( $row === false) { if ($offset!=0) $row = $dbr->selectRow( 'job', '*', '', __METHOD__, array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 )); - + if ($row === false ) { wfProfileOut( __METHOD__ ); return false; - } - } + } + } $offset = $row->job_id; - + // Try to delete it from the master $dbw = wfGetDB( DB_MASTER ); $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); @@ -69,7 +127,7 @@ abstract class Job { // Failed, someone else beat us to it // Try getting a random row $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob', - 'MAX(job_id) as maxjob' ), "job_id >= $offset", __METHOD__ ); + 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ ); if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) { // No jobs to get wfProfileOut( __METHOD__ ); @@ -88,7 +146,7 @@ abstract class Job { $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ ); $affected = $dbw->affectedRows(); $dbw->immediateCommit(); - + if ( !$affected ) { // Random job gone before we exclusively deleted it // Give up @@ -96,34 +154,40 @@ abstract class Job { return false; } } - + // If execution got to here, there's a row in $row that has been deleted from the database // by this thread. Hence the concurrent pop was successful. $namespace = $row->job_namespace; $dbkey = $row->job_title; $title = Title::makeTitleSafe( $namespace, $dbkey ); $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id ); - + // Remove any duplicates it may have later in the queue + // Deadlock prone section + $dbw->begin(); $dbw->delete( 'job', $job->insertFields(), __METHOD__ ); - + $dbw->commit(); + wfProfileOut( __METHOD__ ); return $job; } - /** - * Create an object of a subclass + /** + * Create the appropriate object to handle a specific job + * + * @param $command String: Job command + * @param $title Title: Associated title + * @param $params Array: Job parameters + * @param $id Int: Job identifier + * @return Job */ static function factory( $command, $title, $params = false, $id = 0 ) { - switch ( $command ) { - case 'refreshLinks': - return new RefreshLinksJob( $title, $params, $id ); - case 'htmlCacheUpdate': - case 'html_cache_update': # BC - return new HTMLCacheUpdateJob( $title, $params['table'], $params['start'], $params['end'], $id ); - default: - throw new MWException( "Invalid job command \"$command\"" ); + global $wgJobClasses; + if( isset( $wgJobClasses[$command] ) ) { + $class = $wgJobClasses[$command]; + return new $class( $title, $params, $id ); } + throw new MWException( "Invalid job command `{$command}`" ); } static function makeBlob( $params ) { @@ -142,6 +206,38 @@ abstract class Job { } } + /** + * Batch-insert a group of jobs into the queue. + * This will be wrapped in a transaction with a forced commit. + * + * This may add duplicate at insert time, but they will be + * removed later on, when the first one is popped. + * + * @param $jobs array of Job objects + */ + static function batchInsert( $jobs ) { + if( !count( $jobs ) ) { + return; + } + $dbw = wfGetDB( DB_MASTER ); + $rows = array(); + foreach( $jobs as $job ) { + $rows[] = $job->insertFields(); + if ( count( $rows ) >= 50 ) { + # Do a small transaction to avoid slave lag + $dbw->begin(); + $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); + $dbw->commit(); + $rows = array(); + } + } + if ( $rows ) { + $dbw->begin(); + $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); + $dbw->commit(); + } + } + /*------------------------------------------------------------------------- * Non-static functions *------------------------------------------------------------------------*/ @@ -164,7 +260,7 @@ abstract class Job { $fields = $this->insertFields(); $dbw = wfGetDB( DB_MASTER ); - + if ( $this->removeDuplicates ) { $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ ); if ( $dbw->numRows( $res ) ) { @@ -174,7 +270,7 @@ abstract class Job { $fields['job_id'] = $dbw->nextSequenceValue( 'job_job_id_seq' ); $dbw->insert( 'job', $fields, __METHOD__ ); } - + protected function insertFields() { return array( 'job_cmd' => $this->command, @@ -183,34 +279,7 @@ abstract class Job { 'job_params' => Job::makeBlob( $this->params ) ); } - - /** - * Batch-insert a group of jobs into the queue. - * This will be wrapped in a transaction with a forced commit. - * - * This may add duplicate at insert time, but they will be - * removed later on, when the first one is popped. - * - * @param $jobs array of Job objects - */ - static function batchInsert( $jobs ) { - if( count( $jobs ) ) { - $dbw = wfGetDB( DB_MASTER ); - $dbw->begin(); - foreach( $jobs as $job ) { - $rows[] = $job->insertFields(); - } - $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' ); - $dbw->commit(); - } - } - /** - * Run the job - * @return boolean success - */ - abstract function run(); - function toString() { $paramString = ''; if ( $this->params ) { @@ -233,51 +302,11 @@ abstract class Job { } } - function getLastError() { - return $this->error; - } -} - -class RefreshLinksJob extends Job { - function __construct( $title, $params = '', $id = 0 ) { - parent::__construct( 'refreshLinks', $title, $params, $id ); + protected function setLastError( $error ) { + $this->error = $error; } - /** - * Run a refreshLinks job - * @return boolean success - */ - function run() { - global $wgParser; - wfProfileIn( __METHOD__ ); - - $linkCache =& LinkCache::singleton(); - $linkCache->clear(); - - if ( is_null( $this->title ) ) { - $this->error = "refreshLinks: Invalid title"; - wfProfileOut( __METHOD__ ); - return false; - } - - $revision = Revision::newFromTitle( $this->title ); - if ( !$revision ) { - $this->error = 'refreshLinks: Article not found "' . $this->title->getPrefixedDBkey() . '"'; - wfProfileOut( __METHOD__ ); - return false; - } - - wfProfileIn( __METHOD__.'-parse' ); - $options = new ParserOptions; - $parserOutput = $wgParser->parse( $revision->getText(), $this->title, $options, true, true, $revision->getId() ); - wfProfileOut( __METHOD__.'-parse' ); - wfProfileIn( __METHOD__.'-update' ); - $update = new LinksUpdate( $this->title, $parserOutput, false ); - $update->doUpdate(); - wfProfileOut( __METHOD__.'-update' ); - wfProfileOut( __METHOD__ ); - return true; + function getLastError() { + return $this->error; } } - -?>