Merge "Turn a wfDebug call in MessageCache::get into a wfDebugLog call"

[lhc/web/wiklou.git] / includes / jobqueue / JobRunner.php

diff --git a/includes/jobqueue/JobRunner.php b/includes/jobqueue/JobRunner.php
index 8a708f1..4d2c618 100644 (file)
--- a/includes/jobqueue/JobRunner.php
+++ b/includes/jobqueue/JobRunner.php
@@ -76,6 +76,13 @@ class JobRunner {
                        $this->runJobsLog( "Executed $count periodic queue task(s)." );
                }
 
+               // Bail out if there is too much DB lag
+               list( , $maxLag ) = wfGetLBFactory()->getMainLB( wfWikiID() )->getMaxLag();
+               if ( $maxLag >= 5 ) {
+                       $response['reached'] = 'slave-lag-limit';
+                       return $response;
+               }
+
                // Flush any pending DB writes for sanity
                wfGetLBFactory()->commitMasterChanges();
 
@@ -84,11 +91,13 @@ class JobRunner {
                $backoffDeltas = array(); // map of (type => seconds)
                $wait = 'wait'; // block to read backoffs the first time
 
-               $jobsRun = 0; // counter
+               $jobsRun = 0;
                $timeMsTotal = 0;
                $flags = JobQueueGroup::USE_CACHE;
-               $sTime = microtime( true ); // time since jobs started running
-               $lastTime = microtime( true ); // time since last slave check
+               $checkPeriod = 5.0; // seconds
+               $checkPhase = mt_rand( 0, 1000 * $checkPeriod ) / 1000; // avoid stampedes
+               $startTime = microtime( true ); // time since jobs started running
+               $lastTime = microtime( true ) - $checkPhase; // time since last slave check
                do {
                        // Sync the persistent backoffs with concurrent runners
                        $backoffs = $this->syncBackoffDeltas( $backoffs, $backoffDeltas, $wait );
@@ -121,7 +130,7 @@ class JobRunner {
 
                                // Run the job...
                                wfProfileIn( __METHOD__ . '-' . get_class( $job ) );
-                               $sTime = microtime( true );
+                               $jobStartTime = microtime( true );
                                try {
                                        ++$jobsRun;
                                        $status = $job->run();
@@ -133,7 +142,7 @@ class JobRunner {
                                        $error = get_class( $e ) . ': ' . $e->getMessage();
                                        MWExceptionHandler::logException( $e );
                                }
-                               $timeMs = intval( ( microtime( true ) - $sTime ) * 1000 );
+                               $timeMs = intval( ( microtime( true ) - $jobStartTime ) * 1000 );
                                wfProfileOut( __METHOD__ . '-' . get_class( $job ) );
                                $timeMsTotal += $timeMs;
 
@@ -167,15 +176,20 @@ class JobRunner {
                                if ( $maxJobs && $jobsRun >= $maxJobs ) {
                                        $response['reached'] = 'job-limit';
                                        break;
-                               } elseif ( $maxTime && ( microtime( true ) - $sTime ) > $maxTime ) {
+                               } elseif ( $maxTime && ( microtime( true ) - $startTime ) > $maxTime ) {
                                        $response['reached'] = 'time-limit';
                                        break;
                                }
 
-                               // Don't let any of the main DB slaves get backed up
+                               // Don't let any of the main DB slaves get backed up.
+                               // This only waits for so long before exiting and letting
+                               // other wikis in the farm (on different masters) get a chance.
                                $timePassed = microtime( true ) - $lastTime;
                                if ( $timePassed >= 5 || $timePassed < 0 ) {
-                                       wfWaitForSlaves( $lastTime );
+                                       if ( !wfWaitForSlaves( $lastTime, wfWikiID(), false, 5 ) ) {
+                                               $response['reached'] = 'slave-lag-limit';
+                                               break;
+                                       }
                                        $lastTime = microtime( true );
                                }
                                // Don't let any queue slaves/backups fall behind