Document memory-limit as a possible exit reason

[lhc/web/wiklou.git] / includes / jobqueue / JobRunner.php

diff --git a/includes/jobqueue/JobRunner.php b/includes/jobqueue/JobRunner.php
index a6d088f..ed29e59 100644 (file)
--- a/includes/jobqueue/JobRunner.php
+++ b/includes/jobqueue/JobRunner.php
@@ -80,7 +80,8 @@ class JobRunner implements LoggerAwareInterface {
         * The response map also has:
         *   - backoffs : the (job type => seconds) map of backoff times
         *   - elapsed  : the total time spent running tasks in ms
-        *   - reached  : the reason the script finished, one of (none-ready, job-limit, time-limit)
+        *   - reached  : the reason the script finished, one of (none-ready, job-limit, time-limit,
+        *  memory-limit)
         *
         * This method outputs status information only if a debug handler was set.
         * Any exceptions are caught and logged, but are not reported as output.
@@ -95,7 +96,7 @@ class JobRunner implements LoggerAwareInterface {
        public function run( array $options ) {
                global $wgJobClasses, $wgTrxProfilerLimits;
 
-               $response = array( 'jobs' => array(), 'reached' => 'none-ready' );
+               $response = [ 'jobs' => [], 'reached' => 'none-ready' ];
 
                $type = isset( $options['type'] ) ? $options['type'] : false;
                $maxJobs = isset( $options['maxJobs'] ) ? $options['maxJobs'] : false;
@@ -129,8 +130,8 @@ class JobRunner implements LoggerAwareInterface {
                $trxProfiler->setExpectations( $wgTrxProfilerLimits['JobRunner'], __METHOD__ );
 
                // Some jobs types should not run until a certain timestamp
-               $backoffs = array(); // map of (type => UNIX expiry)
-               $backoffDeltas = array(); // map of (type => seconds)
+               $backoffs = []; // map of (type => UNIX expiry)
+               $backoffDeltas = []; // map of (type => seconds)
                $wait = 'wait'; // block to read backoffs the first time
 
                $group = JobQueueGroup::singleton();
@@ -142,7 +143,7 @@ class JobRunner implements LoggerAwareInterface {
                do {
                        // Sync the persistent backoffs with concurrent runners
                        $backoffs = $this->syncBackoffDeltas( $backoffs, $backoffDeltas, $wait );
-                       $blacklist = $noThrottle ? array() : array_keys( $backoffs );
+                       $blacklist = $noThrottle ? [] : array_keys( $backoffs );
                        $wait = 'nowait'; // less important now
 
                        if ( $type === false ) {
@@ -186,12 +187,12 @@ class JobRunner implements LoggerAwareInterface {
                                                : $ttw;
                                }
 
-                               $response['jobs'][] = array(
+                               $response['jobs'][] = [
                                        'type'   => $jType,
                                        'status' => ( $info['status'] === false ) ? 'failed' : 'ok',
                                        'error'  => $info['error'],
                                        'time'   => $info['timeMs']
-                               );
+                               ];
                                $timeMsTotal += $info['timeMs'];
 
                                // Break out if we hit the job count or wall time limits...
@@ -208,7 +209,12 @@ class JobRunner implements LoggerAwareInterface {
                                // other wikis in the farm (on different masters) get a chance.
                                $timePassed = microtime( true ) - $lastCheckTime;
                                if ( $timePassed >= self::LAG_CHECK_PERIOD || $timePassed < 0 ) {
-                                       if ( !wfWaitForSlaves( $lastCheckTime, false, '*', self::MAX_ALLOWED_LAG ) ) {
+                                       try {
+                                               wfGetLBFactory()->waitForReplication( [
+                                                       'ifWritesSince' => $lastCheckTime,
+                                                       'timeout' => self::MAX_ALLOWED_LAG
+                                               ] );
+                                       } catch ( DBReplicationWaitError $e ) {
                                                $response['reached'] = 'slave-lag-limit';
                                                break;
                                        }
@@ -251,6 +257,7 @@ class JobRunner implements LoggerAwareInterface {
                $this->debugCallback( $msg );
 
                // Run the job...
+               $rssStart = $this->getMaxRssKb();
                $jobStartTime = microtime( true );
                try {
                        $status = $job->run();
@@ -272,6 +279,7 @@ class JobRunner implements LoggerAwareInterface {
                // Clear out title cache data from prior snapshots
                LinkCache::singleton()->clear();
                $timeMs = intval( ( microtime( true ) - $jobStartTime ) * 1000 );
+               $rssEnd = $this->getMaxRssKb();
 
                // Record how long jobs wait before getting popped
                $readyTs = $job->getReadyTimestamp();
@@ -288,6 +296,10 @@ class JobRunner implements LoggerAwareInterface {
                }
                // Track the execution time for jobs
                $stats->timing( "jobqueue.run.$jType", $timeMs );
+               // Track RSS increases for jobs (in case of memory leaks)
+               if ( $rssStart && $rssEnd ) {
+                       $stats->increment( "jobqueue.rss_delta.$jType", $rssEnd - $rssStart );
+               }
 
                if ( $status === false ) {
                        $msg = $job->toString() . " t=$timeMs error={$error}";
@@ -299,7 +311,16 @@ class JobRunner implements LoggerAwareInterface {
                        $this->debugCallback( $msg );
                }
 
-               return array( 'status' => $status, 'error' => $error, 'timeMs' => $timeMs );
+               return [ 'status' => $status, 'error' => $error, 'timeMs' => $timeMs ];
+       }
+
+       /**
+        * @return int|null Max memory RSS in kilobytes
+        */
+       private function getMaxRssKb() {
+               $info = wfGetRusage() ?: [];
+               // see http://linux.die.net/man/2/getrusage
+               return isset( $info['ru_maxrss'] ) ? (int)$info['ru_maxrss'] : null;
        }
 
        /**
@@ -354,14 +375,14 @@ class JobRunner implements LoggerAwareInterface {
                        flock( $handle, LOCK_UN );
                        fclose( $handle );
                        $ctime = microtime( true );
-                       $cBackoffs = json_decode( $content, true ) ?: array();
+                       $cBackoffs = json_decode( $content, true ) ?: [];
                        foreach ( $cBackoffs as $type => $timestamp ) {
                                if ( $timestamp < $ctime ) {
                                        unset( $cBackoffs[$type] );
                                }
                        }
                } else {
-                       $cBackoffs = array();
+                       $cBackoffs = [];
                }
 
                return $cBackoffs;
@@ -392,7 +413,7 @@ class JobRunner implements LoggerAwareInterface {
                }
                $ctime = microtime( true );
                $content = stream_get_contents( $handle );
-               $cBackoffs = json_decode( $content, true ) ?: array();
+               $cBackoffs = json_decode( $content, true ) ?: [];
                foreach ( $deltas as $type => $seconds ) {
                        $cBackoffs[$type] = isset( $cBackoffs[$type] ) && $cBackoffs[$type] >= $ctime
                                ? $cBackoffs[$type] + $seconds
@@ -408,7 +429,7 @@ class JobRunner implements LoggerAwareInterface {
                flock( $handle, LOCK_UN );
                fclose( $handle );
 
-               $deltas = array();
+               $deltas = [];
 
                return $cBackoffs;
        }
@@ -421,10 +442,10 @@ class JobRunner implements LoggerAwareInterface {
        private function checkMemoryOK() {
                static $maxBytes = null;
                if ( $maxBytes === null ) {
-                       $m = array();
+                       $m = [];
                        if ( preg_match( '!^(\d+)(k|m|g|)$!i', ini_get( 'memory_limit' ), $m ) ) {
                                list( , $num, $unit ) = $m;
-                               $conv = array( 'g' => 1073741824, 'm' => 1048576, 'k' => 1024, '' => 1 );
+                               $conv = [ 'g' => 1073741824, 'm' => 1048576, 'k' => 1024, '' => 1 ];
                                $maxBytes = $num * $conv[strtolower( $unit )];
                        } else {
                                $maxBytes = 0;
@@ -448,7 +469,7 @@ class JobRunner implements LoggerAwareInterface {
         */
        private function debugCallback( $msg ) {
                if ( $this->debug ) {
-                       call_user_func_array( $this->debug, array( wfTimestamp( TS_DB ) . " $msg\n" ) );
+                       call_user_func_array( $this->debug, [ wfTimestamp( TS_DB ) . " $msg\n" ] );
                }
        }