Made runJobs.php respect time limits better and try to bail before OOMs

Change-Id: I93b9cebda591f15d42c401f4dc51ecd746d45a0d

diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php
index 9dac031..5eb779c 100644 (file)
--- a/maintenance/runJobs.php
+++ b/maintenance/runJobs.php
@@ -93,9 +93,11 @@ class RunJobs extends Maintenance {
                                ++$jobsRun;
                                $this->runJobsLog( $job->toString() . " STARTING" );
 
+                               // Set timer to stop the job if too much CPU time is used
+                               set_time_limit( $maxTime ?: 0 );
                                // Run the job...
-                               $t = microtime( true );
                                wfProfileIn( __METHOD__ . '-' . get_class( $job ) );
+                               $t = microtime( true );
                                try {
                                        $status = $job->run();
                                        $error = $job->getLastError();
@@ -104,8 +106,10 @@ class RunJobs extends Maintenance {
                                        $error = get_class( $e ) . ': ' . $e->getMessage();
                                        $e->report(); // write error to STDERR and the log
                                }
-                               wfProfileOut( __METHOD__ . '-' . get_class( $job ) );
                                $timeMs = intval( ( microtime( true ) - $t ) * 1000 );
+                               wfProfileOut( __METHOD__ . '-' . get_class( $job ) );
+                               // Disable the timer
+                               set_time_limit( 0 );
 
                                // Mark the job as done on success or when the job cannot be retried
                                if ( $status !== false || !$job->allowRetries() ) {
@@ -135,10 +139,35 @@ class RunJobs extends Maintenance {
                                if ( $jobsRun > 0 && ( $jobsRun % 100 ) == 0 ) {
                                        $group->waitForBackups();
                                }
+
+                               // Bail if near-OOM instead of in a job
+                               $this->assertMemoryOK();
                        }
                } while ( $job ); // stop when there are no jobs
        }
 
+       /**
+        * Make sure that this script is not too close to the memory usage limit
+        * @throws MWException
+        */
+       private function assertMemoryOK() {
+               static $maxBytes = null;
+               if ( $maxBytes === null ) {
+                       $m = array();
+                       if ( preg_match( '!^(\d+)(k|m|g|)$!i', ini_get( 'memory_limit' ), $m ) ) {
+                               list( , $num, $unit ) = $m;
+                               $conv = array( 'g' => 1024*1024*1024, 'm' => 1024*1024, 'k' => 1024, '' => 1 );
+                               $maxBytes = $num * $conv[strtolower( $unit )];
+                       } else {
+                               $maxBytes = 0;
+                       }
+               }
+               $usedBytes = memory_get_usage();
+               if ( $maxBytes && $usedBytes >= .95*$maxBytes ) {
+                       throw new MWException( "Detected excessive memory usage ($usedBytes/$maxBytes)." );
+               }
+       }
+
        /**
         * Log the job message
         * @param $msg String The message to log