Back off from job types longer for DB read-only errors

Such error are likely to persist longer than other random
exceptions. In that case, it is better to avoid burning
through the job retry count.

Change-Id: I6785bd608856f98d21e0b0b05d3899a7081c38e2

diff --git a/includes/jobqueue/JobRunner.php b/includes/jobqueue/JobRunner.php
index 990f112..cacccbe 100644 (file)
--- a/includes/jobqueue/JobRunner.php
+++ b/includes/jobqueue/JobRunner.php
@@ -46,6 +46,7 @@ class JobRunner implements LoggerAwareInterface {
        const MAX_ALLOWED_LAG = 3; // abort if more than this much DB lag is present
        const LAG_CHECK_PERIOD = 1.0; // check replica DB lag this many seconds
        const ERROR_BACKOFF_TTL = 1; // seconds to back off a queue due to errors
+       const READONLY_BACKOFF_TTL = 30; // seconds to back off a queue due to read-only errors
 
        /**
         * @param callable $debug Optional debug output handler
@@ -190,7 +191,7 @@ class JobRunner implements LoggerAwareInterface {
 
                                // Back off of certain jobs for a while (for throttling and for errors)
                                if ( $info['status'] === false && mt_rand( 0, 49 ) == 0 ) {
-                                       $ttw = max( $ttw, self::ERROR_BACKOFF_TTL ); // too many errors
+                                       $ttw = max( $ttw, $this->getErrorBackoffTTL( $info['error'] ) );
                                        $backoffDeltas[$jType] = isset( $backoffDeltas[$jType] )
                                                ? $backoffDeltas[$jType] + $ttw
                                                : $ttw;
@@ -253,6 +254,16 @@ class JobRunner implements LoggerAwareInterface {
                return $response;
        }
 
+       /**
+        * @param string $error
+        * @return int TTL in seconds
+        */
+       private function getErrorBackoffTTL( $error ) {
+               return strpos( $error, 'DBReadOnlyError' ) !== false
+                       ? self::READONLY_BACKOFF_TTL
+                       : self::ERROR_BACKOFF_TTL;
+       }
+
        /**
         * @param Job $job
         * @param LBFactory $lbFactory