objectcache: add "staleTTL" into WANObjectCache::getWithSetCallback()

This simply involves passing it through to the set() call

Also added some related commons to adaptiveTTL() involving
usage of this option.

Change-Id: Id5833a5d4efb6cad2eb646832e5b0188e86e12fc

diff --git a/includes/libs/objectcache/HashBagOStuff.php b/includes/libs/objectcache/HashBagOStuff.php
index 6d583da..f8e3b17 100644 (file)
--- a/includes/libs/objectcache/HashBagOStuff.php
+++ b/includes/libs/objectcache/HashBagOStuff.php
@@ -52,7 +52,7 @@ class HashBagOStuff extends BagOStuff {
 
        protected function expire( $key ) {
                $et = $this->bag[$key][self::KEY_EXP];
-               if ( $et == self::TTL_INDEFINITE || $et > time() ) {
+               if ( $et == self::TTL_INDEFINITE || $et > $this->getCurrentTime() ) {
                        return false;
                }
 
@@ -115,4 +115,8 @@ class HashBagOStuff extends BagOStuff {
        public function clear() {
                $this->bag = [];
        }
+
+       protected function getCurrentTime() {
+               return time();
+       }
 }

diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php
index e63b32e..723ccc0 100644 (file)
--- a/includes/libs/objectcache/WANObjectCache.php
+++ b/includes/libs/objectcache/WANObjectCache.php
@@ -135,7 +135,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
        const TTL_LAGGED = 30;
        /** Idiom for delete() for "no hold-off" */
        const HOLDOFF_NONE = 0;
-       /** Idiom for set() for "do not augment the storage medium TTL" */
+       /** Idiom for set()/getWithSetCallback() for "do not augment the storage medium TTL" */
        const STALE_TTL_NONE = 0;
 
        /** Idiom for getWithSetCallback() for "no minimum required as-of timestamp" */
@@ -868,6 +868,11 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
         *      Default: WANObjectCache::LOW_TTL.
         *   - ageNew: Consider popularity refreshes only once a key reaches this age in seconds.
         *      Default: WANObjectCache::AGE_NEW.
+        *   - staleTTL: Seconds to keep the key around if it is stale. This means that on cache
+        *      miss the callback may get $oldValue/$oldAsOf values for keys that have already been
+        *      expired for this specified time. This is useful if adaptiveTTL() is used on the old
+        *      value's as-of time when it is verified as still being correct.
+        *      Default: WANObjectCache::STALE_TTL_NONE
         * @return mixed Value found or written to the key
         * @note Options added in 1.28: version, busyValue, hotTTR, ageNew, pcGroup, minAsOf
         * @note Callable type hints are not used to avoid class-autoloading
@@ -957,6 +962,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
        protected function doGetWithSetCallback( $key, $ttl, $callback, array $opts, &$asOf = null ) {
                $lowTTL = isset( $opts['lowTTL'] ) ? $opts['lowTTL'] : min( self::LOW_TTL, $ttl );
                $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : self::TSE_NONE;
+               $staleTTL = isset( $opts['staleTTL'] ) ? $opts['staleTTL'] : self::STALE_TTL_NONE;
                $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : [];
                $busyValue = isset( $opts['busyValue'] ) ? $opts['busyValue'] : null;
                $popWindow = isset( $opts['hotTTR'] ) ? $opts['hotTTR'] : self::HOT_TTR;
@@ -1056,6 +1062,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
 
                if ( $valueIsCacheable ) {
                        $setOpts['lockTSE'] = $lockTSE;
+                       $setOpts['staleTTL'] = $staleTTL;
                        // Use best known "since" timestamp if not provided
                        $setOpts += [ 'since' => $preCallbackTime ];
                        // Update the cache; this will fail if the key is tombstoned
@@ -1496,6 +1503,46 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
         *     $ttl = $cache->adaptiveTTL( $mtime, $cache::TTL_DAY );
         * @endcode
         *
+        * Another use case is when there are no applicable "last modified" fields in the DB,
+        * and there are too many dependencies for explicit purges to be viable, and the rate of
+        * change to relevant content is unstable, and it is highly valued to have the cached value
+        * be as up-to-date as possible.
+        *
+        * Example usage:
+        * @code
+        *     $query = "<some complex query>";
+        *     $idListFromComplexQuery = $cache->getWithSetCallback(
+        *         $cache->makeKey( 'complex-graph-query', $hashOfQuery ),
+        *         GraphQueryClass::STARTING_TTL,
+        *         function ( $oldValue, &$ttl, array &$setOpts, $oldAsOf ) use ( $query, $cache ) {
+        *             $gdb = $this->getReplicaGraphDbConnection();
+        *             // Account for any snapshot/replica DB lag
+        *             $setOpts += GraphDatabase::getCacheSetOptions( $gdb );
+        *
+        *             $newList = iterator_to_array( $gdb->query( $query ) );
+        *             sort( $newList, SORT_NUMERIC ); // normalize
+        *
+        *             $minTTL = GraphQueryClass::MIN_TTL;
+        *             $maxTTL = GraphQueryClass::MAX_TTL;
+        *             if ( $oldValue !== false ) {
+        *                 // Note that $oldAsOf is the last time this callback ran
+        *                 $ttl = ( $newList === $oldValue )
+        *                     // No change: cache for 150% of the age of $oldValue
+        *                     ? $cache->adaptiveTTL( $oldAsOf, $maxTTL, $minTTL, 1.5 )
+        *                     // Changed: cache for %50 of the age of $oldValue
+        *                     : $cache->adaptiveTTL( $oldAsOf, $maxTTL, $minTTL, .5 );
+        *             }
+        *
+        *             return $newList;
+        *        },
+        *        [
+        *             // Keep stale values around for doing comparisons for TTL calculations.
+        *             // High values improve long-tail keys hit-rates, though might waste space.
+        *             'staleTTL' => GraphQueryClass::GRACE_TTL
+        *        ]
+        *     );
+        * @endcode
+        *
         * @param int|float $mtime UNIX timestamp
         * @param int $maxTTL Maximum TTL (seconds)
         * @param int $minTTL Minimum TTL (seconds); Default: 30

diff --git a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
index 592f72a..b779231 100644 (file)
--- a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
+++ b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
@@ -266,6 +266,46 @@ class WANObjectCacheTest extends PHPUnit_Framework_TestCase {
                $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts );
                $this->assertEquals( $value, $v, "Value still returned after deleted" );
                $this->assertEquals( 1, $wasSet, "Value process cached while deleted" );
+
+               $backToTheFutureCache = new TimeAdjustableWANObjectCache( [
+                       'cache'   => new TimeAdjustableHashBagOStuff(),
+                       'pool'    => 'empty'
+               ] );
+
+               $oldValReceived = -1;
+               $oldAsOfReceived = -1;
+               $checkFunc = function ( $oldVal, &$ttl, array $setOpts, $oldAsOf )
+               use ( &$oldValReceived, &$oldAsOfReceived, &$wasSet ) {
+                       ++$wasSet;
+                       $oldValReceived = $oldVal;
+                       $oldAsOfReceived = $oldAsOf;
+
+                       return 'xxx' . $wasSet;
+               };
+
+               $wasSet = 0;
+               $key = wfRandomString();
+               $v = $backToTheFutureCache->getWithSetCallback(
+                       $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts );
+               $this->assertEquals( 'xxx1', $v, "Value returned" );
+               $this->assertEquals( false, $oldValReceived, "Callback got no stale value" );
+               $this->assertEquals( null, $oldAsOfReceived, "Callback got no stale value" );
+
+               $backToTheFutureCache->setTime( microtime( true ) + 40 );
+               $v = $backToTheFutureCache->getWithSetCallback(
+                       $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts );
+               $this->assertEquals( 'xxx2', $v, "Value still returned after expired" );
+               $this->assertEquals( 2, $wasSet, "Value recalculated while expired" );
+               $this->assertEquals( 'xxx1', $oldValReceived, "Callback got stale value" );
+               $this->assertNotEquals( null, $oldAsOfReceived, "Callback got stale value" );
+
+               $backToTheFutureCache->setTime( microtime( true ) + 300 );
+               $v = $backToTheFutureCache->getWithSetCallback(
+                       $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts );
+               $this->assertEquals( 'xxx3', $v, "Value still returned after expired" );
+               $this->assertEquals( 3, $wasSet, "Value recalculated while expired" );
+               $this->assertEquals( false, $oldValReceived, "Callback got no stale value" );
+               $this->assertEquals( null, $oldAsOfReceived, "Callback got no stale value" );
        }
 
        public static function getWithSetCallback_provider() {