From 24200e88d243b160f1ac6b6a8e71a7aab9e35550 Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Tue, 13 Sep 2016 00:00:11 -0700 Subject: [PATCH] objectcache: add WANObjectCache::getMultiWithSetCallback() This does what it says on the tin, e.g. a way to fetch multiple keys at once. Callbacks are still called in a serial manner when needed. Change-Id: I8e24a6de7f46499a53ec41636c5a4f106b9b3d09 --- includes/libs/objectcache/WANObjectCache.php | 116 +++++++++++- .../libs/objectcache/WANObjectCacheTest.php | 166 +++++++++++++++++- 2 files changed, 274 insertions(+), 8 deletions(-) diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php index 5f6e3249dd..d7db732c41 100644 --- a/includes/libs/objectcache/WANObjectCache.php +++ b/includes/libs/objectcache/WANObjectCache.php @@ -88,6 +88,9 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { /** @var int ERR_* constant for the "last error" registry */ protected $lastRelayError = self::ERR_NONE; + /** @var mixed[] Temporary warm-up cache */ + private $warmupCache = []; + /** Max time expected to pass between delete() and DB commit finishing */ const MAX_COMMIT_DELAY = 3; /** Max replication+snapshot lag before applying TTL_LAGGED or disallowing set() */ @@ -284,7 +287,14 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { } // Fetch all of the raw values - $wrappedValues = $this->cache->getMulti( array_merge( $valueKeys, $checkKeysFlat ) ); + $keysGet = array_merge( $valueKeys, $checkKeysFlat ); + if ( $this->warmupCache ) { + $wrappedValues = array_intersect_key( $this->warmupCache, array_flip( $keysGet ) ); + $keysGet = array_diff( $keysGet, array_keys( $wrappedValues ) ); // keys left to fetch + } else { + $wrappedValues = []; + } + $wrappedValues += $this->cache->getMulti( $keysGet ); // Time used to compare/init "check" keys (derived after getMulti() to be pessimistic) $now = microtime( true ); @@ -1016,6 +1026,95 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { return $value; } + /** + * Method to fetch/regenerate multiple cache keys at once + * + * This works the same as getWithSetCallback() except: + * - a) The $keys argument expects the result of WANObjectCache::makeMultiKeys() + * - b) The $callback argument expects a callback taking the following arguments: + * - $id: ID of an entity to query + * - $oldValue : the prior cache value or false if none was present + * - &$ttl : a reference to the new value TTL in seconds + * - &$setOpts : a reference to options for set() which can be altered + * - $oldAsOf : generation UNIX timestamp of $oldValue or null if not present + * Aside from the additional $id argument, the other arguments function the same + * way they do in getWithSetCallback(). + * - c) The return value is a map of (cache key => value) in the order of $keyedIds + * + * @see WANObjectCache::getWithSetCallback() + * + * Example usage: + * @code + * $rows = $cache->getMultiWithSetCallback( + * // Map of cache keys to entitiy IDs + * $cache->makeMultiKeys( + * $this->fileVersionIds(), + * function ( $id, WANObjectCache $cache ) { + * return $cache->makeKey( 'file-version', $id ); + * } + * ), + * // Time-to-live (in seconds) + * $cache::TTL_DAY, + * // Function that derives the new key value + * return function ( $id, $oldValue, &$ttl, array &$setOpts ) { + * $dbr = wfGetDB( DB_REPLICA ); + * // Account for any snapshot/replica DB lag + * $setOpts += Database::getCacheSetOptions( $dbr ); + * + * // Load the row for this file + * $row = $dbr->selectRow( 'file', '*', [ 'id' => $id ], __METHOD__ ); + * + * return $row ? (array)$row : false; + * }, + * [ + * // Process cache for 30 seconds + * 'pcTTL' => 30, + * // Use a dedicated 500 item cache (initialized on-the-fly) + * 'pcGroup' => 'file-versions:500' + * ] + * ); + * $files = array_map( [ __CLASS__, 'newFromRow' ], $rows ); + * @endcode + * + * @param ArrayIterator $keyedIds Result of WANObjectCache::makeMultiKeys() + * @param integer $ttl Seconds to live for key updates + * @param callable $callback Callback the yields entity regeneration callbacks + * @param array $opts Options map + * @return array Map of (cache key => value) in the same order as $keyedIds + * @since 1.28 + */ + final public function getMultiWithSetCallback( + ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = [] + ) { + $keysWarmUp = iterator_to_array( $keyedIds, true ); + $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : []; + foreach ( $checkKeys as $i => $checkKeyOrKeys ) { + if ( is_int( $i ) ) { + $keysWarmUp[] = $checkKeyOrKeys; + } else { + $keysWarmUp = array_merge( $keysWarmUp, $checkKeyOrKeys ); + } + } + + $this->warmupCache = $this->cache->getMulti( $keysWarmUp ); + $this->warmupCache += array_fill_keys( $keysWarmUp, false ); + + // Wrap $callback to match the getWithSetCallback() format while passing $id to $callback + $id = null; + $func = function ( $oldValue, &$ttl, array $setOpts, $oldAsOf ) use ( $callback, &$id ) { + return $callback( $id, $oldValue, $ttl, $setOpts, $oldAsOf ); + }; + + $values = []; + foreach ( $keyedIds as $key => $id ) { + $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts ); + } + + $this->warmupCache = []; + + return $values; + } + /** * @see BagOStuff::makeKey() * @param string ... Key component @@ -1036,6 +1135,21 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { return call_user_func_array( [ $this->cache, __FUNCTION__ ], func_get_args() ); } + /** + * @param array $entities List of entity IDs + * @param callable $keyFunc Callback yielding a key from (entity ID, this WANObjectCache) + * @return ArrayIterator Iterator yielding (cache key => entity ID) in $entities order + * @since 1.28 + */ + public function makeMultiKeys( array $entities, callable $keyFunc ) { + $map = []; + foreach ( $entities as $entity ) { + $map[$keyFunc( $entity, $this )] = $entity; + } + + return new ArrayIterator( $map ); + } + /** * Get the "last error" registered; clearLastError() should be called manually * @return int ERR_* class constant for the "last error" registry diff --git a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php index 99b959b3e2..f43a3f3fe9 100644 --- a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php +++ b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php @@ -22,6 +22,7 @@ class WANObjectCacheTest extends MediaWikiTestCase { } $wanCache = TestingAccessWrapper::newFromObject( $this->cache ); + /** @noinspection PhpUndefinedFieldInspection */ $this->internalCache = $wanCache->cache; } @@ -29,13 +30,14 @@ class WANObjectCacheTest extends MediaWikiTestCase { * @dataProvider provideSetAndGet * @covers WANObjectCache::set() * @covers WANObjectCache::get() + * @covers WANObjectCache::makeKey() * @param mixed $value * @param integer $ttl */ public function testSetAndGet( $value, $ttl ) { $curTTL = null; $asOf = null; - $key = wfRandomString(); + $key = $this->cache->makeKey( 'x', wfRandomString() ); $this->cache->get( $key, $curTTL, [], $asOf ); $this->assertNull( $curTTL, "Current TTL is null" ); @@ -71,9 +73,10 @@ class WANObjectCacheTest extends MediaWikiTestCase { /** * @covers WANObjectCache::get() + * @covers WANObjectCache::makeGlobalKey() */ public function testGetNotExists() { - $key = wfRandomString(); + $key = $this->cache->makeGlobalKey( 'y', wfRandomString(), 'p' ); $curTTL = null; $value = $this->cache->get( $key, $curTTL ); @@ -165,7 +168,7 @@ class WANObjectCacheTest extends MediaWikiTestCase { $priorAsOf = null; $wasSet = 0; $func = function( $old, &$ttl, &$opts, $asOf ) - use ( &$wasSet, &$priorValue, &$priorAsOf, $value ) + use ( &$wasSet, &$priorValue, &$priorAsOf, $value ) { ++$wasSet; $priorValue = $old; @@ -188,9 +191,9 @@ class WANObjectCacheTest extends MediaWikiTestCase { $wasSet = 0; $v = $cache->getWithSetCallback( $key, 30, $func, [ - 'lowTTL' => 0, - 'lockTSE' => 5, - ] + $extOpts ); + 'lowTTL' => 0, + 'lockTSE' => 5, + ] + $extOpts ); $this->assertEquals( $value, $v, "Value returned" ); $this->assertEquals( 0, $wasSet, "Value not regenerated" ); @@ -247,6 +250,150 @@ class WANObjectCacheTest extends MediaWikiTestCase { ]; } + /** + * @dataProvider getMultiWithSetCallback_provider + * @covers WANObjectCache::geMultitWithSetCallback() + * @covers WANObjectCache::makeMultiKeys() + * @param array $extOpts + * @param bool $versioned + */ + public function testGetMultiWithSetCallback( array $extOpts, $versioned ) { + $cache = $this->cache; + + $keyA = wfRandomString(); + $keyB = wfRandomString(); + $keyC = wfRandomString(); + $cKey1 = wfRandomString(); + $cKey2 = wfRandomString(); + + $priorValue = null; + $priorAsOf = null; + $wasSet = 0; + $genFunc = function ( $id, $old, &$ttl, &$opts, $asOf ) use ( + &$wasSet, &$priorValue, &$priorAsOf + ) { + ++$wasSet; + $priorValue = $old; + $priorAsOf = $asOf; + $ttl = 20; // override with another value + return "@$id$"; + }; + + $wasSet = 0; + $keyedIds = new ArrayIterator( [ $keyA => 3353 ] ); + $value = "@3353$"; + $v = $cache->getMultiWithSetCallback( + $keyedIds, 30, $genFunc, [ 'lockTSE' => 5 ] + $extOpts ); + $this->assertEquals( $value, $v[$keyA], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated" ); + $this->assertFalse( $priorValue, "No prior value" ); + $this->assertNull( $priorAsOf, "No prior value" ); + + $curTTL = null; + $cache->get( $keyA, $curTTL ); + $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); + $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); + + $wasSet = 0; + $value = "@efef$"; + $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); + $v = $cache->getMultiWithSetCallback( + $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5, ] + $extOpts ); + $this->assertEquals( $value, $v[$keyB], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated" ); + $v = $cache->getMultiWithSetCallback( + $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5, ] + $extOpts ); + $this->assertEquals( $value, $v[$keyB], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value not regenerated" ); + + $priorTime = microtime( true ); + usleep( 1 ); + $wasSet = 0; + $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); + $v = $cache->getMultiWithSetCallback( + $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts + ); + $this->assertEquals( $value, $v[$keyB], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated due to check keys" ); + $this->assertEquals( $value, $priorValue, "Has prior value" ); + $this->assertType( 'float', $priorAsOf, "Has prior value" ); + $t1 = $cache->getCheckKeyTime( $cKey1 ); + $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); + $t2 = $cache->getCheckKeyTime( $cKey2 ); + $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); + + $priorTime = microtime( true ); + $value = "@43636$"; + $wasSet = 0; + $keyedIds = new ArrayIterator( [ $keyC => 43636 ] ); + $v = $cache->getMultiWithSetCallback( + $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts + ); + $this->assertEquals( $value, $v[$keyC], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated due to still-recent check keys" ); + $t1 = $cache->getCheckKeyTime( $cKey1 ); + $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); + $t2 = $cache->getCheckKeyTime( $cKey2 ); + $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); + + $curTTL = null; + $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] ); + if ( $versioned ) { + $this->assertEquals( $value, $v[$cache::VFLD_DATA], "Value returned" ); + } else { + $this->assertEquals( $value, $v, "Value returned" ); + } + $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); + + $wasSet = 0; + $key = wfRandomString(); + $keyedIds = new ArrayIterator( [ $key => 242424 ] ); + $v = $cache->getMultiWithSetCallback( + $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); + $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value returned" ); + $cache->delete( $key ); + $keyedIds = new ArrayIterator( [ $key => 242424 ] ); + $v = $cache->getMultiWithSetCallback( + $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); + $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" ); + $this->assertEquals( 1, $wasSet, "Value process cached while deleted" ); + + $calls = 0; + $ids = [ 1, 2, 3, 4, 5, 6 ]; + $keyFunc = function ( $id, WANObjectCache $wanCache ) { + return $wanCache->makeKey( 'test', $id ); + }; + $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc ); + $genFunc = function ( $id, $oldValue, &$ttl, array &$setops ) use ( &$calls ) { + ++$calls; + + return "val-{$id}"; + }; + $values = $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc ); + + $this->assertEquals( + [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ], + array_values( $values ), + "Correct values in correct order" + ); + $this->assertEquals( + array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $this->cache ) ), + array_keys( $values ), + "Correct keys in correct order" + ); + $this->assertEquals( count( $ids ), $calls ); + + $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc ); + $this->assertEquals( count( $ids ), $calls, "Values cached" ); + } + + public static function getMultiWithSetCallback_provider() { + return [ + [ [], false ], + [ [ 'version' => 1 ], true ] + ]; + } + /** * @covers WANObjectCache::getWithSetCallback() * @covers WANObjectCache::doGetWithSetCallback() @@ -777,9 +924,14 @@ class WANObjectCacheTest extends MediaWikiTestCase { /** * @dataProvider provideAdaptiveTTL * @covers WANObjectCache::adaptiveTTL() + * @param float|int $ago + * @param int $maxTTL + * @param int $minTTL + * @param float $factor + * @param int $adaptiveTTL */ public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor, $adaptiveTTL ) { - $mtime = is_int( $ago ) ? time() - $ago : $ago; + $mtime = $ago ? time() - $ago : $ago; $margin = 5; $ttl = $this->cache->adaptiveTTL( $mtime, $maxTTL, $minTTL, $factor ); -- 2.20.1