From 83fcb86dfabf3fafdf483f6f0ac273c87187e8f7 Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Fri, 26 May 2017 11:12:31 -0700 Subject: [PATCH] objectcache: add getMultiWithUnionSetCallback() method This supports callbacks that fetch all the missing values at once. Change-Id: I74747cc06f97edc9163178180597e6651743b048 --- includes/libs/objectcache/WANObjectCache.php | 122 +++++++++++++++ .../libs/objectcache/WANObjectCacheTest.php | 147 ++++++++++++++++++ 2 files changed, 269 insertions(+) diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php index ff5985487a..423d43eff9 100644 --- a/includes/libs/objectcache/WANObjectCache.php +++ b/includes/libs/objectcache/WANObjectCache.php @@ -1064,6 +1064,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * - c) The return value is a map of (cache key => value) in the order of $keyedIds * * @see WANObjectCache::getWithSetCallback() + * @see WANObjectCache::getMultiWithUnionSetCallback() * * Example usage: * @code @@ -1134,6 +1135,127 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { return $values; } + /** + * Method to fetch/regenerate multiple cache keys at once + * + * This works the same as getWithSetCallback() except: + * - a) The $keys argument expects the result of WANObjectCache::makeMultiKeys() + * - b) The $callback argument expects a callback returning a map of (ID => new value) + * for all entity IDs in $regenById and it takes the following arguments: + * - $ids: a list of entity IDs to regenerate + * - &$ttls: a reference to the (entity ID => new TTL) map + * - &$setOpts: a reference to options for set() which can be altered + * - c) The return value is a map of (cache key => value) in the order of $keyedIds + * - d) The "lockTSE" and "busyValue" options are ignored + * + * @see WANObjectCache::getWithSetCallback() + * @see WANObjectCache::getMultiWithSetCallback() + * + * Example usage: + * @code + * $rows = $cache->getMultiWithUnionSetCallback( + * // Map of cache keys to entity IDs + * $cache->makeMultiKeys( + * $this->fileVersionIds(), + * function ( $id, WANObjectCache $cache ) { + * return $cache->makeKey( 'file-version', $id ); + * } + * ), + * // Time-to-live (in seconds) + * $cache::TTL_DAY, + * // Function that derives the new key value + * function ( array $ids, array &$ttls, array &$setOpts ) { + * $dbr = wfGetDB( DB_REPLICA ); + * // Account for any snapshot/replica DB lag + * $setOpts += Database::getCacheSetOptions( $dbr ); + * + * // Load the rows for these files + * $rows = []; + * $res = $dbr->select( 'file', '*', [ 'id' => $ids ], __METHOD__ ); + * foreach ( $res as $row ) { + * $rows[$row->id] = $row; + * $mtime = wfTimestamp( TS_UNIX, $row->timestamp ); + * $ttls[$row->id] = $this->adaptiveTTL( $mtime, $ttls[$row->id] ); + * } + * + * return $rows; + * }, + * ] + * ); + * $files = array_map( [ __CLASS__, 'newFromRow' ], $rows ); + * @endcode + * + * @param ArrayIterator $keyedIds Result of WANObjectCache::makeMultiKeys() + * @param integer $ttl Seconds to live for key updates + * @param callable $callback Callback the yields entity regeneration callbacks + * @param array $opts Options map + * @return array Map of (cache key => value) in the same order as $keyedIds + * @since 1.30 + */ + final public function getMultiWithUnionSetCallback( + ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = [] + ) { + $idsByValueKey = iterator_to_array( $keyedIds, true ); + $valueKeys = array_keys( $idsByValueKey ); + $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : []; + unset( $opts['lockTSE'] ); // incompatible + unset( $opts['busyValue'] ); // incompatible + + // Load required keys into process cache in one go + $keysGet = $this->getNonProcessCachedKeys( $valueKeys, $opts ); + $this->warmupCache = $this->getRawKeysForWarmup( $keysGet, $checkKeys ); + $this->warmupKeyMisses = 0; + + // IDs of entities known to be in need of regeneration + $idsRegen = []; + + // Find out which keys are missing/deleted/stale + $curTTLs = []; + $asOfs = []; + $curByKey = $this->getMulti( $keysGet, $curTTLs, $checkKeys, $asOfs ); + foreach ( $keysGet as $key ) { + if ( !array_key_exists( $key, $curByKey ) || $curTTLs[$key] < 0 ) { + $idsRegen[] = $idsByValueKey[$key]; + } + } + + // Run the callback to populate the regeneration value map for all required IDs + $newSetOpts = []; + $newTTLsById = array_fill_keys( $idsRegen, $ttl ); + $newValsById = $idsRegen ? $callback( $idsRegen, $newTTLsById, $newSetOpts ) : []; + + // Wrap $callback to match the getWithSetCallback() format while passing $id to $callback + $id = null; // current entity ID + $func = function ( $oldValue, &$ttl, &$setOpts, $oldAsOf ) + use ( $callback, &$id, $newValsById, $newTTLsById, $newSetOpts ) + { + if ( array_key_exists( $id, $newValsById ) ) { + // Value was already regerated as expected, so use the value in $newValsById + $newValue = $newValsById[$id]; + $ttl = $newTTLsById[$id]; + $setOpts = $newSetOpts; + } else { + // Pre-emptive/popularity refresh and version mismatch cases are not detected + // above and thus $newValsById has no entry. Run $callback on this single entity. + $ttls = [ $id => $ttl ]; + $newValue = $callback( [ $id ], $ttls, $setOpts )[$id]; + $ttl = $ttls[$id]; + } + + return $newValue; + }; + + // Run the cache-aside logic using warmupCache instead of persistent cache queries + $values = []; + foreach ( $idsByValueKey as $key => $id ) { // preserve order + $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts ); + } + + $this->warmupCache = []; + + return $values; + } + /** * Locally set a key to expire soon if it is stale based on $purgeTimestamp * diff --git a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php index 3aeed09d46..728e6717d2 100644 --- a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php +++ b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php @@ -426,6 +426,153 @@ class WANObjectCacheTest extends PHPUnit_Framework_TestCase { ]; } + /** + * @dataProvider getMultiWithUnionSetCallback_provider + * @covers WANObjectCache::getMultiWithUnionSetCallback() + * @covers WANObjectCache::makeMultiKeys() + * @param array $extOpts + * @param bool $versioned + */ + public function testGetMultiWithUnionSetCallback( array $extOpts, $versioned ) { + $cache = $this->cache; + + $keyA = wfRandomString(); + $keyB = wfRandomString(); + $keyC = wfRandomString(); + $cKey1 = wfRandomString(); + $cKey2 = wfRandomString(); + + $wasSet = 0; + $genFunc = function ( array $ids, array &$ttls, array &$setOpts ) use ( + &$wasSet, &$priorValue, &$priorAsOf + ) { + $newValues = []; + foreach ( $ids as $id ) { + ++$wasSet; + $newValues[$id] = "@$id$"; + $ttls[$id] = 20; // override with another value + } + + return $newValues; + }; + + $wasSet = 0; + $keyedIds = new ArrayIterator( [ $keyA => 3353 ] ); + $value = "@3353$"; + $v = $cache->getMultiWithUnionSetCallback( + $keyedIds, 30, $genFunc, $extOpts ); + $this->assertEquals( $value, $v[$keyA], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated" ); + + $curTTL = null; + $cache->get( $keyA, $curTTL ); + $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' ); + $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' ); + + $wasSet = 0; + $value = "@efef$"; + $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); + $v = $cache->getMultiWithUnionSetCallback( + $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts ); + $this->assertEquals( $value, $v[$keyB], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated" ); + $this->assertEquals( 0, $cache->getWarmupKeyMisses(), "Keys warmed yet in process cache" ); + $v = $cache->getMultiWithUnionSetCallback( + $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts ); + $this->assertEquals( $value, $v[$keyB], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value not regenerated" ); + $this->assertEquals( 0, $cache->getWarmupKeyMisses(), "Keys warmed in process cache" ); + + $priorTime = microtime( true ); + usleep( 1 ); + $wasSet = 0; + $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] ); + $v = $cache->getMultiWithUnionSetCallback( + $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts + ); + $this->assertEquals( $value, $v[$keyB], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated due to check keys" ); + $t1 = $cache->getCheckKeyTime( $cKey1 ); + $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' ); + $t2 = $cache->getCheckKeyTime( $cKey2 ); + $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' ); + + $priorTime = microtime( true ); + $value = "@43636$"; + $wasSet = 0; + $keyedIds = new ArrayIterator( [ $keyC => 43636 ] ); + $v = $cache->getMultiWithUnionSetCallback( + $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts + ); + $this->assertEquals( $value, $v[$keyC], "Value returned" ); + $this->assertEquals( 1, $wasSet, "Value regenerated due to still-recent check keys" ); + $t1 = $cache->getCheckKeyTime( $cKey1 ); + $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' ); + $t2 = $cache->getCheckKeyTime( $cKey2 ); + $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' ); + + $curTTL = null; + $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] ); + if ( $versioned ) { + $this->assertEquals( $value, $v[$cache::VFLD_DATA], "Value returned" ); + } else { + $this->assertEquals( $value, $v, "Value returned" ); + } + $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" ); + + $wasSet = 0; + $key = wfRandomString(); + $keyedIds = new ArrayIterator( [ $key => 242424 ] ); + $v = $cache->getMultiWithUnionSetCallback( + $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); + $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value returned" ); + $cache->delete( $key ); + $keyedIds = new ArrayIterator( [ $key => 242424 ] ); + $v = $cache->getMultiWithUnionSetCallback( + $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts ); + $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" ); + $this->assertEquals( 1, $wasSet, "Value process cached while deleted" ); + + $calls = 0; + $ids = [ 1, 2, 3, 4, 5, 6 ]; + $keyFunc = function ( $id, WANObjectCache $wanCache ) { + return $wanCache->makeKey( 'test', $id ); + }; + $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc ); + $genFunc = function ( array $ids, array &$ttls, array &$setOpts ) use ( &$calls ) { + $newValues = []; + foreach ( $ids as $id ) { + ++$calls; + $newValues[$id] = "val-{$id}"; + } + + return $newValues; + }; + $values = $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc ); + + $this->assertEquals( + [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ], + array_values( $values ), + "Correct values in correct order" + ); + $this->assertEquals( + array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $this->cache ) ), + array_keys( $values ), + "Correct keys in correct order" + ); + $this->assertEquals( count( $ids ), $calls ); + + $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc ); + $this->assertEquals( count( $ids ), $calls, "Values cached" ); + } + + public static function getMultiWithUnionSetCallback_provider() { + return [ + [ [], false ], + [ [ 'version' => 1 ], true ] + ]; + } + /** * @covers WANObjectCache::getWithSetCallback() * @covers WANObjectCache::doGetWithSetCallback() -- 2.20.1