X-Git-Url: http://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2Flibs%2Fobjectcache%2FWANObjectCache.php;h=69edb11d811a90be8d18a25df66d7f75c58f7e9b;hb=7af9c25c885aaac732b990fb4b0172f36d77129d;hp=2c533b9bc6102947357ebb7791984bbbeff970fc;hpb=db587c50882711fb626dd95ee316f96f845ddeaf;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php index 2c533b9bc6..69edb11d81 100644 --- a/includes/libs/objectcache/WANObjectCache.php +++ b/includes/libs/objectcache/WANObjectCache.php @@ -118,22 +118,25 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt protected $cache; /** @var MapCacheLRU[] Map of group PHP instance caches */ protected $processCaches = []; + /** @var LoggerInterface */ + protected $logger; + /** @var StatsdDataFactoryInterface */ + protected $stats; + /** @var callable|null Function that takes a WAN cache callback and runs it later */ + protected $asyncHandler; + /** @bar bool Whether to use mcrouter key prefixing for routing */ protected $mcrouterAware; /** @var string Physical region for mcrouter use */ protected $region; /** @var string Cache cluster name for mcrouter use */ protected $cluster; - /** @var LoggerInterface */ - protected $logger; - /** @var StatsdDataFactoryInterface */ - protected $stats; /** @var bool Whether to use "interim" caching while keys are tombstoned */ protected $useInterimHoldOffCaching = true; - /** @var callable|null Function that takes a WAN cache callback and runs it later */ - protected $asyncHandler; /** @var float Unix timestamp of the oldest possible valid values */ protected $epoch; + /** @var string Stable secret used for hasing long strings into key components */ + protected $secret; /** @var int Callback stack depth for getWithSetCallback() */ private $callbackDepth = 0; @@ -145,87 +148,104 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt /** @var float|null */ private $wallClockOverride; - /** Max time expected to pass between delete() and DB commit finishing */ + /** @var int Max expected seconds to pass between delete() and DB commit finishing */ const MAX_COMMIT_DELAY = 3; - /** Max replication+snapshot lag before applying TTL_LAGGED or disallowing set() */ + /** @var int Max expected seconds of combined lag from replication and view snapshots */ const MAX_READ_LAG = 7; - /** Seconds to tombstone keys on delete() */ - const HOLDOFF_TTL = 11; // MAX_COMMIT_DELAY + MAX_READ_LAG + 1 - - /** Seconds to keep dependency purge keys around */ - const CHECK_KEY_TTL = self::TTL_YEAR; - /** Seconds to keep interim value keys for tombstoned keys around */ - const INTERIM_KEY_TTL = 1; - - /** Seconds to keep lock keys around */ - const LOCK_TTL = 10; - /** Seconds to no-op key set() calls to avoid large blob I/O stampedes */ - const COOLOFF_TTL = 1; - /** Default remaining TTL at which to consider pre-emptive regeneration */ + /** @var int Seconds to tombstone keys on delete() and treat as volatile after invalidation */ + const HOLDOFF_TTL = self::MAX_COMMIT_DELAY + self::MAX_READ_LAG + 1; + + /** @var int Idiom for getWithSetCallback() meaning "do not store the callback result" */ + const TTL_UNCACHEABLE = -1; + + /** @var int Consider regeneration if the key will expire within this many seconds */ const LOW_TTL = 30; - /** Max TTL to store keys when a data sourced is lagged */ + /** @var int Max TTL, in seconds, to store keys when a data sourced is lagged */ const TTL_LAGGED = 30; - /** Never consider performing "popularity" refreshes until a key reaches this age */ - const AGE_NEW = 60; - /** The time length of the "popularity" refresh window for hot keys */ + /** @var int Expected time-till-refresh, in seconds, if the key is accessed once per second */ const HOT_TTR = 900; - /** Hits/second for a refresh to be expected within the "popularity" window */ - const HIT_RATE_HIGH = 1; - /** Seconds to ramp up to the "popularity" refresh chance after a key is no longer new */ - const RAMPUP_TTL = 30; + /** @var int Minimum key age, in seconds, for expected time-till-refresh to be considered */ + const AGE_NEW = 60; - /** Idiom for getWithSetCallback() meaning "do not store the callback result" */ - const TTL_UNCACHEABLE = -1; - /** Idiom for getWithSetCallback() meaning "no regeneration mutex based on key hotness" */ + /** @var int Idiom for getWithSetCallback() meaning "no cache stampede mutex required" */ const TSE_NONE = -1; - /** Idiom for set()/getWithSetCallback() meaning "no post-expiration persistence" */ + + /** @var int Idiom for set()/getWithSetCallback() meaning "no post-expiration persistence" */ const STALE_TTL_NONE = 0; - /** Idiom for set()/getWithSetCallback() meaning "no post-expiration grace period" */ + /** @var int Idiom for set()/getWithSetCallback() meaning "no post-expiration grace period" */ const GRACE_TTL_NONE = 0; - /** Idiom for delete()/touchCheckKey() meaning "no hold-off period for cache writes" */ - const HOLDOFF_NONE = 0; + /** @var int Idiom for delete()/touchCheckKey() meaning "no hold-off period" */ + const HOLDOFF_TTL_NONE = 0; + /** @var int Alias for HOLDOFF_TTL_NONE (b/c) (deprecated since 1.34) */ + const HOLDOFF_NONE = self::HOLDOFF_TTL_NONE; - /** Idiom for getWithSetCallback() meaning "no minimum required as-of timestamp" */ + /** @var float Idiom for getWithSetCallback() meaning "no minimum required as-of timestamp" */ const MIN_TIMESTAMP_NONE = 0.0; - /** Tiny negative float to use when CTL comes up >= 0 due to clock skew */ - const TINY_NEGATIVE = -0.000001; - /** Tiny positive float to use when using "minTime" to assert an inequality */ - const TINY_POSTIVE = 0.000001; - - /** Milliseconds of delay after get() where set() storms are a consideration with 'lockTSE' */ - const SET_DELAY_HIGH_MS = 50; - /** Min millisecond set() backoff for keys in hold-off (far less than INTERIM_KEY_TTL) */ - const RECENT_SET_LOW_MS = 50; - /** Max millisecond set() backoff for keys in hold-off (far less than INTERIM_KEY_TTL) */ - const RECENT_SET_HIGH_MS = 100; + /** @var string Default process cache name and max key count */ + const PC_PRIMARY = 'primary:1000'; - /** Parameter to get()/getMulti() to return extra information by reference */ + /** @var int Idion for get()/getMulti() to return extra information by reference */ const PASS_BY_REF = -1; - /** Cache format version number */ - const VERSION = 1; - - const FLD_VERSION = 0; // key to cache version number - const FLD_VALUE = 1; // key to the cached value - const FLD_TTL = 2; // key to the original TTL - const FLD_TIME = 3; // key to the cache time - const FLD_FLAGS = 4; // key to the flags bitfield (reserved number) - const FLD_HOLDOFF = 5; // key to any hold-off TTL - - const VALUE_KEY_PREFIX = 'WANCache:v:'; - const INTERIM_KEY_PREFIX = 'WANCache:i:'; - const TIME_KEY_PREFIX = 'WANCache:t:'; - const MUTEX_KEY_PREFIX = 'WANCache:m:'; - const COOLOFF_KEY_PREFIX = 'WANCache:c:'; - - const PURGE_VAL_PREFIX = 'PURGED:'; - - const VFLD_DATA = 'WOC:d'; // key to the value of versioned data - const VFLD_VERSION = 'WOC:v'; // key to the version of the value present - - const PC_PRIMARY = 'primary:1000'; // process cache name and max key count + /** @var int Seconds to keep dependency purge keys around */ + private static $CHECK_KEY_TTL = self::TTL_YEAR; + /** @var int Seconds to keep interim value keys for tombstoned keys around */ + private static $INTERIM_KEY_TTL = 1; + + /** @var int Seconds to keep lock keys around */ + private static $LOCK_TTL = 10; + /** @var int Seconds to no-op key set() calls to avoid large blob I/O stampedes */ + private static $COOLOFF_TTL = 1; + /** @var int Seconds to ramp up the chance of regeneration due to expected time-till-refresh */ + private static $RAMPUP_TTL = 30; + + /** @var float Tiny negative float to use when CTL comes up >= 0 due to clock skew */ + private static $TINY_NEGATIVE = -0.000001; + /** @var float Tiny positive float to use when using "minTime" to assert an inequality */ + private static $TINY_POSTIVE = 0.000001; + + /** @var int Milliseconds of key fetch/validate/regenerate delay prone to set() stampedes */ + private static $SET_DELAY_HIGH_MS = 50; + /** @var int Min millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL) */ + private static $RECENT_SET_LOW_MS = 50; + /** @var int Max millisecond set() backoff during hold-off (far less than INTERIM_KEY_TTL) */ + private static $RECENT_SET_HIGH_MS = 100; + + /** @var int Consider value generation slow if it takes more than this many seconds */ + private static $GENERATION_SLOW_SEC = 3; + + /** @var int Key to the tombstone entry timestamp */ + private static $PURGE_TIME = 0; + /** @var int Key to the tombstone entry hold-off TTL */ + private static $PURGE_HOLDOFF = 1; + + /** @var int Cache format version number */ + private static $VERSION = 1; + + /** @var int Key to WAN cache version number */ + private static $FLD_FORMAT_VERSION = 0; + /** @var int Key to the cached value */ + private static $FLD_VALUE = 1; + /** @var int Key to the original TTL */ + private static $FLD_TTL = 2; + /** @var int Key to the cache timestamp */ + private static $FLD_TIME = 3; + /** @var int Key to the flags bit field (reserved number) */ + private static /** @noinspection PhpUnusedPrivateFieldInspection */ $FLD_FLAGS = 4; + /** @var int Key to collection cache version number */ + private static $FLD_VALUE_VERSION = 5; + /** @var int Key to how long it took to generate the value */ + private static $FLD_GENERATION_TIME = 6; + + private static $VALUE_KEY_PREFIX = 'WANCache:v:'; + private static $INTERIM_KEY_PREFIX = 'WANCache:i:'; + private static $TIME_KEY_PREFIX = 'WANCache:t:'; + private static $MUTEX_KEY_PREFIX = 'WANCache:m:'; + private static $COOLOFF_KEY_PREFIX = 'WANCache:c:'; + + private static $PURGE_VAL_PREFIX = 'PURGED:'; /** * @param array $params @@ -250,13 +270,15 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * is configured to interpret /// key prefixes as routes. This * requires that "region" and "cluster" are both set above. [optional] * - epoch: lowest UNIX timestamp a value/tombstone must have to be valid. [optional] + * - secret: stable secret used for hashing long strings into key components. [optional] */ public function __construct( array $params ) { $this->cache = $params['cache']; $this->region = $params['region'] ?? 'main'; $this->cluster = $params['cluster'] ?? 'wan-main'; $this->mcrouterAware = !empty( $params['mcrouterAware'] ); - $this->epoch = $params['epoch'] ?? 1.0; + $this->epoch = $params['epoch'] ?? 0; + $this->secret = $params['secret'] ?? (string)$this->epoch; $this->setLogger( $params['logger'] ?? new NullLogger() ); $this->stats = $params['stats'] ?? new NullStatsdDataFactory(); @@ -314,17 +336,18 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * Consider using getWithSetCallback() instead of get() and set() cycles. * That method has cache slam avoiding features for hot/expensive keys. * - * Pass $info as WANObjectCache::PASS_BY_REF to transform it into a cache key info map. + * Pass $info as WANObjectCache::PASS_BY_REF to transform it into a cache key metadata map. * This map includes the following metadata: * - asOf: UNIX timestamp of the value or null if the key is nonexistant * - tombAsOf: UNIX timestamp of the tombstone or null if the key is not tombstoned * - lastCKPurge: UNIX timestamp of the highest check key or null if none provided + * - version: cached value version number or null if the key is nonexistant * * Otherwise, $info will transform into the cached value timestamp. * * @param string $key Cache key made from makeKey() or makeGlobalKey() * @param mixed|null &$curTTL Approximate TTL left on the key if present/tombstoned [returned] - * @param array $checkKeys List of "check" keys + * @param string[] $checkKeys The "check" keys used to validate the value * @param mixed|null &$info Key info if WANObjectCache::PASS_BY_REF [returned] * @return mixed Value of cache key or false on failure */ @@ -339,7 +362,8 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt $info = [ 'asOf' => $infoByKey[$key]['asOf'] ?? null, 'tombAsOf' => $infoByKey[$key]['tombAsOf'] ?? null, - 'lastCKPurge' => $infoByKey[$key]['lastCKPurge'] ?? null + 'lastCKPurge' => $infoByKey[$key]['lastCKPurge'] ?? null, + 'version' => $infoByKey[$key]['version'] ?? null ]; } else { $info = $infoByKey[$key]['asOf'] ?? null; // b/c @@ -352,20 +376,23 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * Fetch the value of several keys from cache * * Pass $info as WANObjectCache::PASS_BY_REF to transform it into a map of cache keys - * to cache key info maps, each having the same style as those of WANObjectCache::get(). + * to cache key metadata maps, each having the same style as those of WANObjectCache::get(). * All the cache keys listed in $keys will have an entry. * * Othwerwise, $info will transform into a map of (cache key => cached value timestamp). * Only the cache keys listed in $keys that exists or are tombstoned will have an entry. * + * $checkKeys holds the "check" keys used to validate values of applicable keys. The integer + * indexes hold "check" keys that apply to all of $keys while the string indexes hold "check" + * keys that only apply to the cache key with that name. + * * @see WANObjectCache::get() * - * @param array $keys List of cache keys made from makeKey() or makeGlobalKey() + * @param string[] $keys List of cache keys made from makeKey() or makeGlobalKey() * @param mixed|null &$curTTLs Map of (key => TTL left) for existing/tombstoned keys [returned] - * @param array $checkKeys List of check keys to apply to all $keys. May also apply "check" - * keys to specific cache keys only by using cache keys as keys in the $checkKeys array. + * @param string[]|string[][] $checkKeys Map of (integer or cache key => "check" key(s)) * @param mixed|null &$info Map of (key => info) if WANObjectCache::PASS_BY_REF [returned] - * @return array Map of (key => value) for keys that exist and are not tombstoned + * @return mixed[] Map of (key => value) for existing values; order of $keys is preserved */ final public function getMulti( array $keys, @@ -377,14 +404,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt $curTTLs = []; $infoByKey = []; - $vPrefixLen = strlen( self::VALUE_KEY_PREFIX ); - $valueKeys = self::prefixCacheKeys( $keys, self::VALUE_KEY_PREFIX ); + $vPrefixLen = strlen( self::$VALUE_KEY_PREFIX ); + $valueKeys = self::prefixCacheKeys( $keys, self::$VALUE_KEY_PREFIX ); $checkKeysForAll = []; $checkKeysByKey = []; $checkKeysFlat = []; foreach ( $checkKeys as $i => $checkKeyGroup ) { - $prefixed = self::prefixCacheKeys( (array)$checkKeyGroup, self::TIME_KEY_PREFIX ); + $prefixed = self::prefixCacheKeys( (array)$checkKeyGroup, self::$TIME_KEY_PREFIX ); $checkKeysFlat = array_merge( $checkKeysFlat, $prefixed ); // Are these check keys for a specific cache key, or for all keys being fetched? if ( is_int( $i ) ) { @@ -420,9 +447,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt // Get the main cache value for each key and validate them foreach ( $valueKeys as $vKey ) { $key = substr( $vKey, $vPrefixLen ); // unprefix - list( $value, $curTTL, $asOf, $tombAsOf ) = isset( $wrappedValues[$vKey] ) - ? $this->unwrap( $wrappedValues[$vKey], $now ) - : [ false, null, null, null ]; // not found + list( $value, $keyInfo ) = $this->unwrap( $wrappedValues[$vKey] ?? false, $now ); // Force dependent keys to be seen as stale for a while after purging // to reduce race conditions involving stale data getting cached $purgeValues = $purgeValuesForAll; @@ -432,26 +457,27 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt $lastCKPurge = null; // timestamp of the highest check key foreach ( $purgeValues as $purge ) { - $lastCKPurge = max( $purge[self::FLD_TIME], $lastCKPurge ); - $safeTimestamp = $purge[self::FLD_TIME] + $purge[self::FLD_HOLDOFF]; - if ( $value !== false && $safeTimestamp >= $asOf ) { + $lastCKPurge = max( $purge[self::$PURGE_TIME], $lastCKPurge ); + $safeTimestamp = $purge[self::$PURGE_TIME] + $purge[self::$PURGE_HOLDOFF]; + if ( $value !== false && $safeTimestamp >= $keyInfo['asOf'] ) { // How long ago this value was invalidated by *this* check key - $ago = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE ); + $ago = min( $purge[self::$PURGE_TIME] - $now, self::$TINY_NEGATIVE ); // How long ago this value was invalidated by *any* known check key - $curTTL = min( $curTTL, $ago ); + $keyInfo['curTTL'] = min( $keyInfo['curTTL'], $ago ); } } + $keyInfo[ 'lastCKPurge'] = $lastCKPurge; if ( $value !== false ) { $result[$key] = $value; } - if ( $curTTL !== null ) { - $curTTLs[$key] = $curTTL; + if ( $keyInfo['curTTL'] !== null ) { + $curTTLs[$key] = $keyInfo['curTTL']; } $infoByKey[$key] = ( $info === self::PASS_BY_REF ) - ? [ 'asOf' => $asOf, 'tombAsOf' => $tombAsOf, 'lastCKPurge' => $lastCKPurge ] - : $asOf; // b/c + ? $keyInfo + : $keyInfo['asOf']; // b/c } $info = $infoByKey; @@ -461,10 +487,10 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt /** * @since 1.27 - * @param array $timeKeys List of prefixed time check keys - * @param array $wrappedValues + * @param string[] $timeKeys List of prefixed time check keys + * @param mixed[] $wrappedValues * @param float $now - * @return array List of purge value arrays + * @return array[] List of purge value arrays */ private function processCheckKeys( array $timeKeys, array $wrappedValues, $now ) { $purgeValues = []; @@ -475,7 +501,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt if ( $purge === false ) { // Key is not set or malformed; regenerate $newVal = $this->makePurgeValue( $now, self::HOLDOFF_TTL ); - $this->cache->add( $timeKey, $newVal, self::CHECK_KEY_TTL ); + $this->cache->add( $timeKey, $newVal, self::$CHECK_KEY_TTL ); $purge = $this->parsePurgeValue( $newVal ); } $purgeValues[] = $purge; @@ -520,8 +546,9 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @param mixed $value * @param int $ttl Seconds to live. Special values are: * - WANObjectCache::TTL_INDEFINITE: Cache forever (default) + * - WANObjectCache::TTL_UNCACHEABLE: Do not cache (if the key exists, it is not deleted) * @param array $opts Options map: - * - lag: seconds of replica DB lag. Typically, this is either the replica DB lag + * - lag: Seconds of replica DB lag. Typically, this is either the replica DB lag * before the data was read or, if applicable, the replica DB lag before * the snapshot-isolated transaction the data was read from started. * Use false to indicate that replication is not running. @@ -530,37 +557,48 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * the current time the data was read or (if applicable) the time when * the snapshot-isolated transaction the data was read from started. * Default: 0 seconds - * - pending: whether this data is possibly from an uncommitted write transaction. + * - pending: Whether this data is possibly from an uncommitted write transaction. * Generally, other threads should not see values from the future and * they certainly should not see ones that ended up getting rolled back. * Default: false - * - lockTSE: if excessive replication/snapshot lag is detected, then store the value + * - lockTSE: If excessive replication/snapshot lag is detected, then store the value * with this TTL and flag it as stale. This is only useful if the reads for this key * use getWithSetCallback() with "lockTSE" set. Note that if "staleTTL" is set * then it will still add on to this TTL in the excessive lag scenario. * Default: WANObjectCache::TSE_NONE - * - staleTTL: seconds to keep the key around if it is stale. The get()/getMulti() + * - staleTTL: Seconds to keep the key around if it is stale. The get()/getMulti() * methods return such stale values with a $curTTL of 0, and getWithSetCallback() * will call the regeneration callback in such cases, passing in the old value * and its as-of time to the callback. This is useful if adaptiveTTL() is used * on the old value's as-of time when it is verified as still being correct. - * Default: WANObjectCache::STALE_TTL_NONE. - * - creating: optimize for the case where the key does not already exist. + * Default: WANObjectCache::STALE_TTL_NONE + * - creating: Optimize for the case where the key does not already exist. * Default: false + * - version: Integer version number signifiying the format of the value. + * Default: null + * - walltime: How long the value took to generate in seconds. Default: 0.0 * @note Options added in 1.28: staleTTL * @note Options added in 1.33: creating + * @note Options added in 1.34: version, walltime * @return bool Success */ final public function set( $key, $value, $ttl = self::TTL_INDEFINITE, array $opts = [] ) { $now = $this->getCurrentTime(); + $lag = $opts['lag'] ?? 0; + $age = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0; + $pending = $opts['pending'] ?? false; $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE; $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE; - $age = isset( $opts['since'] ) ? max( 0, $now - $opts['since'] ) : 0; $creating = $opts['creating'] ?? false; - $lag = $opts['lag'] ?? 0; + $version = $opts['version'] ?? null; + $walltime = $opts['walltime'] ?? 0.0; + + if ( $ttl < 0 ) { + return true; + } // Do not cache potentially uncommitted data as it might get rolled back - if ( !empty( $opts['pending'] ) ) { + if ( $pending ) { $this->logger->info( 'Rejected set() for {cachekey} due to pending writes.', [ 'cachekey' => $key ] @@ -619,14 +657,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt } // Wrap that value with time/TTL/version metadata - $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $now ); + $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $version, $now, $walltime ); $storeTTL = $ttl + $staleTTL; if ( $creating ) { - $ok = $this->cache->add( self::VALUE_KEY_PREFIX . $key, $wrapped, $storeTTL ); + $ok = $this->cache->add( self::$VALUE_KEY_PREFIX . $key, $wrapped, $storeTTL ); } else { $ok = $this->cache->merge( - self::VALUE_KEY_PREFIX . $key, + self::$VALUE_KEY_PREFIX . $key, function ( $cache, $key, $cWrapped ) use ( $wrapped ) { // A string value means that it is a tombstone; do nothing in that case return ( is_string( $cWrapped ) ) ? false : $wrapped; @@ -690,7 +728,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * * The $ttl parameter can be used when purging values that have not actually changed * recently. For example, a cleanup script to purge cache entries does not really need - * a hold-off period, so it can use HOLDOFF_NONE. Likewise for user-requested purge. + * a hold-off period, so it can use HOLDOFF_TTL_NONE. Likewise for user-requested purge. * Note that $ttl limits the effective range of 'lockTSE' for getWithSetCallback(). * * If called twice on the same key, then the last hold-off TTL takes precedence. For @@ -703,10 +741,10 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt final public function delete( $key, $ttl = self::HOLDOFF_TTL ) { if ( $ttl <= 0 ) { // Publish the purge to all datacenters - $ok = $this->relayDelete( self::VALUE_KEY_PREFIX . $key ); + $ok = $this->relayDelete( self::$VALUE_KEY_PREFIX . $key ); } else { // Publish the purge to all datacenters - $ok = $this->relayPurge( self::VALUE_KEY_PREFIX . $key, $ttl, self::HOLDOFF_NONE ); + $ok = $this->relayPurge( self::$VALUE_KEY_PREFIX . $key, $ttl, self::HOLDOFF_TTL_NONE ); } $kClass = $this->determineKeyClassForStats( $key ); @@ -795,14 +833,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @see WANObjectCache::getCheckKeyTime() * @see WANObjectCache::getWithSetCallback() * - * @param array $keys + * @param string[] $keys * @return float[] Map of (key => UNIX timestamp) * @since 1.31 */ final public function getMultiCheckKeyTime( array $keys ) { $rawKeys = []; foreach ( $keys as $key ) { - $rawKeys[$key] = self::TIME_KEY_PREFIX . $key; + $rawKeys[$key] = self::$TIME_KEY_PREFIX . $key; } $rawValues = $this->cache->getMulti( $rawKeys ); @@ -812,14 +850,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt foreach ( $rawKeys as $key => $rawKey ) { $purge = $this->parsePurgeValue( $rawValues[$rawKey] ); if ( $purge !== false ) { - $time = $purge[self::FLD_TIME]; + $time = $purge[self::$PURGE_TIME]; } else { // Casting assures identical floats for the next getCheckKeyTime() calls $now = (string)$this->getCurrentTime(); $this->cache->add( $rawKey, $this->makePurgeValue( $now, self::HOLDOFF_TTL ), - self::CHECK_KEY_TTL + self::$CHECK_KEY_TTL ); $time = (float)$now; } @@ -861,12 +899,12 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @see WANObjectCache::resetCheckKey() * * @param string $key Cache key - * @param int $holdoff HOLDOFF_TTL or HOLDOFF_NONE constant + * @param int $holdoff HOLDOFF_TTL or HOLDOFF_TTL_NONE constant * @return bool True if the item was purged or not found, false on failure */ final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) { // Publish the purge to all datacenters - $ok = $this->relayPurge( self::TIME_KEY_PREFIX . $key, self::CHECK_KEY_TTL, $holdoff ); + $ok = $this->relayPurge( self::$TIME_KEY_PREFIX . $key, self::$CHECK_KEY_TTL, $holdoff ); $kClass = $this->determineKeyClassForStats( $key ); $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) ); @@ -903,7 +941,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt */ final public function resetCheckKey( $key ) { // Publish the purge to all datacenters - $ok = $this->relayDelete( self::TIME_KEY_PREFIX . $key ); + $ok = $this->relayDelete( self::$TIME_KEY_PREFIX . $key ); $kClass = $this->determineKeyClassForStats( $key ); $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) ); @@ -1215,63 +1253,37 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt final public function getWithSetCallback( $key, $ttl, $callback, array $opts = [] ) { $version = $opts['version'] ?? null; $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE; - - // Try the process cache if enabled and the cache callback is not within a cache callback. - // Process cache use in nested callbacks is not lag-safe with regard to HOLDOFF_TTL since - // the in-memory value is further lagged than the shared one since it uses a blind TTL. - if ( $pcTTL >= 0 && $this->callbackDepth == 0 ) { - $procCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY ); - if ( $procCache->has( $key, $pcTTL ) ) { - return $procCache->get( $key ); + $pCache = ( $pcTTL >= 0 ) + ? $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY ) + : null; + + // Use the process cache if requested as long as no outer cache callback is running. + // Nested callback process cache use is not lag-safe with regard to HOLDOFF_TTL since + // process cached values are more lagged than persistent ones as they are not purged. + if ( $pCache && $this->callbackDepth == 0 ) { + $cached = $pCache->get( $this->getProcessCacheKey( $key, $version ), INF, false ); + if ( $cached !== false ) { + return $cached; } - } else { - $procCache = null; } - if ( $version !== null ) { - $curAsOf = self::PASS_BY_REF; - $curValue = $this->doGetWithSetCallback( - $key, + $res = $this->fetchOrRegenerate( $key, $ttl, $callback, $opts ); + list( $value, $valueVersion, $curAsOf ) = $res; + if ( $valueVersion !== $version ) { + // Current value has a different version; use the variant key for this version. + // Regenerate the variant value if it is not newer than the main value at $key + // so that purges to the main key propagate to the variant value. + list( $value ) = $this->fetchOrRegenerate( + $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ), $ttl, - // Wrap the value in an array with version metadata but hide it from $callback - function ( $oldValue, &$ttl, &$setOpts, $oldAsOf ) use ( $callback, $version ) { - if ( $this->isVersionedValue( $oldValue, $version ) ) { - $oldData = $oldValue[self::VFLD_DATA]; - } else { - // VFLD_DATA is not set if an old, unversioned, key is present - $oldData = false; - $oldAsOf = null; - } - - return [ - self::VFLD_DATA => $callback( $oldData, $ttl, $setOpts, $oldAsOf ), - self::VFLD_VERSION => $version - ]; - }, - $opts, - $curAsOf + $callback, + [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts ); - if ( $this->isVersionedValue( $curValue, $version ) ) { - // Current value has the requested version; use it - $value = $curValue[self::VFLD_DATA]; - } else { - // Current value has a different version; use the variant key for this version. - // Regenerate the variant value if it is not newer than the main value at $key - // so that purges to they key propagate to the variant value. - $value = $this->doGetWithSetCallback( - $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ), - $ttl, - $callback, - [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts - ); - } - } else { - $value = $this->doGetWithSetCallback( $key, $ttl, $callback, $opts ); } // Update the process cache if enabled - if ( $procCache && $value !== false ) { - $procCache->set( $key, $value ); + if ( $pCache && $value !== false ) { + $pCache->set( $this->getProcessCacheKey( $key, $version ), $value ); } return $value; @@ -1286,77 +1298,80 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @param int $ttl * @param callable $callback * @param array $opts Options map for getWithSetCallback() - * @param float|null &$asOf Cache generation timestamp of returned value [returned] - * @return mixed + * @return array Ordered list of the following: + * - Cached or regenerated value + * - Cached or regenerated value version number or null if not versioned + * - Timestamp of the cached value or null if there is no value * @note Callable type hints are not used to avoid class-autoloading */ - protected function doGetWithSetCallback( $key, $ttl, $callback, array $opts, &$asOf = null ) { - $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl ); - $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE; - $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE; - $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE; + private function fetchOrRegenerate( $key, $ttl, $callback, array $opts ) { $checkKeys = $opts['checkKeys'] ?? []; - $busyValue = $opts['busyValue'] ?? null; - $popWindow = $opts['hotTTR'] ?? self::HOT_TTR; - $ageNew = $opts['ageNew'] ?? self::AGE_NEW; + $graceTTL = $opts['graceTTL'] ?? self::GRACE_TTL_NONE; $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE; + $hotTTR = $opts['hotTTR'] ?? self::HOT_TTR; + $lowTTL = $opts['lowTTL'] ?? min( self::LOW_TTL, $ttl ); + $ageNew = $opts['ageNew'] ?? self::AGE_NEW; $touchedCb = $opts['touchedCallback'] ?? null; $initialTime = $this->getCurrentTime(); $kClass = $this->determineKeyClassForStats( $key ); - // Get the current key value and metadata + // Get the current key value and its metadata $curTTL = self::PASS_BY_REF; $curInfo = self::PASS_BY_REF; /** @var array $curInfo */ $curValue = $this->get( $key, $curTTL, $checkKeys, $curInfo ); // Apply any $touchedCb invalidation timestamp to get the "last purge timestamp" list( $curTTL, $LPT ) = $this->resolveCTL( $curValue, $curTTL, $curInfo, $touchedCb ); - // Best possible return value and its corresponding "as of" timestamp - $value = $curValue; - $asOf = $curInfo['asOf']; - - // Determine if a cached value regeneration is needed or desired + // Use the cached value if it exists and is not due for synchronous regeneration if ( - $this->isValid( $value, $asOf, $minAsOf ) && + $this->isValid( $curValue, $curInfo['asOf'], $minAsOf ) && $this->isAliveOrInGracePeriod( $curTTL, $graceTTL ) ) { $preemptiveRefresh = ( $this->worthRefreshExpiring( $curTTL, $lowTTL ) || - $this->worthRefreshPopular( $asOf, $ageNew, $popWindow, $initialTime ) + $this->worthRefreshPopular( $curInfo['asOf'], $ageNew, $hotTTR, $initialTime ) ); - if ( !$preemptiveRefresh ) { $this->stats->increment( "wanobjectcache.$kClass.hit.good" ); - return $value; + return [ $curValue, $curInfo['version'], $curInfo['asOf'] ]; } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) ) { $this->stats->increment( "wanobjectcache.$kClass.hit.refresh" ); - return $value; + return [ $curValue, $curInfo['version'], $curInfo['asOf'] ]; } } + // Determine if there is stale or volatile cached value that is still usable $isKeyTombstoned = ( $curInfo['tombAsOf'] !== null ); if ( $isKeyTombstoned ) { - // Get the interim key value since the key is tombstoned (write-holed) - list( $value, $asOf ) = $this->getInterimValue( $key, $minAsOf ); + // Key is write-holed; use the (volatile) interim key as an alternative + list( $possValue, $possInfo ) = $this->getInterimValue( $key, $minAsOf ); // Update the "last purge time" since the $touchedCb timestamp depends on $value - $LPT = $this->resolveTouched( $value, $LPT, $touchedCb ); + $LPT = $this->resolveTouched( $possValue, $LPT, $touchedCb ); + } else { + $possValue = $curValue; + $possInfo = $curInfo; } - // Reduce mutex and cache set spam while keys are in the tombstone/holdoff period by - // checking if $value was genereated by a recent thread much less than a second ago. + // Avoid overhead from callback runs, regeneration locks, and cache sets during + // hold-off periods for the key by reusing very recently generated cached values if ( - $this->isValid( $value, $asOf, $minAsOf, $LPT ) && - $this->isVolatileValueAgeNegligible( $initialTime - $asOf ) + $this->isValid( $possValue, $possInfo['asOf'], $minAsOf, $LPT ) && + $this->isVolatileValueAgeNegligible( $initialTime - $possInfo['asOf'] ) ) { $this->stats->increment( "wanobjectcache.$kClass.hit.volatile" ); - return $value; + return [ $possValue, $possInfo['version'], $curInfo['asOf'] ]; } - // Decide if only one thread should handle regeneration at a time - $useMutex = + $lockTSE = $opts['lockTSE'] ?? self::TSE_NONE; + $busyValue = $opts['busyValue'] ?? null; + $staleTTL = $opts['staleTTL'] ?? self::STALE_TTL_NONE; + $version = $opts['version'] ?? null; + + // Determine whether one thread per datacenter should handle regeneration at a time + $useRegenerationLock = // Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to // deduce the key hotness because |$curTTL| will always keep increasing until the // tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE @@ -1369,78 +1384,104 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE ) || // Assume a key is hot if there is no value and a busy fallback is given. // This avoids stampedes on eviction or preemptive regeneration taking too long. - ( $busyValue !== null && $value === false ); - - $hasLock = false; - if ( $useMutex ) { - // Attempt to acquire a non-blocking lock specific to the local datacenter - if ( $this->cache->add( self::MUTEX_KEY_PREFIX . $key, 1, self::LOCK_TTL ) ) { - // Lock acquired; this thread will recompute the value and update cache - $hasLock = true; - } elseif ( $this->isValid( $value, $asOf, $minAsOf ) ) { - // Not acquired and stale cache value exists; use the stale value + ( $busyValue !== null && $possValue === false ); + + // If a regeneration lock is required, threads that do not get the lock will try to use + // the stale value, the interim value, or the $busyValue placeholder, in that order. If + // none of those are set then all threads will bypass the lock and regenerate the value. + $hasLock = $useRegenerationLock && $this->claimStampedeLock( $key ); + if ( $useRegenerationLock && !$hasLock ) { + if ( $this->isValid( $possValue, $possInfo['asOf'], $minAsOf ) ) { $this->stats->increment( "wanobjectcache.$kClass.hit.stale" ); - return $value; - } else { - // Lock not acquired and no stale value exists - if ( $busyValue !== null ) { - // Use the busy fallback value if nothing else - $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss'; - $this->stats->increment( "wanobjectcache.$kClass.$miss.busy" ); + return [ $possValue, $possInfo['version'], $curInfo['asOf'] ]; + } elseif ( $busyValue !== null ) { + $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss'; + $this->stats->increment( "wanobjectcache.$kClass.$miss.busy" ); - return is_callable( $busyValue ) ? $busyValue() : $busyValue; - } + return [ + is_callable( $busyValue ) ? $busyValue() : $busyValue, + $version, + $curInfo['asOf'] + ]; } } - if ( !is_callable( $callback ) ) { - throw new InvalidArgumentException( "Invalid cache miss callback provided." ); - } - - $preCallbackTime = $this->getCurrentTime(); - // Generate the new value from the callback... + // Generate the new value given any prior value with a matching version $setOpts = []; + $preCallbackTime = $this->getCurrentTime(); ++$this->callbackDepth; try { - $value = call_user_func_array( $callback, [ $curValue, &$ttl, &$setOpts, $asOf ] ); + $value = $callback( + ( $curInfo['version'] === $version ) ? $curValue : false, + $ttl, + $setOpts, + ( $curInfo['version'] === $version ) ? $curInfo['asOf'] : null + ); } finally { --$this->callbackDepth; } - $valueIsCacheable = ( $value !== false && $ttl >= 0 ); + $postCallbackTime = $this->getCurrentTime(); - if ( $valueIsCacheable ) { - $ago = max( $this->getCurrentTime() - $initialTime, 0.0 ); - $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1e3 * $ago ); + // How long it took to fetch, validate, and generate the value + $elapsed = max( $postCallbackTime - $initialTime, 0.0 ); + // Attempt to save the newly generated value if applicable + if ( + // Callback yielded a cacheable value + ( $value !== false && $ttl >= 0 ) && + // Current thread was not raced out of a regeneration lock or key is tombstoned + ( !$useRegenerationLock || $hasLock || $isKeyTombstoned ) && + // Key does not appear to be undergoing a set() stampede + $this->checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock ) + ) { + // How long it took to generate the value + $walltime = max( $postCallbackTime - $preCallbackTime, 0.0 ); + $this->stats->timing( "wanobjectcache.$kClass.regen_walltime", 1e3 * $walltime ); + // If the key is write-holed then use the (volatile) interim key as an alternative if ( $isKeyTombstoned ) { - if ( $this->checkAndSetCooloff( $key, $kClass, $ago, $lockTSE, $hasLock ) ) { - // Use the interim key value since the key is tombstoned (write-holed) - $tempTTL = max( self::INTERIM_KEY_TTL, (int)$lockTSE ); - $this->setInterimValue( $key, $value, $tempTTL, $this->getCurrentTime() ); - } - } elseif ( !$useMutex || $hasLock ) { - if ( $this->checkAndSetCooloff( $key, $kClass, $ago, $lockTSE, $hasLock ) ) { - $setOpts['creating'] = ( $curValue === false ); - // Save the value unless a lock-winning thread is already expected to do that - $setOpts['lockTSE'] = $lockTSE; - $setOpts['staleTTL'] = $staleTTL; - // Use best known "since" timestamp if not provided - $setOpts += [ 'since' => $preCallbackTime ]; - // Update the cache; this will fail if the key is tombstoned - $this->set( $key, $value, $ttl, $setOpts ); - } + $this->setInterimValue( $key, $value, $lockTSE, $version, $walltime ); + } else { + $finalSetOpts = [ + 'since' => $setOpts['since'] ?? $preCallbackTime, + 'version' => $version, + 'staleTTL' => $staleTTL, + 'lockTSE' => $lockTSE, // informs lag vs performance trade-offs + 'creating' => ( $curValue === false ), // optimization + 'walltime' => $walltime + ] + $setOpts; + $this->set( $key, $value, $ttl, $finalSetOpts ); } } - if ( $hasLock ) { - $this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, (int)$initialTime - 60 ); - } + $this->yieldStampedeLock( $key, $hasLock ); $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss'; $this->stats->increment( "wanobjectcache.$kClass.$miss.compute" ); - return $value; + return [ $value, $version, $curInfo['asOf'] ]; + } + + /** + * @param string $key + * @return bool Success + */ + private function claimStampedeLock( $key ) { + // Note that locking is not bypassed due to I/O errors; this avoids stampedes + return $this->cache->add( self::$MUTEX_KEY_PREFIX . $key, 1, self::$LOCK_TTL ); + } + + /** + * @param string $key + * @param bool $hasLock + */ + private function yieldStampedeLock( $key, $hasLock ) { + if ( $hasLock ) { + // The backend might be a mcrouter proxy set to broadcast DELETE to *all* the local + // datacenter cache servers via OperationSelectorRoute (for increased consistency). + // Since that would be excessive for these locks, use TOUCH to expire the key. + $this->cache->changeTTL( self::$MUTEX_KEY_PREFIX . $key, $this->getCurrentTime() - 60 ); + } } /** @@ -1448,7 +1489,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @return bool Whether the age of a volatile value is negligible */ private function isVolatileValueAgeNegligible( $age ) { - return ( $age < mt_rand( self::RECENT_SET_LOW_MS, self::RECENT_SET_HIGH_MS ) / 1e3 ); + return ( $age < mt_rand( self::$RECENT_SET_LOW_MS, self::$RECENT_SET_HIGH_MS ) / 1e3 ); } /** @@ -1460,6 +1501,8 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @return bool Whether it is OK to proceed with a key set operation */ private function checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock ) { + $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1e3 * $elapsed ); + // If $lockTSE is set, the lock was bypassed because there was no stale/interim value, // and $elapsed indicates that regeration is slow, then there is a risk of set() // stampedes with large blobs. With a typical scale-out infrastructure, CPU and query @@ -1468,13 +1511,13 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt // consistent hashing). if ( $lockTSE < 0 || $hasLock ) { return true; // either not a priori hot or thread has the lock - } elseif ( $elapsed <= self::SET_DELAY_HIGH_MS * 1e3 ) { + } elseif ( $elapsed <= self::$SET_DELAY_HIGH_MS * 1e3 ) { return true; // not enough time for threads to pile up } $this->cache->clearLastError(); if ( - !$this->cache->add( self::COOLOFF_KEY_PREFIX . $key, 1, self::COOLOFF_TTL ) && + !$this->cache->add( self::$COOLOFF_KEY_PREFIX . $key, 1, self::$COOLOFF_TTL ) && // Don't treat failures due to I/O errors as the key being in cooloff $this->cache->getLastError() === BagOStuff::ERR_NONE ) { @@ -1494,18 +1537,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @return array (current time left or null, UNIX timestamp of last purge or null) * @note Callable type hints are not used to avoid class-autoloading */ - protected function resolveCTL( $value, $curTTL, $curInfo, $touchedCallback ) { + private function resolveCTL( $value, $curTTL, $curInfo, $touchedCallback ) { if ( $touchedCallback === null || $value === false ) { return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'] ) ]; } - if ( !is_callable( $touchedCallback ) ) { - throw new InvalidArgumentException( "Invalid expiration callback provided." ); - } - $touched = $touchedCallback( $value ); if ( $touched !== null && $touched >= $curInfo['asOf'] ) { - $curTTL = min( $curTTL, self::TINY_NEGATIVE, $curInfo['asOf'] - $touched ); + $curTTL = min( $curTTL, self::$TINY_NEGATIVE, $curInfo['asOf'] - $touched ); } return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'], $touched ) ]; @@ -1518,53 +1557,49 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @return float|null UNIX timestamp of last purge or null * @note Callable type hints are not used to avoid class-autoloading */ - protected function resolveTouched( $value, $lastPurge, $touchedCallback ) { - if ( $touchedCallback === null || $value === false ) { - return $lastPurge; - } - - if ( !is_callable( $touchedCallback ) ) { - throw new InvalidArgumentException( "Invalid expiration callback provided." ); - } - - return max( $touchedCallback( $value ), $lastPurge ); + private function resolveTouched( $value, $lastPurge, $touchedCallback ) { + return ( $touchedCallback === null || $value === false ) + ? $lastPurge // nothing to derive the "touched timestamp" from + : max( $touchedCallback( $value ), $lastPurge ); } /** * @param string $key * @param float $minAsOf Minimum acceptable "as of" timestamp - * @return array (cached value or false, cached value timestamp or null) + * @return array (cached value or false, cache key metadata map) */ - protected function getInterimValue( $key, $minAsOf ) { - if ( !$this->useInterimHoldOffCaching ) { - return [ false, null ]; // disabled - } + private function getInterimValue( $key, $minAsOf ) { + $now = $this->getCurrentTime(); + + if ( $this->useInterimHoldOffCaching ) { + $wrapped = $this->cache->get( self::$INTERIM_KEY_PREFIX . $key ); - $wrapped = $this->cache->get( self::INTERIM_KEY_PREFIX . $key ); - list( $value ) = $this->unwrap( $wrapped, $this->getCurrentTime() ); - $valueAsOf = $wrapped[self::FLD_TIME] ?? null; - if ( $this->isValid( $value, $valueAsOf, $minAsOf ) ) { - return [ $value, $valueAsOf ]; + list( $value, $keyInfo ) = $this->unwrap( $wrapped, $now ); + if ( $this->isValid( $value, $keyInfo['asOf'], $minAsOf ) ) { + return [ $value, $keyInfo ]; + } } - return [ false, null ]; + return $this->unwrap( false, $now ); } /** * @param string $key * @param mixed $value - * @param int $tempTTL - * @param float $newAsOf + * @param int $ttl + * @param int|null $version Value version number + * @param float $walltime How long it took to generate the value in seconds */ - protected function setInterimValue( $key, $value, $tempTTL, $newAsOf ) { - $wrapped = $this->wrap( $value, $tempTTL, $newAsOf ); + private function setInterimValue( $key, $value, $ttl, $version, $walltime ) { + $ttl = max( self::$INTERIM_KEY_TTL, (int)$ttl ); + $wrapped = $this->wrap( $value, $ttl, $version, $this->getCurrentTime(), $walltime ); $this->cache->merge( - self::INTERIM_KEY_PREFIX . $key, + self::$INTERIM_KEY_PREFIX . $key, function () use ( $wrapped ) { return $wrapped; }, - $tempTTL, + $ttl, 1 ); } @@ -1593,7 +1628,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * // Map of cache keys to entity IDs * $cache->makeMultiKeys( * $this->fileVersionIds(), - * function ( $id, WANObjectCache $cache ) { + * function ( $id ) use ( $cache ) { * return $cache->makeKey( 'file-version', $id ); * } * ), @@ -1632,20 +1667,16 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @param int $ttl Seconds to live for key updates * @param callable $callback Callback the yields entity regeneration callbacks * @param array $opts Options map - * @return array Map of (cache key => value) in the same order as $keyedIds + * @return mixed[] Map of (cache key => value) in the same order as $keyedIds * @since 1.28 */ final public function getMultiWithSetCallback( ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = [] ) { - $valueKeys = array_keys( $keyedIds->getArrayCopy() ); - $checkKeys = $opts['checkKeys'] ?? []; - $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE; - // Load required keys into process cache in one go $this->warmupCache = $this->getRawKeysForWarmup( - $this->getNonProcessCachedKeys( $valueKeys, $opts, $pcTTL ), - $checkKeys + $this->getNonProcessCachedMultiKeys( $keyedIds, $opts ), + $opts['checkKeys'] ?? [] ); $this->warmupKeyMisses = 0; @@ -1687,7 +1718,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * // Map of cache keys to entity IDs * $cache->makeMultiKeys( * $this->fileVersionIds(), - * function ( $id, WANObjectCache $cache ) { + * function ( $id ) use ( $cache ) { * return $cache->makeKey( 'file-version', $id ); * } * ), @@ -1727,22 +1758,19 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @param int $ttl Seconds to live for key updates * @param callable $callback Callback the yields entity regeneration callbacks * @param array $opts Options map - * @return array Map of (cache key => value) in the same order as $keyedIds + * @return mixed[] Map of (cache key => value) in the same order as $keyedIds * @since 1.30 */ final public function getMultiWithUnionSetCallback( ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = [] ) { - $idsByValueKey = $keyedIds->getArrayCopy(); - $valueKeys = array_keys( $idsByValueKey ); $checkKeys = $opts['checkKeys'] ?? []; - $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE; unset( $opts['lockTSE'] ); // incompatible unset( $opts['busyValue'] ); // incompatible // Load required keys into process cache in one go - $keysGet = $this->getNonProcessCachedKeys( $valueKeys, $opts, $pcTTL ); - $this->warmupCache = $this->getRawKeysForWarmup( $keysGet, $checkKeys ); + $keysByIdGet = $this->getNonProcessCachedMultiKeys( $keyedIds, $opts ); + $this->warmupCache = $this->getRawKeysForWarmup( $keysByIdGet, $checkKeys ); $this->warmupKeyMisses = 0; // IDs of entities known to be in need of regeneration @@ -1751,10 +1779,10 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt // Find out which keys are missing/deleted/stale $curTTLs = []; $asOfs = []; - $curByKey = $this->getMulti( $keysGet, $curTTLs, $checkKeys, $asOfs ); - foreach ( $keysGet as $key ) { + $curByKey = $this->getMulti( $keysByIdGet, $curTTLs, $checkKeys, $asOfs ); + foreach ( $keysByIdGet as $id => $key ) { if ( !array_key_exists( $key, $curByKey ) || $curTTLs[$key] < 0 ) { - $idsRegen[] = $idsByValueKey[$key]; + $idsRegen[] = $id; } } @@ -1786,7 +1814,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt // Run the cache-aside logic using warmupCache instead of persistent cache queries $values = []; - foreach ( $idsByValueKey as $key => $id ) { // preserve order + foreach ( $keyedIds as $key => $id ) { // preserve order $values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts ); } @@ -1809,12 +1837,12 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt */ final public function reap( $key, $purgeTimestamp, &$isStale = false ) { $minAsOf = $purgeTimestamp + self::HOLDOFF_TTL; - $wrapped = $this->cache->get( self::VALUE_KEY_PREFIX . $key ); - if ( is_array( $wrapped ) && $wrapped[self::FLD_TIME] < $minAsOf ) { + $wrapped = $this->cache->get( self::$VALUE_KEY_PREFIX . $key ); + if ( is_array( $wrapped ) && $wrapped[self::$FLD_TIME] < $minAsOf ) { $isStale = true; $this->logger->warning( "Reaping stale value key '$key'." ); $ttlReap = self::HOLDOFF_TTL; // avoids races with tombstone creation - $ok = $this->cache->changeTTL( self::VALUE_KEY_PREFIX . $key, $ttlReap ); + $ok = $this->cache->changeTTL( self::$VALUE_KEY_PREFIX . $key, $ttlReap ); if ( !$ok ) { $this->logger->error( "Could not complete reap of key '$key'." ); } @@ -1837,11 +1865,11 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @since 1.28 */ final public function reapCheckKey( $key, $purgeTimestamp, &$isStale = false ) { - $purge = $this->parsePurgeValue( $this->cache->get( self::TIME_KEY_PREFIX . $key ) ); - if ( $purge && $purge[self::FLD_TIME] < $purgeTimestamp ) { + $purge = $this->parsePurgeValue( $this->cache->get( self::$TIME_KEY_PREFIX . $key ) ); + if ( $purge && $purge[self::$PURGE_TIME] < $purgeTimestamp ) { $isStale = true; $this->logger->warning( "Reaping stale check key '$key'." ); - $ok = $this->cache->changeTTL( self::TIME_KEY_PREFIX . $key, self::TTL_SECOND ); + $ok = $this->cache->changeTTL( self::$TIME_KEY_PREFIX . $key, self::TTL_SECOND ); if ( !$ok ) { $this->logger->error( "Could not complete reap of check key '$key'." ); } @@ -1857,38 +1885,153 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt /** * @see BagOStuff::makeKey() * @param string $class Key class - * @param string|null $component [optional] Key component (starting with a key collection name) - * @return string Colon-delimited list of $keyspace followed by escaped components of $args + * @param string ...$components Key components (starting with a key collection name) + * @return string Colon-delimited list of $keyspace followed by escaped components * @since 1.27 */ - public function makeKey( $class, $component = null ) { + public function makeKey( $class, ...$components ) { return $this->cache->makeKey( ...func_get_args() ); } /** * @see BagOStuff::makeGlobalKey() * @param string $class Key class - * @param string|null $component [optional] Key component (starting with a key collection name) - * @return string Colon-delimited list of $keyspace followed by escaped components of $args + * @param string ...$components Key components (starting with a key collection name) + * @return string Colon-delimited list of $keyspace followed by escaped components * @since 1.27 */ - public function makeGlobalKey( $class, $component = null ) { + public function makeGlobalKey( $class, ...$components ) { return $this->cache->makeGlobalKey( ...func_get_args() ); } /** - * @param array $entities List of entity IDs - * @param callable $keyFunc Callback yielding a key from (entity ID, this WANObjectCache) - * @return ArrayIterator Iterator yielding (cache key => entity ID) in $entities order + * Hash a possibly long string into a suitable component for makeKey()/makeGlobalKey() + * + * @param string $component A raw component used in building a cache key + * @return string 64 character HMAC using a stable secret for public collision resistance + * @since 1.34 + */ + public function hash256( $component ) { + return hash_hmac( 'sha256', $component, $this->secret ); + } + + /** + * Get an iterator of (cache key => entity ID) for a list of entity IDs + * + * The callback takes an ID string and returns a key via makeKey()/makeGlobalKey(). + * There should be no network nor filesystem I/O used in the callback. The entity + * ID/key mapping must be 1:1 or an exception will be thrown. If hashing is needed, + * then use the hash256() method. + * + * Example usage for the default keyspace: + * @code + * $keyedIds = $cache->makeMultiKeys( + * $modules, + * function ( $module ) use ( $cache ) { + * return $cache->makeKey( 'module-info', $module ); + * } + * ); + * @endcode + * + * Example usage for mixed default and global keyspace: + * @code + * $keyedIds = $cache->makeMultiKeys( + * $filters, + * function ( $filter ) use ( $cache ) { + * return ( strpos( $filter, 'central:' ) === 0 ) + * ? $cache->makeGlobalKey( 'regex-filter', $filter ) + * : $cache->makeKey( 'regex-filter', $filter ) + * } + * ); + * @endcode + * + * Example usage with hashing: + * @code + * $keyedIds = $cache->makeMultiKeys( + * $urls, + * function ( $url ) use ( $cache ) { + * return $cache->makeKey( 'url-info', $cache->hash256( $url ) ); + * } + * ); + * @endcode + * + * @see WANObjectCache::makeKey() + * @see WANObjectCache::makeGlobalKey() + * @see WANObjectCache::hash256() + * + * @param string[]|int[] $ids List of entity IDs + * @param callable $keyCallback Function returning makeKey()/makeGlobalKey() on the input ID + * @return ArrayIterator Iterator of (cache key => ID); order of $ids is preserved + * @throws UnexpectedValueException * @since 1.28 */ - final public function makeMultiKeys( array $entities, callable $keyFunc ) { - $map = []; - foreach ( $entities as $entity ) { - $map[$keyFunc( $entity, $this )] = $entity; + final public function makeMultiKeys( array $ids, $keyCallback ) { + $idByKey = []; + foreach ( $ids as $id ) { + // Discourage triggering of automatic makeKey() hashing in some backends + if ( strlen( $id ) > 64 ) { + $this->logger->warning( __METHOD__ . ": long ID '$id'; use hash256()" ); + } + $key = $keyCallback( $id, $this ); + // Edge case: ignore key collisions due to duplicate $ids like "42" and 42 + if ( !isset( $idByKey[$key] ) ) { + $idByKey[$key] = $id; + } elseif ( (string)$id !== (string)$idByKey[$key] ) { + throw new UnexpectedValueException( + "Cache key collision; IDs ('$id','{$idByKey[$key]}') map to '$key'" + ); + } + } + + return new ArrayIterator( $idByKey ); + } + + /** + * Get an (ID => value) map from (i) a non-unique list of entity IDs, and (ii) the list + * of corresponding entity values by first appearance of each ID in the entity ID list + * + * For use with getMultiWithSetCallback() and getMultiWithUnionSetCallback(). + * + * *Only* use this method if the entity ID/key mapping is trivially 1:1 without exception. + * Key generation method must utitilize the *full* entity ID in the key (not a hash of it). + * + * Example usage: + * @code + * $poems = $cache->getMultiWithSetCallback( + * $cache->makeMultiKeys( + * $uuids, + * function ( $uuid ) use ( $cache ) { + * return $cache->makeKey( 'poem', $uuid ); + * } + * ), + * $cache::TTL_DAY, + * function ( $uuid ) use ( $url ) { + * return $this->http->run( [ 'method' => 'GET', 'url' => "$url/$uuid" ] ); + * } + * ); + * $poemsByUUID = $cache->multiRemap( $uuids, $poems ); + * @endcode + * + * @see WANObjectCache::makeMultiKeys() + * @see WANObjectCache::getMultiWithSetCallback() + * @see WANObjectCache::getMultiWithUnionSetCallback() + * + * @param string[]|int[] $ids Entity ID list makeMultiKeys() + * @param mixed[] $res Result of getMultiWithSetCallback()/getMultiWithUnionSetCallback() + * @return mixed[] Map of (ID => value); order of $ids is preserved + * @since 1.34 + */ + final public function multiRemap( array $ids, array $res ) { + if ( count( $ids ) !== count( $res ) ) { + // If makeMultiKeys() is called on a list of non-unique IDs, then the resulting + // ArrayIterator will have less entries due to "first appearance" de-duplication + $ids = array_keys( array_flip( $ids ) ); + if ( count( $ids ) !== count( $res ) ) { + throw new UnexpectedValueException( "Multi-key result does not match ID list" ); + } } - return new ArrayIterator( $map ); + return array_combine( $ids, $res ); } /** @@ -2049,7 +2192,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * This must set the key to "PURGED::" * * @param string $key Cache key - * @param int $ttl How long to keep the tombstone [seconds] + * @param int $ttl Seconds to keep the tombstone around * @param int $holdoff HOLDOFF_* constant controlling how long to ignore sets for this key * @return bool Success */ @@ -2059,14 +2202,14 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt // Wildcards select all matching routes, e.g. the WAN cluster on all DCs $ok = $this->cache->set( "/*/{$this->cluster}/{$key}", - $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ), + $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_TTL_NONE ), $ttl ); } else { // This handles the mcrouter and the single-DC case $ok = $this->cache->set( $key, - $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ), + $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_TTL_NONE ), $ttl ); } @@ -2095,10 +2238,11 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt /** * @param string $key - * @param int $ttl + * @param int $ttl Seconds to live * @param callable $callback * @param array $opts * @return bool Success + * @note Callable type hints are not used to avoid class-autoloading */ private function scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) { if ( !$this->asyncHandler ) { @@ -2108,7 +2252,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt $func = $this->asyncHandler; $func( function () use ( $key, $ttl, $callback, $opts ) { $opts['minAsOf'] = INF; // force a refresh - $this->doGetWithSetCallback( $key, $ttl, $callback, $opts ); + $this->fetchOrRegenerate( $key, $ttl, $callback, $opts ); } ); return true; @@ -2127,7 +2271,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @param int $graceTTL Consider using stale values if $curTTL is greater than this * @return bool */ - protected function isAliveOrInGracePeriod( $curTTL, $graceTTL ) { + private function isAliveOrInGracePeriod( $curTTL, $graceTTL ) { if ( $curTTL > 0 ) { return true; } elseif ( $graceTTL <= 0 ) { @@ -2197,17 +2341,18 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt return false; } + $popularHitsPerSec = 1; // Lifecycle is: new, ramp-up refresh chance, full refresh chance. - // Note that the "expected # of refreshes" for the ramp-up time range is half of what it - // would be if P(refresh) was at its full value during that time range. - $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::RAMPUP_TTL / 2, 1 ); + // Note that the "expected # of refreshes" for the ramp-up time range is half + // of what it would be if P(refresh) was at its full value during that time range. + $refreshWindowSec = max( $timeTillRefresh - $ageNew - self::$RAMPUP_TTL / 2, 1 ); // P(refresh) * (# hits in $refreshWindowSec) = (expected # of refreshes) - // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1 + // P(refresh) * ($refreshWindowSec * $popularHitsPerSec) = 1 (by definition) // P(refresh) = 1/($refreshWindowSec * $popularHitsPerSec) - $chance = 1 / ( self::HIT_RATE_HIGH * $refreshWindowSec ); + $chance = 1 / ( $popularHitsPerSec * $refreshWindowSec ); // Ramp up $chance from 0 to its nominal value over RAMPUP_TTL seconds to avoid stampedes - $chance *= ( $timeOld <= self::RAMPUP_TTL ) ? $timeOld / self::RAMPUP_TTL : 1; + $chance *= ( $timeOld <= self::$RAMPUP_TTL ) ? $timeOld / self::$RAMPUP_TTL : 1; return mt_rand( 1, 1e9 ) <= 1e9 * $chance; } @@ -2223,7 +2368,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt */ protected function isValid( $value, $asOf, $minAsOf, $purgeTime = null ) { // Avoid reading any key not generated after the latest delete() or touch - $safeMinAsOf = max( $minAsOf, $purgeTime + self::TINY_POSTIVE ); + $safeMinAsOf = max( $minAsOf, $purgeTime + self::$TINY_POSTIVE ); if ( $value === false ) { return false; @@ -2235,68 +2380,82 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt } /** - * Do not use this method outside WANObjectCache - * * @param mixed $value - * @param int $ttl [0=forever] + * @param int $ttl Seconds to live or zero for "indefinite" + * @param int|null $version Value version number or null if not versioned * @param float $now Unix Current timestamp just before calling set() + * @param float $walltime How long it took to generate the value in seconds * @return array */ - protected function wrap( $value, $ttl, $now ) { - return [ - self::FLD_VERSION => self::VERSION, - self::FLD_VALUE => $value, - self::FLD_TTL => $ttl, - self::FLD_TIME => $now + private function wrap( $value, $ttl, $version, $now, $walltime ) { + // Returns keys in ascending integer order for PHP7 array packing: + // https://nikic.github.io/2014/12/22/PHPs-new-hashtable-implementation.html + $wrapped = [ + self::$FLD_FORMAT_VERSION => self::$VERSION, + self::$FLD_VALUE => $value, + self::$FLD_TTL => $ttl, + self::$FLD_TIME => $now ]; + if ( $version !== null ) { + $wrapped[self::$FLD_VALUE_VERSION] = $version; + } + if ( $walltime >= self::$GENERATION_SLOW_SEC ) { + $wrapped[self::$FLD_GENERATION_TIME] = $walltime; + } + + return $wrapped; } /** - * Do not use this method outside WANObjectCache - * - * The cached value will be false if absent/tombstoned/malformed - * - * @param array|string|bool $wrapped + * @param array|string|bool $wrapped The entry at a cache key * @param float $now Unix Current timestamp (preferrably pre-query) - * @return array (cached value or false, current TTL, value timestamp, tombstone timestamp) + * @return array (value or false if absent/tombstoned/malformed, value metadata map). + * The cache key metadata includes the following metadata: + * - asOf: UNIX timestamp of the value or null if there is no value + * - curTTL: remaining time-to-live (negative if tombstoned) or null if there is no value + * - version: value version number or null if the if there is no value + * - tombAsOf: UNIX timestamp of the tombstone or null if there is no tombstone */ - protected function unwrap( $wrapped, $now ) { - // Check if the value is a tombstone - $purge = $this->parsePurgeValue( $wrapped ); - if ( $purge !== false ) { - // Purged values should always have a negative current $ttl - $curTTL = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE ); - return [ false, $curTTL, null, $purge[self::FLD_TIME] ]; - } - - if ( !is_array( $wrapped ) // not found - || !isset( $wrapped[self::FLD_VERSION] ) // wrong format - || $wrapped[self::FLD_VERSION] !== self::VERSION // wrong version - ) { - return [ false, null, null, null ]; - } - - if ( $wrapped[self::FLD_TTL] > 0 ) { - // Get the approximate time left on the key - $age = $now - $wrapped[self::FLD_TIME]; - $curTTL = max( $wrapped[self::FLD_TTL] - $age, 0.0 ); + private function unwrap( $wrapped, $now ) { + $value = false; + $info = [ 'asOf' => null, 'curTTL' => null, 'version' => null, 'tombAsOf' => null ]; + + if ( is_array( $wrapped ) ) { + // Entry expected to be a cached value; validate it + if ( + ( $wrapped[self::$FLD_FORMAT_VERSION] ?? null ) === self::$VERSION && + $wrapped[self::$FLD_TIME] >= $this->epoch + ) { + if ( $wrapped[self::$FLD_TTL] > 0 ) { + // Get the approximate time left on the key + $age = $now - $wrapped[self::$FLD_TIME]; + $curTTL = max( $wrapped[self::$FLD_TTL] - $age, 0.0 ); + } else { + // Key had no TTL, so the time left is unbounded + $curTTL = INF; + } + $value = $wrapped[self::$FLD_VALUE]; + $info['version'] = $wrapped[self::$FLD_VALUE_VERSION] ?? null; + $info['asOf'] = $wrapped[self::$FLD_TIME]; + $info['curTTL'] = $curTTL; + } } else { - // Key had no TTL, so the time left is unbounded - $curTTL = INF; - } - - if ( $wrapped[self::FLD_TIME] < $this->epoch ) { - // Values this old are ignored - return [ false, null, null, null ]; + // Entry expected to be a tombstone; parse it + $purge = $this->parsePurgeValue( $wrapped ); + if ( $purge !== false ) { + // Tombstoned keys should always have a negative current $ttl + $info['curTTL'] = min( $purge[self::$PURGE_TIME] - $now, self::$TINY_NEGATIVE ); + $info['tombAsOf'] = $purge[self::$PURGE_TIME]; + } } - return [ $wrapped[self::FLD_VALUE], $curTTL, $wrapped[self::FLD_TIME], null ]; + return [ $value, $info ]; } /** - * @param array $keys + * @param string[] $keys * @param string $prefix - * @return string[] + * @return string[] Prefix keys; the order of $keys is preserved */ protected static function prefixCacheKeys( array $keys, $prefix ) { $res = []; @@ -2311,7 +2470,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @param string $key String of the format :[:]... * @return string A collection name to describe this class of key */ - protected function determineKeyClassForStats( $key ) { + private function determineKeyClassForStats( $key ) { $parts = explode( ':', $key, 3 ); return $parts[1] ?? $parts[0]; // sanity @@ -2322,14 +2481,16 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @return array|bool Array containing a UNIX timestamp (float) and holdoff period (integer), * or false if value isn't a valid purge value */ - protected function parsePurgeValue( $value ) { + private function parsePurgeValue( $value ) { if ( !is_string( $value ) ) { return false; } $segments = explode( ':', $value, 3 ); - if ( !isset( $segments[0] ) || !isset( $segments[1] ) - || "{$segments[0]}:" !== self::PURGE_VAL_PREFIX + if ( + !isset( $segments[0] ) || + !isset( $segments[1] ) || + "{$segments[0]}:" !== self::$PURGE_VAL_PREFIX ) { return false; } @@ -2345,8 +2506,8 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt } return [ - self::FLD_TIME => (float)$segments[1], - self::FLD_HOLDOFF => (int)$segments[2], + self::$PURGE_TIME => (float)$segments[1], + self::$PURGE_HOLDOFF => (int)$segments[2], ]; } @@ -2355,62 +2516,58 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt * @param int $holdoff In seconds * @return string Wrapped purge value */ - protected function makePurgeValue( $timestamp, $holdoff ) { - return self::PURGE_VAL_PREFIX . (float)$timestamp . ':' . (int)$holdoff; - } - - /** - * @param mixed $value - * @param int $version - * @return bool - */ - protected function isVersionedValue( $value, $version ) { - return ( - is_array( $value ) && - array_key_exists( self::VFLD_DATA, $value ) && - array_key_exists( self::VFLD_VERSION, $value ) && - $value[self::VFLD_VERSION] === $version - ); + private function makePurgeValue( $timestamp, $holdoff ) { + return self::$PURGE_VAL_PREFIX . (float)$timestamp . ':' . (int)$holdoff; } /** * @param string $group * @return MapCacheLRU */ - protected function getProcessCache( $group ) { + private function getProcessCache( $group ) { if ( !isset( $this->processCaches[$group] ) ) { - list( , $n ) = explode( ':', $group ); - $this->processCaches[$group] = new MapCacheLRU( (int)$n ); + list( , $size ) = explode( ':', $group ); + $this->processCaches[$group] = new MapCacheLRU( (int)$size ); } return $this->processCaches[$group]; } /** - * @param array $keys + * @param string $key + * @param int $version + * @return string + */ + private function getProcessCacheKey( $key, $version ) { + return $key . ' ' . (int)$version; + } + + /** + * @param ArrayIterator $keys * @param array $opts - * @param int $pcTTL - * @return array List of keys + * @return string[] Map of (ID => cache key) */ - private function getNonProcessCachedKeys( array $keys, array $opts, $pcTTL ) { - $keysFound = []; - if ( isset( $opts['pcTTL'] ) && $opts['pcTTL'] > 0 && $this->callbackDepth == 0 ) { - $pcGroup = $opts['pcGroup'] ?? self::PC_PRIMARY; - $procCache = $this->getProcessCache( $pcGroup ); - foreach ( $keys as $key ) { - if ( $procCache->has( $key, $pcTTL ) ) { - $keysFound[] = $key; + private function getNonProcessCachedMultiKeys( ArrayIterator $keys, array $opts ) { + $pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE; + + $keysMissing = []; + if ( $pcTTL > 0 && $this->callbackDepth == 0 ) { + $version = $opts['version'] ?? null; + $pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY ); + foreach ( $keys as $key => $id ) { + if ( !$pCache->has( $this->getProcessCacheKey( $key, $version ), $pcTTL ) ) { + $keysMissing[$id] = $key; } } } - return array_diff( $keys, $keysFound ); + return $keysMissing; } /** - * @param array $keys - * @param array $checkKeys - * @return array Map of (cache key => mixed) + * @param string[] $keys + * @param string[]|string[][] $checkKeys + * @return string[] List of cache keys */ private function getRawKeysForWarmup( array $keys, array $checkKeys ) { if ( !$keys ) { @@ -2420,18 +2577,18 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt $keysWarmUp = []; // Get all the value keys to fetch... foreach ( $keys as $key ) { - $keysWarmUp[] = self::VALUE_KEY_PREFIX . $key; + $keysWarmUp[] = self::$VALUE_KEY_PREFIX . $key; } // Get all the check keys to fetch... foreach ( $checkKeys as $i => $checkKeyOrKeys ) { if ( is_int( $i ) ) { // Single check key that applies to all value keys - $keysWarmUp[] = self::TIME_KEY_PREFIX . $checkKeyOrKeys; + $keysWarmUp[] = self::$TIME_KEY_PREFIX . $checkKeyOrKeys; } else { // List of check keys that apply to value key $i $keysWarmUp = array_merge( $keysWarmUp, - self::prefixCacheKeys( $checkKeyOrKeys, self::TIME_KEY_PREFIX ) + self::prefixCacheKeys( $checkKeyOrKeys, self::$TIME_KEY_PREFIX ) ); } }