X-Git-Url: https://git.heureux-cyclage.org/?p=lhc%2Fweb%2Fwiklou.git;a=blobdiff_plain;f=includes%2Flibs%2Fobjectcache%2FWANObjectCache.php;h=73e4a9a263027ff19efe1e893cdbd7a6259cf0d0;hp=1f757a41e9ff4703427bf8c3bf331ae820282626;hb=444073ddbc915d0dd64ed98a88431e3972e1e39f;hpb=fb291fe9d31cba69ae8cebeb5ba04f76f35d7816 diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php index 1f757a41e9..73e4a9a263 100644 --- a/includes/libs/objectcache/WANObjectCache.php +++ b/includes/libs/objectcache/WANObjectCache.php @@ -19,6 +19,7 @@ * @ingroup Cache */ +use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface; use Psr\Log\LoggerAwareInterface; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; @@ -88,6 +89,8 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { protected $purgeRelayer; /** @var LoggerInterface */ protected $logger; + /** @var StatsdDataFactoryInterface */ + protected $stats; /** @var int ERR_* constant for the "last error" registry */ protected $lastRelayError = self::ERR_NONE; @@ -177,6 +180,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * - channels : Map of (action => channel string). Actions include "purge". * - relayers : Map of (action => EventRelayer object). Actions include "purge". * - logger : LoggerInterface object + * - stats : LoggerInterface object */ public function __construct( array $params ) { $this->cache = $params['cache']; @@ -187,6 +191,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { ? $params['relayers']['purge'] : new EventRelayerNull( [] ); $this->setLogger( isset( $params['logger'] ) ? $params['logger'] : new NullLogger() ); + $this->stats = isset( $params['stats'] ) ? $params['stats'] : new NullStatsdDataFactory(); } public function setLogger( LoggerInterface $logger ) { @@ -239,7 +244,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * Consider using getWithSetCallback() instead of get() and set() cycles. * That method has cache slam avoiding features for hot/expensive keys. * - * @param string $key Cache key + * @param string $key Cache key made from makeKey() or makeGlobalKey() * @param mixed &$curTTL Approximate TTL left on the key if present/tombstoned [returned] * @param array $checkKeys List of "check" keys * @param float &$asOf UNIX timestamp of cached value; null on failure [returned] @@ -260,7 +265,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * * @see WANObjectCache::get() * - * @param array $keys List of cache keys + * @param array $keys List of cache keys made from makeKey() or makeGlobalKey() * @param array &$curTTLs Map of (key => approximate TTL left) for existing keys [returned] * @param array $checkKeys List of check keys to apply to all $keys. May also apply "check" * keys to specific cache keys only by using cache keys as keys in the $checkKeys array. @@ -442,7 +447,8 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { // Do not cache potentially uncommitted data as it might get rolled back if ( !empty( $opts['pending'] ) ) { - $this->logger->info( "Rejected set() for $key due to pending writes." ); + $this->logger->info( 'Rejected set() for {cachekey} due to pending writes.', + [ 'cachekey' => $key ] ); return true; // no-op the write for being unsafe } @@ -456,16 +462,19 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { $wrapExtra[self::FLD_FLAGS] = self::FLG_STALE; // mark as stale // Case B: any long-running transaction; ignore this set() } elseif ( $age > self::MAX_READ_LAG ) { - $this->logger->info( "Rejected set() for $key due to snapshot lag." ); + $this->logger->info( 'Rejected set() for {cachekey} due to snapshot lag.', + [ 'cachekey' => $key ] ); return true; // no-op the write for being unsafe // Case C: high replication lag; lower TTL instead of ignoring all set()s } elseif ( $lag === false || $lag > self::MAX_READ_LAG ) { $ttl = $ttl ? min( $ttl, self::TTL_LAGGED ) : self::TTL_LAGGED; - $this->logger->warning( "Lowered set() TTL for $key due to replication lag." ); + $this->logger->warning( 'Lowered set() TTL for {cachekey} due to replication lag.', + [ 'cachekey' => $key ] ); // Case D: medium length request with medium replication lag; ignore this set() } else { - $this->logger->info( "Rejected set() for $key due to high read lag." ); + $this->logger->info( 'Rejected set() for {cachekey} due to high read lag.', + [ 'cachekey' => $key ] ); return true; // no-op the write for being unsafe } @@ -686,8 +695,11 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * having to inspect a "current time left" variable (e.g. $curTTL, $curTTLs), a cache * regeneration will automatically be triggered using the callback. * - * The simplest way to avoid stampedes for hot keys is to use - * the 'lockTSE' option in $opts. If cache purges are needed, also: + * The $ttl argument and "hotTTR" option (in $opts) use time-dependant randomization + * to avoid stampedes. Keys that are slow to regenerate and either heavily used + * or subject to explicit (unpredictable) purges, may need additional mechanisms. + * The simplest way to avoid stampedes for such keys is to use 'lockTSE' (in $opts). + * If explicit purges are needed, also: * - a) Pass $key into $checkKeys * - b) Use touchCheckKey( $key ) instead of delete( $key ) * @@ -796,7 +808,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * @see WANObjectCache::get() * @see WANObjectCache::set() * - * @param string $key Cache key + * @param string $key Cache key made from makeKey() or makeGlobalKey() * @param int $ttl Seconds to live for key updates. Special values are: * - WANObjectCache::TTL_INDEFINITE: Cache forever * - WANObjectCache::TTL_UNCACHEABLE: Do not cache at all @@ -839,11 +851,13 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * This is useful if the source of a key is suspected of having possibly changed * recently, and the caller wants any such changes to be reflected. * Default: WANObjectCache::MIN_TIMESTAMP_NONE. - * - hotTTR: Expected time-till-refresh for keys that average ~1 hit/second. - * This should be greater than "ageNew". Keys with higher hit rates will regenerate - * more often. This is useful when a popular key is changed but the cache purge was - * delayed or lost. Seldom used keys are rarely affected by this setting, unless an - * extremely low "hotTTR" value is passed in. + * - hotTTR: Expected time-till-refresh (TTR) for keys that average ~1 hit/second (1 Hz). + * Keys with a hit rate higher than 1Hz will refresh sooner than this TTR and vise versa. + * Such refreshes won't happen until keys are "ageNew" seconds old. The TTR is useful at + * reducing the impact of missed cache purges, since the effect of a heavily referenced + * key being stale is worse than that of a rarely referenced key. Unlike simply lowering + * $ttl, seldomly used keys are largely unaffected by this option, which makes it possible + * to have a high hit rate for the "long-tail" of less-used keys. * Default: WANObjectCache::HOT_TTR. * - lowTTL: Consider pre-emptive updates when the current TTL (seconds) of the key is less * than this. It becomes more likely over time, becoming certain once the key is expired. @@ -946,6 +960,9 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { $minTime = isset( $opts['minAsOf'] ) ? $opts['minAsOf'] : self::MIN_TIMESTAMP_NONE; $versioned = isset( $opts['version'] ); + // Get a collection name to describe this class of key + $kClass = $this->determineKeyClass( $key ); + // Get the current key value $curTTL = null; $cValue = $this->get( $key, $curTTL, $checkKeys, $asOf ); // current value @@ -959,11 +976,17 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { && !$this->worthRefreshExpiring( $curTTL, $lowTTL ) && !$this->worthRefreshPopular( $asOf, $ageNew, $popWindow, $preCallbackTime ) ) { + $this->stats->increment( "wanobjectcache.$kClass.hit.good" ); + return $value; } // A deleted key with a negative TTL left must be tombstoned $isTombstone = ( $curTTL !== null && $value === false ); + if ( $isTombstone && $lockTSE <= 0 ) { + // Use the INTERIM value for tombstoned keys to reduce regeneration load + $lockTSE = 1; + } // Assume a key is hot if requested soon after invalidation $isHot = ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE ); // Use the mutex if there is no value and a busy fallback is given @@ -981,21 +1004,23 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { // Lock acquired; this thread should update the key $lockAcquired = true; } elseif ( $value !== false && $this->isValid( $value, $versioned, $asOf, $minTime ) ) { + $this->stats->increment( "wanobjectcache.$kClass.hit.stale" ); // If it cannot be acquired; then the stale value can be used return $value; } else { // Use the INTERIM value for tombstoned keys to reduce regeneration load. // For hot keys, either another thread has the lock or the lock failed; // use the INTERIM value from the last thread that regenerated it. - $wrapped = $this->cache->get( self::INTERIM_KEY_PREFIX . $key ); - list( $value ) = $this->unwrap( $wrapped, microtime( true ) ); - if ( $value !== false && $this->isValid( $value, $versioned, $asOf, $minTime ) ) { - $asOf = $wrapped[self::FLD_TIME]; + $value = $this->getInterimValue( $key, $versioned, $minTime, $asOf ); + if ( $value !== false ) { + $this->stats->increment( "wanobjectcache.$kClass.hit.volatile" ); return $value; } // Use the busy fallback value if nothing else if ( $busyValue !== null ) { + $this->stats->increment( "wanobjectcache.$kClass.miss.busy" ); + return is_callable( $busyValue ) ? $busyValue() : $busyValue; } } @@ -1013,24 +1038,19 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { } finally { --$this->callbackDepth; } + $valueIsCacheable = ( $value !== false && $ttl >= 0 ); + // When delete() is called, writes are write-holed by the tombstone, // so use a special INTERIM key to pass the new value around threads. - if ( ( $isTombstone && $lockTSE > 0 ) && $value !== false && $ttl >= 0 ) { + if ( ( $isTombstone && $lockTSE > 0 ) && $valueIsCacheable ) { $tempTTL = max( 1, (int)$lockTSE ); // set() expects seconds $newAsOf = microtime( true ); $wrapped = $this->wrap( $value, $tempTTL, $newAsOf ); // Avoid using set() to avoid pointless mcrouter broadcasting - $this->cache->merge( - self::INTERIM_KEY_PREFIX . $key, - function () use ( $wrapped ) { - return $wrapped; - }, - $tempTTL, - 1 - ); + $this->setInterimValue( $key, $wrapped, $tempTTL ); } - if ( $value !== false && $ttl >= 0 ) { + if ( $valueIsCacheable ) { $setOpts['lockTSE'] = $lockTSE; // Use best known "since" timestamp if not provided $setOpts += [ 'since' => $preCallbackTime ]; @@ -1040,12 +1060,49 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { if ( $lockAcquired ) { // Avoid using delete() to avoid pointless mcrouter broadcasting - $this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, 1 ); + $this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, (int)$preCallbackTime - 60 ); } + $this->stats->increment( "wanobjectcache.$kClass.miss.compute" ); + return $value; } + /** + * @param string $key + * @param bool $versioned + * @param float $minTime + * @param mixed $asOf + * @return mixed + */ + protected function getInterimValue( $key, $versioned, $minTime, &$asOf ) { + $wrapped = $this->cache->get( self::INTERIM_KEY_PREFIX . $key ); + list( $value ) = $this->unwrap( $wrapped, microtime( true ) ); + if ( $value !== false && $this->isValid( $value, $versioned, $asOf, $minTime ) ) { + $asOf = $wrapped[self::FLD_TIME]; + + return $value; + } + + return false; + } + + /** + * @param string $key + * @param array $wrapped + * @param int $tempTTL + */ + protected function setInterimValue( $key, $wrapped, $tempTTL ) { + $this->cache->merge( + self::INTERIM_KEY_PREFIX . $key, + function () use ( $wrapped ) { + return $wrapped; + }, + $tempTTL, + 1 + ); + } + /** * Method to fetch multiple cache keys at once with regeneration * @@ -1083,7 +1140,15 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * $setOpts += Database::getCacheSetOptions( $dbr ); * * // Load the row for this file - * $row = $dbr->selectRow( 'file', '*', [ 'id' => $id ], __METHOD__ ); + * $queryInfo = File::getQueryInfo(); + * $row = $dbr->selectRow( + * $queryInfo['tables'], + * $queryInfo['fields'], + * [ 'id' => $id ], + * __METHOD__, + * [], + * $queryInfo['joins'] + * ); * * return $row ? (array)$row : false; * }, @@ -1169,7 +1234,15 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { * * // Load the rows for these files * $rows = []; - * $res = $dbr->select( 'file', '*', [ 'id' => $ids ], __METHOD__ ); + * $queryInfo = File::getQueryInfo(); + * $res = $dbr->select( + * $queryInfo['tables'], + * $queryInfo['fields'], + * [ 'id' => $ids ], + * __METHOD__, + * [], + * $queryInfo['joins'] + * ); * foreach ( $res as $row ) { * $rows[$row->id] = $row; * $mtime = wfTimestamp( TS_UNIX, $row->timestamp ); @@ -1315,8 +1388,8 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { /** * @see BagOStuff::makeKey() - * @param string $keys,... Key component - * @return string + * @param string $keys,... Key component (starting with a key collection name) + * @return string Colon-delimited list of $keyspace followed by escaped components of $args * @since 1.27 */ public function makeKey() { @@ -1325,8 +1398,8 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { /** * @see BagOStuff::makeGlobalKey() - * @param string $keys,... Key component - * @return string + * @param string $keys,... Key component (starting with a key collection name) + * @return string Colon-delimited list of $keyspace followed by escaped components of $args * @since 1.27 */ public function makeGlobalKey() { @@ -1507,7 +1580,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { } /** - * Check if a key should be regenerated (using random probability) + * Check if a key is nearing expiration and thus due for randomized regeneration * * This returns false if $curTTL >= $lowTTL. Otherwise, the chance * of returning true increases steadily from 0% to 100% as the $curTTL @@ -1657,6 +1730,16 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { return $res; } + /** + * @param string $key String of the format :[:]... + * @return string + */ + protected function determineKeyClass( $key ) { + $parts = explode( ':', $key ); + + return isset( $parts[1] ) ? $parts[1] : $parts[0]; // sanity + } + /** * @param string $value Wrapped value like "PURGED::" * @return array|bool Array containing a UNIX timestamp (float) and holdoff period (integer), @@ -1724,7 +1807,7 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface { return array_diff( $keys, $keysFound ); } - /** + /** * @param array $keys * @param array $checkKeys * @return array Map of (cache key => mixed)