X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2Flibs%2Fobjectcache%2FWANObjectCache.php;h=f416ec54ccf48fcf9dd81596b3abba4adf77a72a;hb=921f9b69ba93807ad589c39f136a555f81ccf85a;hp=45caa783e12dd5166fc808c1d3bc2dcdc791b432;hpb=91a9a2ae4f3c41b302e6512f370da31249b9a5c7;p=lhc%2Fweb%2Fwiklou.git

diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php
index 45caa783e1..f416ec54cc 100644
--- a/includes/libs/objectcache/WANObjectCache.php
+++ b/includes/libs/objectcache/WANObjectCache.php
@@ -134,6 +134,8 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	protected $asyncHandler;
 	/** @var float Unix timestamp of the oldest possible valid values */
 	protected $epoch;
+	/** @var string Stable secret used for hasing long strings into key components */
+	protected $secret;
 
 	/** @var int Callback stack depth for getWithSetCallback() */
 	private $callbackDepth = 0;
@@ -256,6 +258,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 *       is configured to interpret /<region>/<cluster>/ key prefixes as routes. This
 	 *       requires that "region" and "cluster" are both set above. [optional]
 	 *   - epoch: lowest UNIX timestamp a value/tombstone must have to be valid. [optional]
+	 *   - secret: stable secret used for hashing long strings into key components. [optional]
 	 */
 	public function __construct( array $params ) {
 		$this->cache = $params['cache'];
@@ -263,6 +266,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 		$this->cluster = $params['cluster'] ?? 'wan-main';
 		$this->mcrouterAware = !empty( $params['mcrouterAware'] );
 		$this->epoch = $params['epoch'] ?? self::EPOCH_UNIX_ONE_SECOND;
+		$this->secret = $params['secret'] ?? (string)$this->epoch;
 
 		$this->setLogger( $params['logger'] ?? new NullLogger() );
 		$this->stats = $params['stats'] ?? new NullStatsdDataFactory();
@@ -331,7 +335,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 *
 	 * @param string $key Cache key made from makeKey() or makeGlobalKey()
 	 * @param mixed|null &$curTTL Approximate TTL left on the key if present/tombstoned [returned]
-	 * @param array $checkKeys List of "check" keys
+	 * @param string[] $checkKeys The "check" keys used to validate the value
 	 * @param mixed|null &$info Key info if WANObjectCache::PASS_BY_REF [returned]
 	 * @return mixed Value of cache key or false on failure
 	 */
@@ -366,14 +370,17 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 * Othwerwise, $info will transform into a map of (cache key => cached value timestamp).
 	 * Only the cache keys listed in $keys that exists or are tombstoned will have an entry.
 	 *
+	 * $checkKeys holds the "check" keys used to validate values of applicable keys. The integer
+	 * indexes hold "check" keys that apply to all of $keys while the string indexes hold "check"
+	 * keys that only apply to the cache key with that name.
+	 *
 	 * @see WANObjectCache::get()
 	 *
-	 * @param array $keys List of cache keys made from makeKey() or makeGlobalKey()
+	 * @param string[] $keys List of cache keys made from makeKey() or makeGlobalKey()
 	 * @param mixed|null &$curTTLs Map of (key => TTL left) for existing/tombstoned keys [returned]
-	 * @param array $checkKeys List of check keys to apply to all $keys. May also apply "check"
-	 *  keys to specific cache keys only by using cache keys as keys in the $checkKeys array.
+	 * @param string[]|string[][] $checkKeys Map of (integer or cache key => "check" key(s))
 	 * @param mixed|null &$info Map of (key => info) if WANObjectCache::PASS_BY_REF [returned]
-	 * @return array Map of (key => value) for keys that exist and are not tombstoned
+	 * @return mixed[] Map of (key => value) for existing values; order of $keys is preserved
 	 */
 	final public function getMulti(
 		array $keys,
@@ -468,10 +475,10 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 
 	/**
 	 * @since 1.27
-	 * @param array $timeKeys List of prefixed time check keys
-	 * @param array $wrappedValues
+	 * @param string[] $timeKeys List of prefixed time check keys
+	 * @param mixed[] $wrappedValues
 	 * @param float $now
-	 * @return array List of purge value arrays
+	 * @return array[] List of purge value arrays
 	 */
 	private function processCheckKeys( array $timeKeys, array $wrappedValues, $now ) {
 		$purgeValues = [];
@@ -814,7 +821,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 * @see WANObjectCache::getCheckKeyTime()
 	 * @see WANObjectCache::getWithSetCallback()
 	 *
-	 * @param array $keys
+	 * @param string[] $keys
 	 * @return float[] Map of (key => UNIX timestamp)
 	 * @since 1.31
 	 */
@@ -1234,18 +1241,18 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	final public function getWithSetCallback( $key, $ttl, $callback, array $opts = [] ) {
 		$version = $opts['version'] ?? null;
 		$pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
+		$pCache = ( $pcTTL >= 0 )
+			? $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY )
+			: null;
 
 		// Use the process cache if requested as long as no outer cache callback is running.
 		// Nested callback process cache use is not lag-safe with regard to HOLDOFF_TTL since
 		// process cached values are more lagged than persistent ones as they are not purged.
-		if ( $pcTTL >= 0 && $this->callbackDepth == 0 ) {
-			$pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
+		if ( $pCache && $this->callbackDepth == 0 ) {
 			$cached = $pCache->get( $this->getProcessCacheKey( $key, $version ), INF, false );
 			if ( $cached !== false ) {
 				return $cached;
 			}
-		} else {
-			$pCache = null;
 		}
 
 		$res = $this->fetchOrRegenerate( $key, $ttl, $callback, $opts );
@@ -1367,30 +1374,25 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 			// This avoids stampedes on eviction or preemptive regeneration taking too long.
 			( $busyValue !== null && $possValue === false );
 
-		// If a regeneration lock is required, threads that do not get the lock will use any
-		// available stale or volatile value. If there is none, then the cheap/placeholder
-		// value from $busyValue will be used if provided; failing that, all threads will try
-		// to regenerate the value and ignore the lock.
-		if ( $useRegenerationLock ) {
-			$hasLock = $this->cache->add( self::MUTEX_KEY_PREFIX . $key, 1, self::LOCK_TTL );
-			if ( !$hasLock ) {
-				if ( $this->isValid( $possValue, $possInfo['asOf'], $minAsOf ) ) {
-					$this->stats->increment( "wanobjectcache.$kClass.hit.stale" );
-
-					return [ $possValue, $possInfo['version'], $curInfo['asOf'] ];
-				} elseif ( $busyValue !== null ) {
-					$miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
-					$this->stats->increment( "wanobjectcache.$kClass.$miss.busy" );
-
-					return [
-						is_callable( $busyValue ) ? $busyValue() : $busyValue,
-						$version,
-						$curInfo['asOf']
-					];
-				}
+		// If a regeneration lock is required, threads that do not get the lock will try to use
+		// the stale value, the interim value, or the $busyValue placeholder, in that order. If
+		// none of those are set then all threads will bypass the lock and regenerate the value.
+		$hasLock = $useRegenerationLock && $this->claimStampedeLock( $key );
+		if ( $useRegenerationLock && !$hasLock ) {
+			if ( $this->isValid( $possValue, $possInfo['asOf'], $minAsOf ) ) {
+				$this->stats->increment( "wanobjectcache.$kClass.hit.stale" );
+
+				return [ $possValue, $possInfo['version'], $curInfo['asOf'] ];
+			} elseif ( $busyValue !== null ) {
+				$miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
+				$this->stats->increment( "wanobjectcache.$kClass.$miss.busy" );
+
+				return [
+					is_callable( $busyValue ) ? $busyValue() : $busyValue,
+					$version,
+					$curInfo['asOf']
+				];
 			}
-		} else {
-			$hasLock = false;
 		}
 
 		// Generate the new value given any prior value with a matching version
@@ -1423,6 +1425,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 		) {
 			// How long it took to generate the value
 			$walltime = max( $postCallbackTime - $preCallbackTime, 0.0 );
+			$this->stats->timing( "wanobjectcache.$kClass.regen_walltime", 1e3 * $walltime );
 			// If the key is write-holed then use the (volatile) interim key as an alternative
 			if ( $isKeyTombstoned ) {
 				$this->setInterimValue( $key, $value, $lockTSE, $version, $walltime );
@@ -1439,9 +1442,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 			}
 		}
 
-		if ( $hasLock ) {
-			$this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, (int)$initialTime - 60 );
-		}
+		$this->yieldStampedeLock( $key, $hasLock );
 
 		$miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
 		$this->stats->increment( "wanobjectcache.$kClass.$miss.compute" );
@@ -1449,6 +1450,28 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 		return [ $value, $version, $curInfo['asOf'] ];
 	}
 
+	/**
+	 * @param string $key
+	 * @return bool Success
+	 */
+	private function claimStampedeLock( $key ) {
+		// Note that locking is not bypassed due to I/O errors; this avoids stampedes
+		return $this->cache->add( self::MUTEX_KEY_PREFIX . $key, 1, self::LOCK_TTL );
+	}
+
+	/**
+	 * @param string $key
+	 * @param bool $hasLock
+	 */
+	private function yieldStampedeLock( $key, $hasLock ) {
+		if ( $hasLock ) {
+			// The backend might be a mcrouter proxy set to broadcast DELETE to *all* the local
+			// datacenter cache servers via OperationSelectorRoute (for increased consistency).
+			// Since that would be excessive for these locks, use TOUCH to expire the key.
+			$this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, $this->getCurrentTime() - 60 );
+		}
+	}
+
 	/**
 	 * @param float $age Age of volatile/interim key in seconds
 	 * @return bool Whether the age of a volatile value is negligible
@@ -1593,7 +1616,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 *         // Map of cache keys to entity IDs
 	 *         $cache->makeMultiKeys(
 	 *             $this->fileVersionIds(),
-	 *             function ( $id, WANObjectCache $cache ) {
+	 *             function ( $id ) use ( $cache ) {
 	 *                 return $cache->makeKey( 'file-version', $id );
 	 *             }
 	 *         ),
@@ -1632,17 +1655,15 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 * @param int $ttl Seconds to live for key updates
 	 * @param callable $callback Callback the yields entity regeneration callbacks
 	 * @param array $opts Options map
-	 * @return array Map of (cache key => value) in the same order as $keyedIds
+	 * @return mixed[] Map of (cache key => value) in the same order as $keyedIds
 	 * @since 1.28
 	 */
 	final public function getMultiWithSetCallback(
 		ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
 	) {
-		$valueKeys = array_keys( $keyedIds->getArrayCopy() );
-
 		// Load required keys into process cache in one go
 		$this->warmupCache = $this->getRawKeysForWarmup(
-			$this->getNonProcessCachedKeys( $valueKeys, $opts ),
+			$this->getNonProcessCachedMultiKeys( $keyedIds, $opts ),
 			$opts['checkKeys'] ?? []
 		);
 		$this->warmupKeyMisses = 0;
@@ -1685,7 +1706,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 *         // Map of cache keys to entity IDs
 	 *         $cache->makeMultiKeys(
 	 *             $this->fileVersionIds(),
-	 *             function ( $id, WANObjectCache $cache ) {
+	 *             function ( $id ) use ( $cache ) {
 	 *                 return $cache->makeKey( 'file-version', $id );
 	 *             }
 	 *         ),
@@ -1725,21 +1746,19 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	 * @param int $ttl Seconds to live for key updates
 	 * @param callable $callback Callback the yields entity regeneration callbacks
 	 * @param array $opts Options map
-	 * @return array Map of (cache key => value) in the same order as $keyedIds
+	 * @return mixed[] Map of (cache key => value) in the same order as $keyedIds
 	 * @since 1.30
 	 */
 	final public function getMultiWithUnionSetCallback(
 		ArrayIterator $keyedIds, $ttl, callable $callback, array $opts = []
 	) {
-		$idsByValueKey = $keyedIds->getArrayCopy();
-		$valueKeys = array_keys( $idsByValueKey );
 		$checkKeys = $opts['checkKeys'] ?? [];
 		unset( $opts['lockTSE'] ); // incompatible
 		unset( $opts['busyValue'] ); // incompatible
 
 		// Load required keys into process cache in one go
-		$keysGet = $this->getNonProcessCachedKeys( $valueKeys, $opts );
-		$this->warmupCache = $this->getRawKeysForWarmup( $keysGet, $checkKeys );
+		$keysByIdGet = $this->getNonProcessCachedMultiKeys( $keyedIds, $opts );
+		$this->warmupCache = $this->getRawKeysForWarmup( $keysByIdGet, $checkKeys );
 		$this->warmupKeyMisses = 0;
 
 		// IDs of entities known to be in need of regeneration
@@ -1748,10 +1767,10 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 		// Find out which keys are missing/deleted/stale
 		$curTTLs = [];
 		$asOfs = [];
-		$curByKey = $this->getMulti( $keysGet, $curTTLs, $checkKeys, $asOfs );
-		foreach ( $keysGet as $key ) {
+		$curByKey = $this->getMulti( $keysByIdGet, $curTTLs, $checkKeys, $asOfs );
+		foreach ( $keysByIdGet as $id => $key ) {
 			if ( !array_key_exists( $key, $curByKey ) || $curTTLs[$key] < 0 ) {
-				$idsRegen[] = $idsByValueKey[$key];
+				$idsRegen[] = $id;
 			}
 		}
 
@@ -1783,7 +1802,7 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 
 		// Run the cache-aside logic using warmupCache instead of persistent cache queries
 		$values = [];
-		foreach ( $idsByValueKey as $key => $id ) { // preserve order
+		foreach ( $keyedIds as $key => $id ) { // preserve order
 			$values[$key] = $this->getWithSetCallback( $key, $ttl, $func, $opts );
 		}
 
@@ -1874,18 +1893,133 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	}
 
 	/**
-	 * @param array $entities List of entity IDs
-	 * @param callable $keyFunc Callback yielding a key from (entity ID, this WANObjectCache)
-	 * @return ArrayIterator Iterator yielding (cache key => entity ID) in $entities order
+	 * Hash a possibly long string into a suitable component for makeKey()/makeGlobalKey()
+	 *
+	 * @param string $component A raw component used in building a cache key
+	 * @return string 64 character HMAC using a stable secret for public collision resistance
+	 * @since 1.34
+	 */
+	public function hash256( $component ) {
+		return hash_hmac( 'sha256', $component, $this->secret );
+	}
+
+	/**
+	 * Get an iterator of (cache key => entity ID) for a list of entity IDs
+	 *
+	 * The callback takes an ID string and returns a key via makeKey()/makeGlobalKey().
+	 * There should be no network nor filesystem I/O used in the callback. The entity
+	 * ID/key mapping must be 1:1 or an exception will be thrown. If hashing is needed,
+	 * then use the hash256() method.
+	 *
+	 * Example usage for the default keyspace:
+	 * @code
+	 *     $keyedIds = $cache->makeMultiKeys(
+	 *         $modules,
+	 *         function ( $module ) use ( $cache ) {
+	 *             return $cache->makeKey( 'module-info', $module );
+	 *         }
+	 *     );
+	 * @endcode
+	 *
+	 * Example usage for mixed default and global keyspace:
+	 * @code
+	 *     $keyedIds = $cache->makeMultiKeys(
+	 *         $filters,
+	 *         function ( $filter ) use ( $cache ) {
+	 *             return ( strpos( $filter, 'central:' ) === 0 )
+	 *                 ? $cache->makeGlobalKey( 'regex-filter', $filter )
+	 *                 : $cache->makeKey( 'regex-filter', $filter )
+	 *         }
+	 *     );
+	 * @endcode
+	 *
+	 * Example usage with hashing:
+	 * @code
+	 *     $keyedIds = $cache->makeMultiKeys(
+	 *         $urls,
+	 *         function ( $url ) use ( $cache ) {
+	 *             return $cache->makeKey( 'url-info', $cache->hash256( $url ) );
+	 *         }
+	 *     );
+	 * @endcode
+	 *
+	 * @see WANObjectCache::makeKey()
+	 * @see WANObjectCache::makeGlobalKey()
+	 * @see WANObjectCache::hash256()
+	 *
+	 * @param string[]|int[] $ids List of entity IDs
+	 * @param callable $keyCallback Function returning makeKey()/makeGlobalKey() on the input ID
+	 * @return ArrayIterator Iterator of (cache key => ID); order of $ids is preserved
+	 * @throws UnexpectedValueException
 	 * @since 1.28
 	 */
-	final public function makeMultiKeys( array $entities, callable $keyFunc ) {
-		$map = [];
-		foreach ( $entities as $entity ) {
-			$map[$keyFunc( $entity, $this )] = $entity;
+	final public function makeMultiKeys( array $ids, $keyCallback ) {
+		$idByKey = [];
+		foreach ( $ids as $id ) {
+			// Discourage triggering of automatic makeKey() hashing in some backends
+			if ( strlen( $id ) > 64 ) {
+				$this->logger->warning( __METHOD__ . ": long ID '$id'; use hash256()" );
+			}
+			$key = $keyCallback( $id, $this );
+			// Edge case: ignore key collisions due to duplicate $ids like "42" and 42
+			if ( !isset( $idByKey[$key] ) ) {
+				$idByKey[$key] = $id;
+			} elseif ( (string)$id !== (string)$idByKey[$key] ) {
+				throw new UnexpectedValueException(
+					"Cache key collision; IDs ('$id','{$idByKey[$key]}') map to '$key'"
+				);
+			}
+		}
+
+		return new ArrayIterator( $idByKey );
+	}
+
+	/**
+	 * Get an (ID => value) map from (i) a non-unique list of entity IDs, and (ii) the list
+	 * of corresponding entity values by first appearance of each ID in the entity ID list
+	 *
+	 * For use with getMultiWithSetCallback() and getMultiWithUnionSetCallback().
+	 *
+	 * *Only* use this method if the entity ID/key mapping is trivially 1:1 without exception.
+	 * Key generation method must utitilize the *full* entity ID in the key (not a hash of it).
+	 *
+	 * Example usage:
+	 * @code
+	 *     $poems = $cache->getMultiWithSetCallback(
+	 *         $cache->makeMultiKeys(
+	 *             $uuids,
+	 *             function ( $uuid ) use ( $cache ) {
+	 *                 return $cache->makeKey( 'poem', $uuid );
+	 *             }
+	 *         ),
+	 *         $cache::TTL_DAY,
+	 *         function ( $uuid ) use ( $url ) {
+	 *             return $this->http->run( [ 'method' => 'GET', 'url' => "$url/$uuid" ] );
+	 *         }
+	 *     );
+	 *     $poemsByUUID = $cache->multiRemap( $uuids, $poems );
+	 * @endcode
+	 *
+	 * @see WANObjectCache::makeMultiKeys()
+	 * @see WANObjectCache::getMultiWithSetCallback()
+	 * @see WANObjectCache::getMultiWithUnionSetCallback()
+	 *
+	 * @param string[]|int[] $ids Entity ID list makeMultiKeys()
+	 * @param mixed[] $res Result of getMultiWithSetCallback()/getMultiWithUnionSetCallback()
+	 * @return mixed[] Map of (ID => value); order of $ids is preserved
+	 * @since 1.34
+	 */
+	final public function multiRemap( array $ids, array $res ) {
+		if ( count( $ids ) !== count( $res ) ) {
+			// If makeMultiKeys() is called on a list of non-unique IDs, then the resulting
+			// ArrayIterator will have less entries due to "first appearance" de-duplication
+			$ids = array_keys( array_flip( $ids ) );
+			if ( count( $ids ) !== count( $res ) ) {
+				throw new UnexpectedValueException( "Multi-key result does not match ID list" );
+			}
 		}
 
-		return new ArrayIterator( $map );
+		return array_combine( $ids, $res );
 	}
 
 	/**
@@ -2306,9 +2440,9 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	}
 
 	/**
-	 * @param array $keys
+	 * @param string[] $keys
 	 * @param string $prefix
-	 * @return string[]
+	 * @return string[] Prefix keys; the order of $keys is preserved
 	 */
 	protected static function prefixCacheKeys( array $keys, $prefix ) {
 		$res = [];
@@ -2394,31 +2528,31 @@ class WANObjectCache implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInt
 	}
 
 	/**
-	 * @param array $keys
+	 * @param ArrayIterator $keys
 	 * @param array $opts
-	 * @return string[] List of keys
+	 * @return string[] Map of (ID => cache key)
 	 */
-	private function getNonProcessCachedKeys( array $keys, array $opts ) {
+	private function getNonProcessCachedMultiKeys( ArrayIterator $keys, array $opts ) {
 		$pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
 
-		$keysFound = [];
+		$keysMissing = [];
 		if ( $pcTTL > 0 && $this->callbackDepth == 0 ) {
 			$version = $opts['version'] ?? null;
 			$pCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
-			foreach ( $keys as $key ) {
-				if ( $pCache->has( $this->getProcessCacheKey( $key, $version ), $pcTTL ) ) {
-					$keysFound[] = $key;
+			foreach ( $keys as $key => $id ) {
+				if ( !$pCache->has( $this->getProcessCacheKey( $key, $version ), $pcTTL ) ) {
+					$keysMissing[$id] = $key;
 				}
 			}
 		}
 
-		return array_diff( $keys, $keysFound );
+		return $keysMissing;
 	}
 
 	/**
-	 * @param array $keys
-	 * @param array $checkKeys
-	 * @return array Map of (cache key => mixed)
+	 * @param string[] $keys
+	 * @param string[]|string[][] $checkKeys
+	 * @return string[] List of cache keys
 	 */
 	private function getRawKeysForWarmup( array $keys, array $checkKeys ) {
 		if ( !$keys ) {