X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2Flibs%2Fobjectcache%2FMediumSpecificBagOStuff.php;h=9d361876ce388f2ac8ad5e753b8ae581e2302dea;hb=4334e1cc02b9749e92c514bd2aa46b347010f913;hp=23cf607c7c9464b6a7c9d61044c04173b8eda3cc;hpb=c19e0a2ee969213c96a4153904ed4ff0cdec4e17;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/libs/objectcache/MediumSpecificBagOStuff.php b/includes/libs/objectcache/MediumSpecificBagOStuff.php index 23cf607c7c..9d361876ce 100644 --- a/includes/libs/objectcache/MediumSpecificBagOStuff.php +++ b/includes/libs/objectcache/MediumSpecificBagOStuff.php @@ -160,57 +160,9 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @return bool Success */ public function set( $key, $value, $exptime = 0, $flags = 0 ) { - if ( - is_int( $value ) || // avoid breaking incr()/decr() - ( $flags & self::WRITE_ALLOW_SEGMENTS ) != self::WRITE_ALLOW_SEGMENTS || - is_infinite( $this->segmentationSize ) - ) { - return $this->doSet( $key, $value, $exptime, $flags ); - } - - $serialized = $this->serialize( $value ); - $segmentSize = $this->getSegmentationSize(); - $maxTotalSize = $this->getSegmentedValueMaxSize(); - - $size = strlen( $serialized ); - if ( $size <= $segmentSize ) { - // Since the work of serializing it was already done, just use it inline - return $this->doSet( - $key, - SerializedValueContainer::newUnified( $serialized ), - $exptime, - $flags - ); - } elseif ( $size > $maxTotalSize ) { - $this->setLastError( "Key $key exceeded $maxTotalSize bytes." ); - - return false; - } - - $chunksByKey = []; - $segmentHashes = []; - $count = intdiv( $size, $segmentSize ) + ( ( $size % $segmentSize ) ? 1 : 0 ); - for ( $i = 0; $i < $count; ++$i ) { - $segment = substr( $serialized, $i * $segmentSize, $segmentSize ); - $hash = sha1( $segment ); - $chunkKey = $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $hash ); - $chunksByKey[$chunkKey] = $segment; - $segmentHashes[] = $hash; - } - - $flags &= ~self::WRITE_ALLOW_SEGMENTS; // sanity - $ok = $this->setMulti( $chunksByKey, $exptime, $flags ); - if ( $ok ) { - // Only when all segments are stored should the main key be changed - $ok = $this->doSet( - $key, - SerializedValueContainer::newSegmented( $segmentHashes ), - $exptime, - $flags - ); - } - - return $ok; + list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags ); + // Only when all segments (if any) are stored should the main key be changed + return $usable ? $this->doSet( $key, $entry, $exptime, $flags ) : false; } /** @@ -236,7 +188,7 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @return bool True if the item was deleted or not found, false on failure */ public function delete( $key, $flags = 0 ) { - if ( ( $flags & self::WRITE_PRUNE_SEGMENTS ) != self::WRITE_PRUNE_SEGMENTS ) { + if ( !$this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) { return $this->doDelete( $key, $flags ); } @@ -256,7 +208,7 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { $mainValue->{SerializedValueContainer::SEGMENTED_HASHES} ); - return $this->deleteMulti( $orderedKeys, $flags ); + return $this->deleteMulti( $orderedKeys, $flags & ~self::WRITE_PRUNE_SEGMENTS ); } /** @@ -268,6 +220,23 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { */ abstract protected function doDelete( $key, $flags = 0 ); + public function add( $key, $value, $exptime = 0, $flags = 0 ) { + list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags ); + // Only when all segments (if any) are stored should the main key be changed + return $usable ? $this->doAdd( $key, $entry, $exptime, $flags ) : false; + } + + /** + * Insert an item if it does not already exist + * + * @param string $key + * @param mixed $value + * @param int $exptime + * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33) + * @return bool Success + */ + abstract protected function doAdd( $key, $value, $exptime = 0, $flags = 0 ); + /** * Merge changes into the existing cache value (possibly creating a new one) * @@ -283,7 +252,6 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @param int $attempts The amount of times to attempt a merge in case of failure * @param int $flags Bitfield of BagOStuff::WRITE_* constants * @return bool Success - * @throws InvalidArgumentException */ public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) { return $this->mergeViaCas( $key, $callback, $exptime, $attempts, $flags ); @@ -297,51 +265,56 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @param int $flags Bitfield of BagOStuff::WRITE_* constants * @return bool Success * @see BagOStuff::merge() - * */ final protected function mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags ) { + $attemptsLeft = $attempts; do { - $casToken = null; // passed by reference + $token = null; // passed by reference // Get the old value and CAS token from cache $this->clearLastError(); $currentValue = $this->resolveSegments( $key, - $this->doGet( $key, self::READ_LATEST, $casToken ) + $this->doGet( $key, $flags, $token ) ); if ( $this->getLastError() ) { + // Don't spam slow retries due to network problems (retry only on races) $this->logger->warning( - __METHOD__ . ' failed due to I/O error on get() for {key}.', + __METHOD__ . ' failed due to read I/O error on get() for {key}.', [ 'key' => $key ] ); - - return false; // don't spam retries (retry only on races) + $success = false; + break; } // Derive the new value from the old value $value = call_user_func( $callback, $this, $key, $currentValue, $exptime ); - $hadNoCurrentValue = ( $currentValue === false ); + $keyWasNonexistant = ( $currentValue === false ); + $valueMatchesOldValue = ( $value === $currentValue ); unset( $currentValue ); // free RAM in case the value is large $this->clearLastError(); - if ( $value === false ) { + if ( $value === false || $exptime < 0 ) { $success = true; // do nothing - } elseif ( $hadNoCurrentValue ) { + } elseif ( $valueMatchesOldValue && $attemptsLeft !== $attempts ) { + $success = true; // recently set by another thread to the same value + } elseif ( $keyWasNonexistant ) { // Try to create the key, failing if it gets created in the meantime $success = $this->add( $key, $value, $exptime, $flags ); } else { // Try to update the key, failing if it gets changed in the meantime - $success = $this->cas( $casToken, $key, $value, $exptime, $flags ); + $success = $this->cas( $token, $key, $value, $exptime, $flags ); } if ( $this->getLastError() ) { + // Don't spam slow retries due to network problems (retry only on races) $this->logger->warning( - __METHOD__ . ' failed due to I/O error for {key}.', + __METHOD__ . ' failed due to write I/O error for {key}.', [ 'key' => $key ] ); - - return false; // IO error; don't spam retries + $success = false; + break; } - } while ( !$success && --$attempts ); + } while ( !$success && --$attemptsLeft ); return $success; } @@ -357,21 +330,58 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @return bool Success */ protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) { + if ( $casToken === null ) { + $this->logger->warning( + __METHOD__ . ' got empty CAS token for {key}.', + [ 'key' => $key ] + ); + + return false; // caller may have meant to use add()? + } + + list( $entry, $usable ) = $this->makeValueOrSegmentList( $key, $value, $exptime, $flags ); + // Only when all segments (if any) are stored should the main key be changed + return $usable ? $this->doCas( $casToken, $key, $entry, $exptime, $flags ) : false; + } + + /** + * Check and set an item + * + * @param mixed $casToken + * @param string $key + * @param mixed $value + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + */ + protected function doCas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) { + // @TODO: the lock() call assumes that all other relavent sets() use one if ( !$this->lock( $key, 0 ) ) { return false; // non-blocking } $curCasToken = null; // passed by reference + $this->clearLastError(); $this->doGet( $key, self::READ_LATEST, $curCasToken ); - if ( $casToken === $curCasToken ) { - $success = $this->set( $key, $value, $exptime, $flags ); + if ( is_object( $curCasToken ) ) { + // Using === does not work with objects since it checks for instance identity + throw new UnexpectedValueException( "CAS token cannot be an object" ); + } + if ( $this->getLastError() ) { + // Fail if the old CAS token could not be read + $success = false; + $this->logger->warning( + __METHOD__ . ' failed due to write I/O error for {key}.', + [ 'key' => $key ] + ); + } elseif ( $casToken === $curCasToken ) { + $success = $this->doSet( $key, $value, $exptime, $flags ); } else { + $success = false; // mismatched or failed $this->logger->info( __METHOD__ . ' failed due to race condition for {key}.', [ 'key' => $key ] ); - - $success = false; // mismatched or failed } $this->unlock( $key ); @@ -407,12 +417,13 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @return bool */ protected function doChangeTTL( $key, $exptime, $flags ) { - $expiry = $this->convertToExpiry( $exptime ); - $delete = ( $expiry != 0 && $expiry < $this->getCurrentTime() ); - if ( !$this->lock( $key, 0 ) ) { return false; } + + $expiry = $this->getExpirationAsTimestamp( $exptime ); + $delete = ( $expiry != self::TTL_INDEFINITE && $expiry < $this->getCurrentTime() ); + // Use doGet() to avoid having to trigger resolveSegments() $blob = $this->doGet( $key, self::READ_LATEST ); if ( $blob ) { @@ -587,9 +598,10 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @since 1.24 */ public function setMulti( array $data, $exptime = 0, $flags = 0 ) { - if ( ( $flags & self::WRITE_ALLOW_SEGMENTS ) === self::WRITE_ALLOW_SEGMENTS ) { + if ( $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) ) { throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' ); } + return $this->doSetMulti( $data, $exptime, $flags ); } @@ -604,6 +616,7 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { foreach ( $data as $key => $value ) { $res = $this->doSet( $key, $value, $exptime, $flags ) && $res; } + return $res; } @@ -618,9 +631,10 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * @since 1.33 */ public function deleteMulti( array $keys, $flags = 0 ) { - if ( ( $flags & self::WRITE_ALLOW_SEGMENTS ) === self::WRITE_ALLOW_SEGMENTS ) { - throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' ); + if ( $this->fieldHasFlags( $flags, self::WRITE_PRUNE_SEGMENTS ) ) { + throw new InvalidArgumentException( __METHOD__ . ' got WRITE_PRUNE_SEGMENTS' ); } + return $this->doDeleteMulti( $keys, $flags ); } @@ -657,37 +671,16 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { return $res; } - /** - * Decrease stored value of $key by $value while preserving its TTL - * @param string $key - * @param int $value Value to subtract from $key (default: 1) [optional] - * @return int|bool New value or false on failure - */ - public function decr( $key, $value = 1 ) { - return $this->incr( $key, -$value ); - } - - /** - * Increase stored value of $key by $value while preserving its TTL - * - * This will create the key with value $init and TTL $ttl instead if not present - * - * @param string $key - * @param int $ttl - * @param int $value - * @param int $init - * @return int|bool New value or false on failure - * @since 1.24 - */ - public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) { + public function incrWithInit( $key, $exptime, $value = 1, $init = null, $flags = 0 ) { + $init = is_int( $init ) ? $init : $value; $this->clearLastError(); - $newValue = $this->incr( $key, $value ); + $newValue = $this->incr( $key, $value, $flags ); if ( $newValue === false && !$this->getLastError() ) { // No key set; initialize - $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false; + $newValue = $this->add( $key, (int)$init, $exptime, $flags ) ? $init : false; if ( $newValue === false && !$this->getLastError() ) { // Raced out initializing; increment - $newValue = $this->incr( $key, $value ); + $newValue = $this->incr( $key, $value, $flags ); } } @@ -757,36 +750,70 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { $this->lastError = $err; } - /** - * Let a callback be run to avoid wasting time on special blocking calls - * - * The callbacks may or may not be called ever, in any particular order. - * They are likely to be invoked when something WRITE_SYNC is used used. - * They should follow a caching pattern as shown below, so that any code - * using the work will get it's result no matter what happens. - * @code - * $result = null; - * $workCallback = function () use ( &$result ) { - * if ( !$result ) { - * $result = .... - * } - * return $result; - * } - * @endcode - * - * @param callable $workCallback - * @since 1.28 - */ final public function addBusyCallback( callable $workCallback ) { $this->busyCallbacks[] = $workCallback; } /** + * Determine the entry (inline or segment list) to store under a key to save the value + * + * @param string $key + * @param mixed $value * @param int $exptime - * @return bool + * @param int $flags + * @return array (inline value or segment list, whether the entry is usable) + * @since 1.34 + */ + final protected function makeValueOrSegmentList( $key, $value, $exptime, $flags ) { + $entry = $value; + $usable = true; + + if ( + $this->fieldHasFlags( $flags, self::WRITE_ALLOW_SEGMENTS ) && + !is_int( $value ) && // avoid breaking incr()/decr() + is_finite( $this->segmentationSize ) + ) { + $segmentSize = $this->segmentationSize; + $maxTotalSize = $this->segmentedValueMaxSize; + + $serialized = $this->serialize( $value ); + $size = strlen( $serialized ); + if ( $size > $maxTotalSize ) { + $this->logger->warning( + "Value for {key} exceeds $maxTotalSize bytes; cannot segment.", + [ 'key' => $key ] + ); + } elseif ( $size <= $segmentSize ) { + // The serialized value was already computed, so just use it inline + $entry = SerializedValueContainer::newUnified( $serialized ); + } else { + // Split the serialized value into chunks and store them at different keys + $chunksByKey = []; + $segmentHashes = []; + $count = intdiv( $size, $segmentSize ) + ( ( $size % $segmentSize ) ? 1 : 0 ); + for ( $i = 0; $i < $count; ++$i ) { + $segment = substr( $serialized, $i * $segmentSize, $segmentSize ); + $hash = sha1( $segment ); + $chunkKey = $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $hash ); + $chunksByKey[$chunkKey] = $segment; + $segmentHashes[] = $hash; + } + $flags &= ~self::WRITE_ALLOW_SEGMENTS; // sanity + $usable = $this->setMulti( $chunksByKey, $exptime, $flags ); + $entry = SerializedValueContainer::newSegmented( $segmentHashes ); + } + } + + return [ $entry, $usable ]; + } + + /** + * @param int|float $exptime + * @return bool Whether the expiry is non-infinite, and, negative or not a UNIX timestamp + * @since 1.34 */ - final protected function expiryIsRelative( $exptime ) { - return ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) ); + final protected function isRelativeExpiration( $exptime ) { + return ( $exptime !== self::TTL_INDEFINITE && $exptime < ( 10 * self::TTL_YEAR ) ); } /** @@ -799,11 +826,16 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * - positive (>= 10 years): absolute UNIX timestamp; return this value * * @param int $exptime - * @return int Absolute TTL or 0 for indefinite + * @return int Expiration timestamp or TTL_INDEFINITE for indefinite + * @since 1.34 */ - final protected function convertToExpiry( $exptime ) { - return $this->expiryIsRelative( $exptime ) - ? (int)$this->getCurrentTime() + $exptime + final protected function getExpirationAsTimestamp( $exptime ) { + if ( $exptime == self::TTL_INDEFINITE ) { + return $exptime; + } + + return $this->isRelativeExpiration( $exptime ) + ? intval( $this->getCurrentTime() + $exptime ) : $exptime; } @@ -818,12 +850,17 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { * - positive (>= 10 years): absolute UNIX timestamp; return offset to current time * * @param int $exptime - * @return int Relative TTL or 0 for indefinite + * @return int Relative TTL or TTL_INDEFINITE for indefinite + * @since 1.34 */ - final protected function convertToRelative( $exptime ) { - return $this->expiryIsRelative( $exptime ) || !$exptime - ? (int)$exptime - : max( $exptime - (int)$this->getCurrentTime(), 1 ); + final protected function getExpirationAsTTL( $exptime ) { + if ( $exptime == self::TTL_INDEFINITE ) { + return $exptime; + } + + return $this->isRelativeExpiration( $exptime ) + ? $exptime + : (int)max( $exptime - $this->getCurrentTime(), 1 ); } /** @@ -844,14 +881,6 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { return ( $value === (string)$integer ); } - /** - * Construct a cache key. - * - * @param string $keyspace - * @param array $args - * @return string Colon-delimited list of $keyspace followed by escaped components of $args - * @since 1.27 - */ public function makeKeyInternal( $keyspace, $args ) { $key = $keyspace; foreach ( $args as $arg ) { @@ -893,18 +922,10 @@ abstract class MediumSpecificBagOStuff extends BagOStuff { return $this->attrMap[$flag] ?? self::QOS_UNKNOWN; } - /** - * @return int|float The chunk size, in bytes, of segmented objects (INF for no limit) - * @since 1.34 - */ public function getSegmentationSize() { return $this->segmentationSize; } - /** - * @return int|float Maximum total segmented object size in bytes (INF for no limit) - * @since 1.34 - */ public function getSegmentedValueMaxSize() { return $this->segmentedValueMaxSize; }