includes/libs/objectcache/BagOStuff.php

   1 <?php
   2 /**
   3  * Copyright © 2003-2004 Brion Vibber <brion@pobox.com>
   4  * https://www.mediawiki.org/
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19  * http://www.gnu.org/copyleft/gpl.html
  20  *
  21  * @file
  22  * @ingroup Cache
  23  */
  24
  25 /**
  26  * @defgroup Cache Cache
  27  */
  28
  29 use Psr\Log\LoggerAwareInterface;
  30 use Psr\Log\LoggerInterface;
  31 use Psr\Log\NullLogger;
  32 use Wikimedia\ScopedCallback;
  33
  34 /**
  35  * Class representing a cache/ephemeral data store
  36  *
  37  * This interface is intended to be more or less compatible with the PHP memcached client.
  38  *
  39  * Instances of this class should be created with an intended access scope, such as:
  40  *   - a) A single PHP thread on a server (e.g. stored in a PHP variable)
  41  *   - b) A single application server (e.g. stored in APC or sqlite)
  42  *   - c) All application servers in datacenter (e.g. stored in memcached or mysql)
  43  *   - d) All application servers in all datacenters (e.g. stored via mcrouter or dynomite)
  44  *
  45  * Callers should use the proper factory methods that yield BagOStuff instances. Site admins
  46  * should make sure the configuration for those factory methods matches their access scope.
  47  * BagOStuff subclasses have widely varying levels of support for replication features.
  48  *
  49  * For any given instance, methods like lock(), unlock(), merge(), and set() with WRITE_SYNC
  50  * should semantically operate over its entire access scope; any nodes/threads in that scope
  51  * should serialize appropriately when using them. Likewise, a call to get() with READ_LATEST
  52  * from one node in its access scope should reflect the prior changes of any other node its
  53  * access scope. Any get() should reflect the changes of any prior set() with WRITE_SYNC.
  54  *
  55  * Subclasses should override the default "segmentationSize" field with an appropriate value.
  56  * The value should not be larger than what the storage backend (by default) supports. It also
  57  * should be roughly informed by common performance bottlenecks (e.g. values over a certain size
  58  * having poor scalability). The same goes for the "segmentedValueMaxSize" member, which limits
  59  * the maximum size and chunk count (indirectly) of values.
  60  *
  61  * @ingroup Cache
  62  */
  63 abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInterface {
  64         /** @var LoggerInterface */
  65         protected $logger;
  66
  67         /** @var callable|null */
  68         protected $asyncHandler;
  69         /** @var int[] Map of (ATTR_* class constant => QOS_* class constant) */
  70         protected $attrMap = [];
  71
  72         /** @var bool */
  73         protected $debugMode = false;
  74
  75         /** @var float|null */
  76         private $wallClockOverride;
  77
  78         /** Bitfield constants for get()/getMulti(); these are only advisory */
  79         const READ_LATEST = 1; // if supported, avoid reading stale data due to replication
  80         const READ_VERIFIED = 2; // promise that the caller handles detection of staleness
  81         /** Bitfield constants for set()/merge(); these are only advisory */
  82         const WRITE_SYNC = 4; // if supported, block until the write is fully replicated
  83         const WRITE_CACHE_ONLY = 8; // only change state of the in-memory cache
  84         const WRITE_ALLOW_SEGMENTS = 16; // allow partitioning of the value if it is large
  85         const WRITE_PRUNE_SEGMENTS = 32; // delete all the segments if the value is partitioned
  86         const WRITE_BACKGROUND = 64; // if supported, do not block on completion until the next read
  87
  88         /**
  89          * Parameters include:
  90          *   - logger: Psr\Log\LoggerInterface instance
  91          *   - asyncHandler: Callable to use for scheduling tasks after the web request ends.
  92          *      In CLI mode, it should run the task immediately.
  93          * @param array $params
  94          * @phan-param array{logger?:Psr\Log\LoggerInterface,asyncHandler?:callable} $params
  95          */
  96         public function __construct( array $params = [] ) {
  97                 $this->setLogger( $params['logger'] ?? new NullLogger() );
  98                 $this->asyncHandler = $params['asyncHandler'] ?? null;
  99         }
 100
 101         /**
 102          * @param LoggerInterface $logger
 103          * @return void
 104          */
 105         public function setLogger( LoggerInterface $logger ) {
 106                 $this->logger = $logger;
 107         }
 108
 109         /**
 110          * @param bool $enabled
 111          */
 112         public function setDebug( $enabled ) {
 113                 $this->debugMode = $enabled;
 114         }
 115
 116         /**
 117          * Get an item with the given key, regenerating and setting it if not found
 118          *
 119          * The callback can take $ttl as argument by reference and modify it.
 120          * Nothing is stored nor deleted if the callback returns false.
 121          *
 122          * @param string $key
 123          * @param int $ttl Time-to-live (seconds)
 124          * @param callable $callback Callback that derives the new value
 125          * @param int $flags Bitfield of BagOStuff::READ_* or BagOStuff::WRITE_* constants [optional]
 126          * @return mixed The cached value if found or the result of $callback otherwise
 127          * @since 1.27
 128          */
 129         final public function getWithSetCallback( $key, $ttl, $callback, $flags = 0 ) {
 130                 $value = $this->get( $key, $flags );
 131
 132                 if ( $value === false ) {
 133                         $value = $callback( $ttl );
 134                         if ( $value !== false && $ttl >= 0 ) {
 135                                 $this->set( $key, $value, $ttl, $flags );
 136                         }
 137                 }
 138
 139                 return $value;
 140         }
 141
 142         /**
 143          * Get an item with the given key
 144          *
 145          * If the key includes a deterministic input hash (e.g. the key can only have
 146          * the correct value) or complete staleness checks are handled by the caller
 147          * (e.g. nothing relies on the TTL), then the READ_VERIFIED flag should be set.
 148          * This lets tiered backends know they can safely upgrade a cached value to
 149          * higher tiers using standard TTLs.
 150          *
 151          * @param string $key
 152          * @param int $flags Bitfield of BagOStuff::READ_* constants [optional]
 153          * @return mixed Returns false on failure or if the item does not exist
 154          */
 155         abstract public function get( $key, $flags = 0 );
 156
 157         /**
 158          * Set an item
 159          *
 160          * @param string $key
 161          * @param mixed $value
 162          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 163          * @param int $flags Bitfield of BagOStuff::WRITE_* constants
 164          * @return bool Success
 165          */
 166         abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
 167
 168         /**
 169          * Delete an item
 170          *
 171          * For large values written using WRITE_ALLOW_SEGMENTS, this only deletes the main
 172          * segment list key unless WRITE_PRUNE_SEGMENTS is in the flags. While deleting the segment
 173          * list key has the effect of functionally deleting the key, it leaves unused blobs in cache.
 174          *
 175          * @param string $key
 176          * @return bool True if the item was deleted or not found, false on failure
 177          * @param int $flags Bitfield of BagOStuff::WRITE_* constants
 178          */
 179         abstract public function delete( $key, $flags = 0 );
 180
 181         /**
 182          * Insert an item if it does not already exist
 183          *
 184          * @param string $key
 185          * @param mixed $value
 186          * @param int $exptime
 187          * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33)
 188          * @return bool Success
 189          */
 190         abstract public function add( $key, $value, $exptime = 0, $flags = 0 );
 191
 192         /**
 193          * Merge changes into the existing cache value (possibly creating a new one)
 194          *
 195          * The callback function returns the new value given the current value
 196          * (which will be false if not present), and takes the arguments:
 197          * (this BagOStuff, cache key, current value, TTL).
 198          * The TTL parameter is reference set to $exptime. It can be overriden in the callback.
 199          * Nothing is stored nor deleted if the callback returns false.
 200          *
 201          * @param string $key
 202          * @param callable $callback Callback method to be executed
 203          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 204          * @param int $attempts The amount of times to attempt a merge in case of failure
 205          * @param int $flags Bitfield of BagOStuff::WRITE_* constants
 206          * @return bool Success
 207          * @throws InvalidArgumentException
 208          */
 209         abstract public function merge(
 210                 $key,
 211                 callable $callback,
 212                 $exptime = 0,
 213                 $attempts = 10,
 214                 $flags = 0
 215         );
 216
 217         /**
 218          * Change the expiration on a key if it exists
 219          *
 220          * If an expiry in the past is given then the key will immediately be expired
 221          *
 222          * For large values written using WRITE_ALLOW_SEGMENTS, this only changes the TTL of the
 223          * main segment list key. While lowering the TTL of the segment list key has the effect of
 224          * functionally lowering the TTL of the key, it might leave unused blobs in cache for longer.
 225          * Raising the TTL of such keys is not effective, since the expiration of a single segment
 226          * key effectively expires the entire value.
 227          *
 228          * @param string $key
 229          * @param int $exptime TTL or UNIX timestamp
 230          * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33)
 231          * @return bool Success Returns false on failure or if the item does not exist
 232          * @since 1.28
 233          */
 234         abstract public function changeTTL( $key, $exptime = 0, $flags = 0 );
 235
 236         /**
 237          * Acquire an advisory lock on a key string
 238          *
 239          * Note that if reentry is enabled, duplicate calls ignore $expiry
 240          *
 241          * @param string $key
 242          * @param int $timeout Lock wait timeout; 0 for non-blocking [optional]
 243          * @param int $expiry Lock expiry [optional]; 1 day maximum
 244          * @param string $rclass Allow reentry if set and the current lock used this value
 245          * @return bool Success
 246          */
 247         abstract public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' );
 248
 249         /**
 250          * Release an advisory lock on a key string
 251          *
 252          * @param string $key
 253          * @return bool Success
 254          */
 255         abstract public function unlock( $key );
 256
 257         /**
 258          * Get a lightweight exclusive self-unlocking lock
 259          *
 260          * Note that the same lock cannot be acquired twice.
 261          *
 262          * This is useful for task de-duplication or to avoid obtrusive
 263          * (though non-corrupting) DB errors like INSERT key conflicts
 264          * or deadlocks when using LOCK IN SHARE MODE.
 265          *
 266          * @param string $key
 267          * @param int $timeout Lock wait timeout; 0 for non-blocking [optional]
 268          * @param int $expiry Lock expiry [optional]; 1 day maximum
 269          * @param string $rclass Allow reentry if set and the current lock used this value
 270          * @return ScopedCallback|null Returns null on failure
 271          * @since 1.26
 272          */
 273         final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
 274                 $expiry = min( $expiry ?: INF, self::TTL_DAY );
 275
 276                 if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
 277                         return null;
 278                 }
 279
 280                 $lSince = $this->getCurrentTime(); // lock timestamp
 281
 282                 return new ScopedCallback( function () use ( $key, $lSince, $expiry ) {
 283                         $latency = 0.050; // latency skew (err towards keeping lock present)
 284                         $age = ( $this->getCurrentTime() - $lSince + $latency );
 285                         if ( ( $age + $latency ) >= $expiry ) {
 286                                 $this->logger->warning(
 287                                         "Lock for {key} held too long ({age} sec).",
 288                                         [ 'key' => $key, 'age' => $age ]
 289                                 );
 290                                 return; // expired; it's not "safe" to delete the key
 291                         }
 292                         $this->unlock( $key );
 293                 } );
 294         }
 295
 296         /**
 297          * Delete all objects expiring before a certain date.
 298          * @param string|int $timestamp The reference date in MW or TS_UNIX format
 299          * @param callable|null $progress Optional, a function which will be called
 300          *     regularly during long-running operations with the percentage progress
 301          *     as the first parameter. [optional]
 302          * @param int $limit Maximum number of keys to delete [default: INF]
 303          *
 304          * @return bool Success; false if unimplemented
 305          */
 306         abstract public function deleteObjectsExpiringBefore(
 307                 $timestamp,
 308                 callable $progress = null,
 309                 $limit = INF
 310         );
 311
 312         /**
 313          * Get an associative array containing the item for each of the keys that have items.
 314          * @param string[] $keys List of keys
 315          * @param int $flags Bitfield; supports READ_LATEST [optional]
 316          * @return mixed[] Map of (key => value) for existing keys
 317          */
 318         abstract public function getMulti( array $keys, $flags = 0 );
 319
 320         /**
 321          * Batch insertion/replace
 322          *
 323          * This does not support WRITE_ALLOW_SEGMENTS to avoid excessive read I/O
 324          *
 325          * WRITE_BACKGROUND can be used for bulk insertion where the response is not vital
 326          *
 327          * @param mixed[] $data Map of (key => value)
 328          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 329          * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33)
 330          * @return bool Success
 331          * @since 1.24
 332          */
 333         abstract public function setMulti( array $data, $exptime = 0, $flags = 0 );
 334
 335         /**
 336          * Batch deletion
 337          *
 338          * This does not support WRITE_ALLOW_SEGMENTS to avoid excessive read I/O
 339          *
 340          * WRITE_BACKGROUND can be used for bulk deletion where the response is not vital
 341          *
 342          * @param string[] $keys List of keys
 343          * @param int $flags Bitfield of BagOStuff::WRITE_* constants
 344          * @return bool Success
 345          * @since 1.33
 346          */
 347         abstract public function deleteMulti( array $keys, $flags = 0 );
 348
 349         /**
 350          * Change the expiration of multiple keys that exist
 351          *
 352          * @see BagOStuff::changeTTL()
 353          *
 354          * @param string[] $keys List of keys
 355          * @param int $exptime TTL or UNIX timestamp
 356          * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33)
 357          * @return bool Success
 358          * @since 1.34
 359          */
 360         abstract public function changeTTLMulti( array $keys, $exptime, $flags = 0 );
 361
 362         /**
 363          * Increase stored value of $key by $value while preserving its TTL
 364          * @param string $key Key to increase
 365          * @param int $value Value to add to $key (default: 1) [optional]
 366          * @param int $flags Bit field of class WRITE_* constants [optional]
 367          * @return int|bool New value or false on failure
 368          */
 369         abstract public function incr( $key, $value = 1, $flags = 0 );
 370
 371         /**
 372          * Decrease stored value of $key by $value while preserving its TTL
 373          * @param string $key
 374          * @param int $value Value to subtract from $key (default: 1) [optional]
 375          * @param int $flags Bit field of class WRITE_* constants [optional]
 376          * @return int|bool New value or false on failure
 377          */
 378         abstract public function decr( $key, $value = 1, $flags = 0 );
 379
 380         /**
 381          * Increase the value of the given key (no TTL change) if it exists or create it otherwise
 382          *
 383          * This will create the key with the value $init and TTL $ttl instead if not present.
 384          * Callers should make sure that both ($init - $value) and $ttl are invariants for all
 385          * operations to any given key. The value of $init should be at least that of $value.
 386          *
 387          * @param string $key Key built via makeKey() or makeGlobalKey()
 388          * @param int $exptime Time-to-live (in seconds) or a UNIX timestamp expiration
 389          * @param int $value Amount to increase the key value by [default: 1]
 390          * @param int|null $init Value to initialize the key to if it does not exist [default: $value]
 391          * @param int $flags Bit field of class WRITE_* constants [optional]
 392          * @return int|bool New value or false on failure
 393          * @since 1.24
 394          */
 395         abstract public function incrWithInit( $key, $exptime, $value = 1, $init = null, $flags = 0 );
 396
 397         /**
 398          * Get the "last error" registered; clearLastError() should be called manually
 399          * @return int ERR_* constant for the "last error" registry
 400          * @since 1.23
 401          */
 402         abstract public function getLastError();
 403
 404         /**
 405          * Clear the "last error" registry
 406          * @since 1.23
 407          */
 408         abstract public function clearLastError();
 409
 410         /**
 411          * Let a callback be run to avoid wasting time on special blocking calls
 412          *
 413          * The callbacks may or may not be called ever, in any particular order.
 414          * They are likely to be invoked when something WRITE_SYNC is used used.
 415          * They should follow a caching pattern as shown below, so that any code
 416          * using the work will get it's result no matter what happens.
 417          * @code
 418          *     $result = null;
 419          *     $workCallback = function () use ( &$result ) {
 420          *         if ( !$result ) {
 421          *             $result = ....
 422          *         }
 423          *         return $result;
 424          *     }
 425          * @endcode
 426          *
 427          * @param callable $workCallback
 428          * @since 1.28
 429          */
 430         abstract public function addBusyCallback( callable $workCallback );
 431
 432         /**
 433          * Construct a cache key.
 434          *
 435          * @since 1.27
 436          * @param string $keyspace
 437          * @param array $args
 438          * @return string Colon-delimited list of $keyspace followed by escaped components of $args
 439          */
 440         abstract public function makeKeyInternal( $keyspace, $args );
 441
 442         /**
 443          * Make a global cache key.
 444          *
 445          * @since 1.27
 446          * @param string $class Key class
 447          * @param string ...$components Key components (starting with a key collection name)
 448          * @return string Colon-delimited list of $keyspace followed by escaped components
 449          */
 450         abstract public function makeGlobalKey( $class, ...$components );
 451
 452         /**
 453          * Make a cache key, scoped to this instance's keyspace.
 454          *
 455          * @since 1.27
 456          * @param string $class Key class
 457          * @param string ...$components Key components (starting with a key collection name)
 458          * @return string Colon-delimited list of $keyspace followed by escaped components
 459          */
 460         abstract public function makeKey( $class, ...$components );
 461
 462         /**
 463          * @param int $flag ATTR_* class constant
 464          * @return int QOS_* class constant
 465          * @since 1.28
 466          */
 467         public function getQoS( $flag ) {
 468                 return $this->attrMap[$flag] ?? self::QOS_UNKNOWN;
 469         }
 470
 471         /**
 472          * @return int|float The chunk size, in bytes, of segmented objects (INF for no limit)
 473          * @since 1.34
 474          */
 475         public function getSegmentationSize() {
 476                 return INF;
 477         }
 478
 479         /**
 480          * @return int|float Maximum total segmented object size in bytes (INF for no limit)
 481          * @since 1.34
 482          */
 483         public function getSegmentedValueMaxSize() {
 484                 return INF;
 485         }
 486
 487         /**
 488          * @param int $field
 489          * @param int $flags
 490          * @return bool
 491          * @since 1.34
 492          */
 493         final protected function fieldHasFlags( $field, $flags ) {
 494                 return ( ( $field & $flags ) === $flags );
 495         }
 496
 497         /**
 498          * Merge the flag maps of one or more BagOStuff objects into a "lowest common denominator" map
 499          *
 500          * @param BagOStuff[] $bags
 501          * @return int[] Resulting flag map (class ATTR_* constant => class QOS_* constant)
 502          */
 503         final protected function mergeFlagMaps( array $bags ) {
 504                 $map = [];
 505                 foreach ( $bags as $bag ) {
 506                         foreach ( $bag->attrMap as $attr => $rank ) {
 507                                 if ( isset( $map[$attr] ) ) {
 508                                         $map[$attr] = min( $map[$attr], $rank );
 509                                 } else {
 510                                         $map[$attr] = $rank;
 511                                 }
 512                         }
 513                 }
 514
 515                 return $map;
 516         }
 517
 518         /**
 519          * @internal For testing only
 520          * @return float UNIX timestamp
 521          * @codeCoverageIgnore
 522          */
 523         public function getCurrentTime() {
 524                 return $this->wallClockOverride ?: microtime( true );
 525         }
 526
 527         /**
 528          * @internal For testing only
 529          * @param float|null &$time Mock UNIX timestamp
 530          * @codeCoverageIgnore
 531          */
 532         public function setMockTime( &$time ) {
 533                 $this->wallClockOverride =& $time;
 534         }
 535 }