includes/libs/objectcache/BagOStuff.php

   1 <?php
   2 /**
   3  * Copyright © 2003-2004 Brion Vibber <brion@pobox.com>
   4  * https://www.mediawiki.org/
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19  * http://www.gnu.org/copyleft/gpl.html
  20  *
  21  * @file
  22  * @ingroup Cache
  23  */
  24
  25 /**
  26  * @defgroup Cache Cache
  27  */
  28
  29 use Psr\Log\LoggerAwareInterface;
  30 use Psr\Log\LoggerInterface;
  31 use Psr\Log\NullLogger;
  32
  33 /**
  34  * interface is intended to be more or less compatible with
  35  * the PHP memcached client.
  36  *
  37  * backends for local hash array and SQL table included:
  38  * @code
  39  *   $bag = new HashBagOStuff();
  40  *   $bag = new SqlBagOStuff(); # connect to db first
  41  * @endcode
  42  *
  43  * @ingroup Cache
  44  */
  45 abstract class BagOStuff implements LoggerAwareInterface {
  46         /** @var array[] Lock tracking */
  47         protected $locks = array();
  48
  49         /** @var integer */
  50         protected $lastError = self::ERR_NONE;
  51
  52         /** @var string */
  53         protected $keyspace = 'local';
  54
  55         /** @var LoggerInterface */
  56         protected $logger;
  57
  58         /** @var bool */
  59         private $debugMode = false;
  60
  61         /** Possible values for getLastError() */
  62         const ERR_NONE = 0; // no error
  63         const ERR_NO_RESPONSE = 1; // no response
  64         const ERR_UNREACHABLE = 2; // can't connect
  65         const ERR_UNEXPECTED = 3; // response gave some error
  66
  67         /** Bitfield constants for get()/getMulti() */
  68         const READ_LATEST = 1; // use latest data for replicated stores
  69         const READ_VERIFIED = 2; // promise that caller can tell when keys are stale
  70         /** Bitfield constants for set()/merge() */
  71         const WRITE_SYNC = 1; // synchronously write to all locations for replicated stores
  72
  73         public function __construct( array $params = array() ) {
  74                 if ( isset( $params['logger'] ) ) {
  75                         $this->setLogger( $params['logger'] );
  76                 } else {
  77                         $this->setLogger( new NullLogger() );
  78                 }
  79
  80                 if ( isset( $params['keyspace'] ) ) {
  81                         $this->keyspace = $params['keyspace'];
  82                 }
  83         }
  84
  85         /**
  86          * @param LoggerInterface $logger
  87          * @return null
  88          */
  89         public function setLogger( LoggerInterface $logger ) {
  90                 $this->logger = $logger;
  91         }
  92
  93         /**
  94          * @param bool $bool
  95          */
  96         public function setDebug( $bool ) {
  97                 $this->debugMode = $bool;
  98         }
  99
 100         /**
 101          * Get an item with the given key, regenerating and setting it if not found
 102          *
 103          * If the callback returns false, then nothing is stored.
 104          *
 105          * @param string $key
 106          * @param int $ttl Time-to-live (seconds)
 107          * @param callable $callback Callback that derives the new value
 108          * @param integer $flags Bitfield of BagOStuff::READ_* constants [optional]
 109          * @return mixed The cached value if found or the result of $callback otherwise
 110          * @since 1.27
 111          */
 112         final public function getWithSetCallback( $key, $ttl, $callback, $flags = 0 ) {
 113                 $value = $this->get( $key, $flags );
 114
 115                 if ( $value === false ) {
 116                         if ( !is_callable( $callback ) ) {
 117                                 throw new InvalidArgumentException( "Invalid cache miss callback provided." );
 118                         }
 119                         $value = call_user_func( $callback );
 120                         if ( $value !== false ) {
 121                                 $this->set( $key, $value, $ttl );
 122                         }
 123                 }
 124
 125                 return $value;
 126         }
 127
 128         /**
 129          * Get an item with the given key
 130          *
 131          * If the key includes a determistic input hash (e.g. the key can only have
 132          * the correct value) or complete staleness checks are handled by the caller
 133          * (e.g. nothing relies on the TTL), then the READ_VERIFIED flag should be set.
 134          * This lets tiered backends know they can safely upgrade a cached value to
 135          * higher tiers using standard TTLs.
 136          *
 137          * @param string $key
 138          * @param integer $flags Bitfield of BagOStuff::READ_* constants [optional]
 139          * @param integer $oldFlags [unused]
 140          * @return mixed Returns false on failure and if the item does not exist
 141          */
 142         public function get( $key, $flags = 0, $oldFlags = null ) {
 143                 // B/C for ( $key, &$casToken = null, $flags = 0 )
 144                 $flags = is_int( $oldFlags ) ? $oldFlags : $flags;
 145
 146                 return $this->doGet( $key, $flags );
 147         }
 148
 149         /**
 150          * @param string $key
 151          * @param integer $flags Bitfield of BagOStuff::READ_* constants [optional]
 152          * @return mixed Returns false on failure and if the item does not exist
 153          */
 154         abstract protected function doGet( $key, $flags = 0 );
 155
 156         /**
 157          * @note: This method is only needed if cas() is not is used for merge()
 158          *
 159          * @param string $key
 160          * @param mixed $casToken
 161          * @param integer $flags Bitfield of BagOStuff::READ_* constants [optional]
 162          * @return mixed Returns false on failure and if the item does not exist
 163          * @throws Exception
 164          */
 165         protected function getWithToken( $key, &$casToken, $flags = 0 ) {
 166                 throw new Exception( __METHOD__ . ' not implemented.' );
 167         }
 168
 169         /**
 170          * Set an item
 171          *
 172          * @param string $key
 173          * @param mixed $value
 174          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 175          * @param int $flags Bitfield of BagOStuff::WRITE_* constants
 176          * @return bool Success
 177          */
 178         abstract public function set( $key, $value, $exptime = 0, $flags = 0 );
 179
 180         /**
 181          * Delete an item
 182          *
 183          * @param string $key
 184          * @return bool True if the item was deleted or not found, false on failure
 185          */
 186         abstract public function delete( $key );
 187
 188         /**
 189          * Merge changes into the existing cache value (possibly creating a new one).
 190          * The callback function returns the new value given the current value
 191          * (which will be false if not present), and takes the arguments:
 192          * (this BagOStuff, cache key, current value).
 193          *
 194          * @param string $key
 195          * @param callable $callback Callback method to be executed
 196          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 197          * @param int $attempts The amount of times to attempt a merge in case of failure
 198          * @param int $flags Bitfield of BagOStuff::WRITE_* constants
 199          * @return bool Success
 200          * @throws InvalidArgumentException
 201          */
 202         public function merge( $key, $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
 203                 if ( !is_callable( $callback ) ) {
 204                         throw new InvalidArgumentException( "Got invalid callback." );
 205                 }
 206
 207                 return $this->mergeViaLock( $key, $callback, $exptime, $attempts, $flags );
 208         }
 209
 210         /**
 211          * @see BagOStuff::merge()
 212          *
 213          * @param string $key
 214          * @param callable $callback Callback method to be executed
 215          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 216          * @param int $attempts The amount of times to attempt a merge in case of failure
 217          * @return bool Success
 218          */
 219         protected function mergeViaCas( $key, $callback, $exptime = 0, $attempts = 10 ) {
 220                 do {
 221                         $this->clearLastError();
 222                         $casToken = null; // passed by reference
 223                         $currentValue = $this->getWithToken( $key, $casToken, BagOStuff::READ_LATEST );
 224                         if ( $this->getLastError() ) {
 225                                 return false; // don't spam retries (retry only on races)
 226                         }
 227
 228                         // Derive the new value from the old value
 229                         $value = call_user_func( $callback, $this, $key, $currentValue );
 230
 231                         $this->clearLastError();
 232                         if ( $value === false ) {
 233                                 $success = true; // do nothing
 234                         } elseif ( $currentValue === false ) {
 235                                 // Try to create the key, failing if it gets created in the meantime
 236                                 $success = $this->add( $key, $value, $exptime );
 237                         } else {
 238                                 // Try to update the key, failing if it gets changed in the meantime
 239                                 $success = $this->cas( $casToken, $key, $value, $exptime );
 240                         }
 241                         if ( $this->getLastError() ) {
 242                                 return false; // IO error; don't spam retries
 243                         }
 244                 } while ( !$success && --$attempts );
 245
 246                 return $success;
 247         }
 248
 249         /**
 250          * Check and set an item
 251          *
 252          * @param mixed $casToken
 253          * @param string $key
 254          * @param mixed $value
 255          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 256          * @return bool Success
 257          * @throws Exception
 258          */
 259         protected function cas( $casToken, $key, $value, $exptime = 0 ) {
 260                 throw new Exception( "CAS is not implemented in " . __CLASS__ );
 261         }
 262
 263         /**
 264          * @see BagOStuff::merge()
 265          *
 266          * @param string $key
 267          * @param callable $callback Callback method to be executed
 268          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 269          * @param int $attempts The amount of times to attempt a merge in case of failure
 270          * @param int $flags Bitfield of BagOStuff::WRITE_* constants
 271          * @return bool Success
 272          */
 273         protected function mergeViaLock( $key, $callback, $exptime = 0, $attempts = 10, $flags = 0 ) {
 274                 if ( !$this->lock( $key, 6 ) ) {
 275                         return false;
 276                 }
 277
 278                 $this->clearLastError();
 279                 $currentValue = $this->get( $key, BagOStuff::READ_LATEST );
 280                 if ( $this->getLastError() ) {
 281                         $success = false;
 282                 } else {
 283                         // Derive the new value from the old value
 284                         $value = call_user_func( $callback, $this, $key, $currentValue );
 285                         if ( $value === false ) {
 286                                 $success = true; // do nothing
 287                         } else {
 288                                 $success = $this->set( $key, $value, $exptime, $flags ); // set the new value
 289                         }
 290                 }
 291
 292                 if ( !$this->unlock( $key ) ) {
 293                         // this should never happen
 294                         trigger_error( "Could not release lock for key '$key'." );
 295                 }
 296
 297                 return $success;
 298         }
 299
 300         /**
 301          * Acquire an advisory lock on a key string
 302          *
 303          * Note that if reentry is enabled, duplicate calls ignore $expiry
 304          *
 305          * @param string $key
 306          * @param int $timeout Lock wait timeout; 0 for non-blocking [optional]
 307          * @param int $expiry Lock expiry [optional]; 1 day maximum
 308          * @param string $rclass Allow reentry if set and the current lock used this value
 309          * @return bool Success
 310          */
 311         public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
 312                 // Avoid deadlocks and allow lock reentry if specified
 313                 if ( isset( $this->locks[$key] ) ) {
 314                         if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) {
 315                                 ++$this->locks[$key]['depth'];
 316                                 return true;
 317                         } else {
 318                                 return false;
 319                         }
 320                 }
 321
 322                 $expiry = min( $expiry ?: INF, 86400 );
 323
 324                 $this->clearLastError();
 325                 $timestamp = microtime( true ); // starting UNIX timestamp
 326                 if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
 327                         $locked = true;
 328                 } elseif ( $this->getLastError() || $timeout <= 0 ) {
 329                         $locked = false; // network partition or non-blocking
 330                 } else {
 331                         $uRTT = ceil( 1e6 * ( microtime( true ) - $timestamp ) ); // estimate RTT (us)
 332                         $sleep = 2 * $uRTT; // rough time to do get()+set()
 333
 334                         $attempts = 0; // failed attempts
 335                         do {
 336                                 if ( ++$attempts >= 3 && $sleep <= 5e5 ) {
 337                                         // Exponentially back off after failed attempts to avoid network spam.
 338                                         // About 2*$uRTT*(2^n-1) us of "sleep" happen for the next n attempts.
 339                                         $sleep *= 2;
 340                                 }
 341                                 usleep( $sleep ); // back off
 342                                 $this->clearLastError();
 343                                 $locked = $this->add( "{$key}:lock", 1, $expiry );
 344                                 if ( $this->getLastError() ) {
 345                                         $locked = false; // network partition
 346                                         break;
 347                                 }
 348                         } while ( !$locked && ( microtime( true ) - $timestamp ) < $timeout );
 349                 }
 350
 351                 if ( $locked ) {
 352                         $this->locks[$key] = array( 'class' => $rclass, 'depth' => 1 );
 353                 }
 354
 355                 return $locked;
 356         }
 357
 358         /**
 359          * Release an advisory lock on a key string
 360          *
 361          * @param string $key
 362          * @return bool Success
 363          */
 364         public function unlock( $key ) {
 365                 if ( isset( $this->locks[$key] ) && --$this->locks[$key]['depth'] <= 0 ) {
 366                         unset( $this->locks[$key] );
 367
 368                         return $this->delete( "{$key}:lock" );
 369                 }
 370
 371                 return true;
 372         }
 373
 374         /**
 375          * Get a lightweight exclusive self-unlocking lock
 376          *
 377          * Note that the same lock cannot be acquired twice.
 378          *
 379          * This is useful for task de-duplication or to avoid obtrusive
 380          * (though non-corrupting) DB errors like INSERT key conflicts
 381          * or deadlocks when using LOCK IN SHARE MODE.
 382          *
 383          * @param string $key
 384          * @param int $timeout Lock wait timeout; 0 for non-blocking [optional]
 385          * @param int $expiry Lock expiry [optional]; 1 day maximum
 386          * @param string $rclass Allow reentry if set and the current lock used this value
 387          * @return ScopedCallback|null Returns null on failure
 388          * @since 1.26
 389          */
 390         final public function getScopedLock( $key, $timeout = 6, $expiry = 30, $rclass = '' ) {
 391                 $expiry = min( $expiry ?: INF, 86400 );
 392
 393                 if ( !$this->lock( $key, $timeout, $expiry, $rclass ) ) {
 394                         return null;
 395                 }
 396
 397                 $lSince = microtime( true ); // lock timestamp
 398                 // PHP 5.3: Can't use $this in a closure
 399                 $that = $this;
 400                 $logger = $this->logger;
 401
 402                 return new ScopedCallback( function() use ( $that, $logger, $key, $lSince, $expiry ) {
 403                         $latency = .050; // latency skew (err towards keeping lock present)
 404                         $age = ( microtime( true ) - $lSince + $latency );
 405                         if ( ( $age + $latency ) >= $expiry ) {
 406                                 $logger->warning( "Lock for $key held too long ($age sec)." );
 407                                 return; // expired; it's not "safe" to delete the key
 408                         }
 409                         $that->unlock( $key );
 410                 } );
 411         }
 412
 413         /**
 414          * Delete all objects expiring before a certain date.
 415          * @param string $date The reference date in MW format
 416          * @param callable|bool $progressCallback Optional, a function which will be called
 417          *     regularly during long-running operations with the percentage progress
 418          *     as the first parameter.
 419          *
 420          * @return bool Success, false if unimplemented
 421          */
 422         public function deleteObjectsExpiringBefore( $date, $progressCallback = false ) {
 423                 // stub
 424                 return false;
 425         }
 426
 427         /**
 428          * Get an associative array containing the item for each of the keys that have items.
 429          * @param array $keys List of strings
 430          * @param integer $flags Bitfield; supports READ_LATEST [optional]
 431          * @return array
 432          */
 433         public function getMulti( array $keys, $flags = 0 ) {
 434                 $res = array();
 435                 foreach ( $keys as $key ) {
 436                         $val = $this->get( $key );
 437                         if ( $val !== false ) {
 438                                 $res[$key] = $val;
 439                         }
 440                 }
 441                 return $res;
 442         }
 443
 444         /**
 445          * Batch insertion
 446          * @param array $data $key => $value assoc array
 447          * @param int $exptime Either an interval in seconds or a unix timestamp for expiry
 448          * @return bool Success
 449          * @since 1.24
 450          */
 451         public function setMulti( array $data, $exptime = 0 ) {
 452                 $res = true;
 453                 foreach ( $data as $key => $value ) {
 454                         if ( !$this->set( $key, $value, $exptime ) ) {
 455                                 $res = false;
 456                         }
 457                 }
 458                 return $res;
 459         }
 460
 461         /**
 462          * @param string $key
 463          * @param mixed $value
 464          * @param int $exptime
 465          * @return bool Success
 466          */
 467         public function add( $key, $value, $exptime = 0 ) {
 468                 if ( $this->get( $key ) === false ) {
 469                         return $this->set( $key, $value, $exptime );
 470                 }
 471                 return false; // key already set
 472         }
 473
 474         /**
 475          * Increase stored value of $key by $value while preserving its TTL
 476          * @param string $key Key to increase
 477          * @param int $value Value to add to $key (Default 1)
 478          * @return int|bool New value or false on failure
 479          */
 480         public function incr( $key, $value = 1 ) {
 481                 if ( !$this->lock( $key ) ) {
 482                         return false;
 483                 }
 484                 $n = $this->get( $key );
 485                 if ( $this->isInteger( $n ) ) { // key exists?
 486                         $n += intval( $value );
 487                         $this->set( $key, max( 0, $n ) ); // exptime?
 488                 } else {
 489                         $n = false;
 490                 }
 491                 $this->unlock( $key );
 492
 493                 return $n;
 494         }
 495
 496         /**
 497          * Decrease stored value of $key by $value while preserving its TTL
 498          * @param string $key
 499          * @param int $value
 500          * @return int|bool New value or false on failure
 501          */
 502         public function decr( $key, $value = 1 ) {
 503                 return $this->incr( $key, - $value );
 504         }
 505
 506         /**
 507          * Increase stored value of $key by $value while preserving its TTL
 508          *
 509          * This will create the key with value $init and TTL $ttl if not present
 510          *
 511          * @param string $key
 512          * @param int $ttl
 513          * @param int $value
 514          * @param int $init
 515          * @return bool
 516          * @since 1.24
 517          */
 518         public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) {
 519                 return $this->incr( $key, $value ) ||
 520                         $this->add( $key, (int)$init, $ttl ) || $this->incr( $key, $value );
 521         }
 522
 523         /**
 524          * Get the "last error" registered; clearLastError() should be called manually
 525          * @return int ERR_* constant for the "last error" registry
 526          * @since 1.23
 527          */
 528         public function getLastError() {
 529                 return $this->lastError;
 530         }
 531
 532         /**
 533          * Clear the "last error" registry
 534          * @since 1.23
 535          */
 536         public function clearLastError() {
 537                 $this->lastError = self::ERR_NONE;
 538         }
 539
 540         /**
 541          * Set the "last error" registry
 542          * @param int $err ERR_* constant
 543          * @since 1.23
 544          */
 545         protected function setLastError( $err ) {
 546                 $this->lastError = $err;
 547         }
 548
 549         /**
 550          * Modify a cache update operation array for EventRelayer::notify()
 551          *
 552          * This is used for relayed writes, e.g. for broadcasting a change
 553          * to multiple data-centers. If the array contains a 'val' field
 554          * then the command involves setting a key to that value. Note that
 555          * for simplicity, 'val' is always a simple scalar value. This method
 556          * is used to possibly serialize the value and add any cache-specific
 557          * key/values needed for the relayer daemon (e.g. memcached flags).
 558          *
 559          * @param array $event
 560          * @return array
 561          * @since 1.26
 562          */
 563         public function modifySimpleRelayEvent( array $event ) {
 564                 return $event;
 565         }
 566
 567         /**
 568          * @param string $text
 569          */
 570         protected function debug( $text ) {
 571                 if ( $this->debugMode ) {
 572                         $this->logger->debug( "{class} debug: $text", array(
 573                                 'class' => get_class( $this ),
 574                         ) );
 575                 }
 576         }
 577
 578         /**
 579          * Convert an optionally relative time to an absolute time
 580          * @param int $exptime
 581          * @return int
 582          */
 583         protected function convertExpiry( $exptime ) {
 584                 if ( ( $exptime != 0 ) && ( $exptime < 86400 * 3650 /* 10 years */ ) ) {
 585                         return time() + $exptime;
 586                 } else {
 587                         return $exptime;
 588                 }
 589         }
 590
 591         /**
 592          * Convert an optionally absolute expiry time to a relative time. If an
 593          * absolute time is specified which is in the past, use a short expiry time.
 594          *
 595          * @param int $exptime
 596          * @return int
 597          */
 598         protected function convertToRelative( $exptime ) {
 599                 if ( $exptime >= 86400 * 3650 /* 10 years */ ) {
 600                         $exptime -= time();
 601                         if ( $exptime <= 0 ) {
 602                                 $exptime = 1;
 603                         }
 604                         return $exptime;
 605                 } else {
 606                         return $exptime;
 607                 }
 608         }
 609
 610         /**
 611          * Check if a value is an integer
 612          *
 613          * @param mixed $value
 614          * @return bool
 615          */
 616         protected function isInteger( $value ) {
 617                 return ( is_int( $value ) || ctype_digit( $value ) );
 618         }
 619
 620         /**
 621          * Construct a cache key.
 622          *
 623          * @since 1.27
 624          * @param string $keyspace
 625          * @param array $args
 626          * @return string
 627          */
 628         public function makeKeyInternal( $keyspace, $args ) {
 629                 $key = $keyspace . ':' . implode( ':', $args );
 630                 return strtr( $key, ' ', '_' );
 631         }
 632
 633         /**
 634          * Make a global cache key.
 635          *
 636          * @since 1.27
 637          * @param string ... Key component (variadic)
 638          * @return string
 639          */
 640         public function makeGlobalKey() {
 641                 return $this->makeKeyInternal( 'global', func_get_args() );
 642         }
 643
 644         /**
 645          * Make a cache key, scoped to this instance's keyspace.
 646          *
 647          * @since 1.27
 648          * @param string ... Key component (variadic)
 649          * @return string
 650          */
 651         public function makeKey() {
 652                 return $this->makeKeyInternal( $this->keyspace, func_get_args() );
 653         }
 654 }