includes/libs/objectcache/RedisBagOStuff.php

   1 <?php
   2 /**
   3  * Object caching using Redis (http://redis.io/).
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 /**
  24  * Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4
  25  *
  26  * @see https://github.com/phpredis/phpredis/blob/d310ed7c8/Changelog.md
  27  * @note Avoid use of Redis::MULTI transactions for twemproxy support
  28  *
  29  * @ingroup Cache
  30  * @ingroup Redis
  31  */
  32 class RedisBagOStuff extends MediumSpecificBagOStuff {
  33         /** @var RedisConnectionPool */
  34         protected $redisPool;
  35         /** @var array List of server names */
  36         protected $servers;
  37         /** @var array Map of (tag => server name) */
  38         protected $serverTagMap;
  39         /** @var bool */
  40         protected $automaticFailover;
  41
  42         /**
  43          * Construct a RedisBagOStuff object. Parameters are:
  44          *
  45          *   - servers: An array of server names. A server name may be a hostname,
  46          *     a hostname/port combination or the absolute path of a UNIX socket.
  47          *     If a hostname is specified but no port, the standard port number
  48          *     6379 will be used. Arrays keys can be used to specify the tag to
  49          *     hash on in place of the host/port. Required.
  50          *
  51          *   - connectTimeout: The timeout for new connections, in seconds. Optional,
  52          *     default is 1 second.
  53          *
  54          *   - persistent: Set this to true to allow connections to persist across
  55          *     multiple web requests. False by default.
  56          *
  57          *   - password: The authentication password, will be sent to Redis in
  58          *     clear text. Optional, if it is unspecified, no AUTH command will be
  59          *     sent.
  60          *
  61          *   - automaticFailover: If this is false, then each key will be mapped to
  62          *     a single server, and if that server is down, any requests for that key
  63          *     will fail. If this is true, a connection failure will cause the client
  64          *     to immediately try the next server in the list (as determined by a
  65          *     consistent hashing algorithm). True by default. This has the
  66          *     potential to create consistency issues if a server is slow enough to
  67          *     flap, for example if it is in swap death.
  68          * @param array $params
  69          */
  70         function __construct( $params ) {
  71                 parent::__construct( $params );
  72                 $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
  73                 foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
  74                         if ( isset( $params[$opt] ) ) {
  75                                 $redisConf[$opt] = $params[$opt];
  76                         }
  77                 }
  78                 $this->redisPool = RedisConnectionPool::singleton( $redisConf );
  79
  80                 $this->servers = $params['servers'];
  81                 foreach ( $this->servers as $key => $server ) {
  82                         $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
  83                 }
  84
  85                 $this->automaticFailover = $params['automaticFailover'] ?? true;
  86
  87                 $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_NONE;
  88         }
  89
  90         protected function doGet( $key, $flags = 0, &$casToken = null ) {
  91                 $casToken = null;
  92
  93                 $conn = $this->getConnection( $key );
  94                 if ( !$conn ) {
  95                         return false;
  96                 }
  97
  98                 $e = null;
  99                 try {
 100                         $value = $conn->get( $key );
 101                         $casToken = $value;
 102                         $result = $this->unserialize( $value );
 103                 } catch ( RedisException $e ) {
 104                         $result = false;
 105                         $this->handleException( $conn, $e );
 106                 }
 107
 108                 $this->logRequest( 'get', $key, $conn->getServer(), $e );
 109
 110                 return $result;
 111         }
 112
 113         protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
 114                 $conn = $this->getConnection( $key );
 115                 if ( !$conn ) {
 116                         return false;
 117                 }
 118
 119                 $ttl = $this->getExpirationAsTTL( $exptime );
 120
 121                 $e = null;
 122                 try {
 123                         if ( $ttl ) {
 124                                 $result = $conn->setex( $key, $ttl, $this->serialize( $value ) );
 125                         } else {
 126                                 $result = $conn->set( $key, $this->serialize( $value ) );
 127                         }
 128                 } catch ( RedisException $e ) {
 129                         $result = false;
 130                         $this->handleException( $conn, $e );
 131                 }
 132
 133                 $this->logRequest( 'set', $key, $conn->getServer(), $e );
 134
 135                 return $result;
 136         }
 137
 138         protected function doDelete( $key, $flags = 0 ) {
 139                 $conn = $this->getConnection( $key );
 140                 if ( !$conn ) {
 141                         return false;
 142                 }
 143
 144                 $e = null;
 145                 try {
 146                         // Note that redis does not return false if the key was not there
 147                         $result = ( $conn->del( $key ) !== false );
 148                 } catch ( RedisException $e ) {
 149                         $result = false;
 150                         $this->handleException( $conn, $e );
 151                 }
 152
 153                 $this->logRequest( 'delete', $key, $conn->getServer(), $e );
 154
 155                 return $result;
 156         }
 157
 158         protected function doGetMulti( array $keys, $flags = 0 ) {
 159                 /** @var RedisConnRef[]|Redis[] $conns */
 160                 $conns = [];
 161                 $batches = [];
 162                 foreach ( $keys as $key ) {
 163                         $conn = $this->getConnection( $key );
 164                         if ( $conn ) {
 165                                 $server = $conn->getServer();
 166                                 $conns[$server] = $conn;
 167                                 $batches[$server][] = $key;
 168                         }
 169                 }
 170
 171                 $result = [];
 172                 foreach ( $batches as $server => $batchKeys ) {
 173                         $conn = $conns[$server];
 174
 175                         $e = null;
 176                         try {
 177                                 // Avoid mget() to reduce CPU hogging from a single request
 178                                 $conn->multi( Redis::PIPELINE );
 179                                 foreach ( $batchKeys as $key ) {
 180                                         $conn->get( $key );
 181                                 }
 182                                 $batchResult = $conn->exec();
 183                                 if ( $batchResult === false ) {
 184                                         $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
 185                                         continue;
 186                                 }
 187
 188                                 foreach ( $batchResult as $i => $value ) {
 189                                         if ( $value !== false ) {
 190                                                 $result[$batchKeys[$i]] = $this->unserialize( $value );
 191                                         }
 192                                 }
 193                         } catch ( RedisException $e ) {
 194                                 $this->handleException( $conn, $e );
 195                         }
 196
 197                         $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
 198                 }
 199
 200                 return $result;
 201         }
 202
 203         protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
 204                 /** @var RedisConnRef[]|Redis[] $conns */
 205                 $conns = [];
 206                 $batches = [];
 207                 foreach ( $data as $key => $value ) {
 208                         $conn = $this->getConnection( $key );
 209                         if ( $conn ) {
 210                                 $server = $conn->getServer();
 211                                 $conns[$server] = $conn;
 212                                 $batches[$server][] = $key;
 213                         }
 214                 }
 215
 216                 $ttl = $this->getExpirationAsTTL( $exptime );
 217                 $op = $ttl ? 'setex' : 'set';
 218
 219                 $result = true;
 220                 foreach ( $batches as $server => $batchKeys ) {
 221                         $conn = $conns[$server];
 222
 223                         $e = null;
 224                         try {
 225                                 // Avoid mset() to reduce CPU hogging from a single request
 226                                 $conn->multi( Redis::PIPELINE );
 227                                 foreach ( $batchKeys as $key ) {
 228                                         if ( $ttl ) {
 229                                                 $conn->setex( $key, $ttl, $this->serialize( $data[$key] ) );
 230                                         } else {
 231                                                 $conn->set( $key, $this->serialize( $data[$key] ) );
 232                                         }
 233                                 }
 234                                 $batchResult = $conn->exec();
 235                                 if ( $batchResult === false ) {
 236                                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
 237                                         continue;
 238                                 }
 239                                 $result = $result && !in_array( false, $batchResult, true );
 240                         } catch ( RedisException $e ) {
 241                                 $this->handleException( $conn, $e );
 242                                 $result = false;
 243                         }
 244
 245                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
 246                 }
 247
 248                 return $result;
 249         }
 250
 251         protected function doDeleteMulti( array $keys, $flags = 0 ) {
 252                 /** @var RedisConnRef[]|Redis[] $conns */
 253                 $conns = [];
 254                 $batches = [];
 255                 foreach ( $keys as $key ) {
 256                         $conn = $this->getConnection( $key );
 257                         if ( $conn ) {
 258                                 $server = $conn->getServer();
 259                                 $conns[$server] = $conn;
 260                                 $batches[$server][] = $key;
 261                         }
 262                 }
 263
 264                 $result = true;
 265                 foreach ( $batches as $server => $batchKeys ) {
 266                         $conn = $conns[$server];
 267
 268                         $e = null;
 269                         try {
 270                                 // Avoid delete() with array to reduce CPU hogging from a single request
 271                                 $conn->multi( Redis::PIPELINE );
 272                                 foreach ( $batchKeys as $key ) {
 273                                         $conn->del( $key );
 274                                 }
 275                                 $batchResult = $conn->exec();
 276                                 if ( $batchResult === false ) {
 277                                         $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
 278                                         continue;
 279                                 }
 280                                 // Note that redis does not return false if the key was not there
 281                                 $result = $result && !in_array( false, $batchResult, true );
 282                         } catch ( RedisException $e ) {
 283                                 $this->handleException( $conn, $e );
 284                                 $result = false;
 285                         }
 286
 287                         $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
 288                 }
 289
 290                 return $result;
 291         }
 292
 293         public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
 294                 /** @var RedisConnRef[]|Redis[] $conns */
 295                 $conns = [];
 296                 $batches = [];
 297                 foreach ( $keys as $key ) {
 298                         $conn = $this->getConnection( $key );
 299                         if ( $conn ) {
 300                                 $server = $conn->getServer();
 301                                 $conns[$server] = $conn;
 302                                 $batches[$server][] = $key;
 303                         }
 304                 }
 305
 306                 $relative = $this->isRelativeExpiration( $exptime );
 307                 $op = ( $exptime == self::TTL_INDEFINITE )
 308                         ? 'persist'
 309                         : ( $relative ? 'expire' : 'expireAt' );
 310
 311                 $result = true;
 312                 foreach ( $batches as $server => $batchKeys ) {
 313                         $conn = $conns[$server];
 314
 315                         $e = null;
 316                         try {
 317                                 $conn->multi( Redis::PIPELINE );
 318                                 foreach ( $batchKeys as $key ) {
 319                                         if ( $exptime == self::TTL_INDEFINITE ) {
 320                                                 $conn->persist( $key );
 321                                         } elseif ( $relative ) {
 322                                                 $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
 323                                         } else {
 324                                                 $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
 325                                         }
 326                                 }
 327                                 $batchResult = $conn->exec();
 328                                 if ( $batchResult === false ) {
 329                                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
 330                                         continue;
 331                                 }
 332                                 $result = in_array( false, $batchResult, true ) ? false : $result;
 333                         } catch ( RedisException $e ) {
 334                                 $this->handleException( $conn, $e );
 335                                 $result = false;
 336                         }
 337
 338                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
 339                 }
 340
 341                 return $result;
 342         }
 343
 344         protected function doAdd( $key, $value, $expiry = 0, $flags = 0 ) {
 345                 $conn = $this->getConnection( $key );
 346                 if ( !$conn ) {
 347                         return false;
 348                 }
 349
 350                 $ttl = $this->getExpirationAsTTL( $expiry );
 351                 try {
 352                         $result = $conn->set(
 353                                 $key,
 354                                 $this->serialize( $value ),
 355                                 $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
 356                         );
 357                 } catch ( RedisException $e ) {
 358                         $result = false;
 359                         $this->handleException( $conn, $e );
 360                 }
 361
 362                 $this->logRequest( 'add', $key, $conn->getServer(), $result );
 363
 364                 return $result;
 365         }
 366
 367         public function incr( $key, $value = 1 ) {
 368                 $conn = $this->getConnection( $key );
 369                 if ( !$conn ) {
 370                         return false;
 371                 }
 372
 373                 try {
 374                         if ( !$conn->exists( $key ) ) {
 375                                 return false;
 376                         }
 377                         // @FIXME: on races, the key may have a 0 TTL
 378                         $result = $conn->incrBy( $key, $value );
 379                 } catch ( RedisException $e ) {
 380                         $result = false;
 381                         $this->handleException( $conn, $e );
 382                 }
 383
 384                 $this->logRequest( 'incr', $key, $conn->getServer(), $result );
 385
 386                 return $result;
 387         }
 388
 389         protected function doChangeTTL( $key, $exptime, $flags ) {
 390                 $conn = $this->getConnection( $key );
 391                 if ( !$conn ) {
 392                         return false;
 393                 }
 394
 395                 $relative = $this->isRelativeExpiration( $exptime );
 396                 try {
 397                         if ( $exptime == self::TTL_INDEFINITE ) {
 398                                 $result = $conn->persist( $key );
 399                                 $this->logRequest( 'persist', $key, $conn->getServer(), $result );
 400                         } elseif ( $relative ) {
 401                                 $result = $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
 402                                 $this->logRequest( 'expire', $key, $conn->getServer(), $result );
 403                         } else {
 404                                 $result = $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
 405                                 $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
 406                         }
 407                 } catch ( RedisException $e ) {
 408                         $result = false;
 409                         $this->handleException( $conn, $e );
 410                 }
 411
 412                 return $result;
 413         }
 414
 415         /**
 416          * @param string $key
 417          * @return RedisConnRef|Redis|null Redis handle wrapper for the key or null on failure
 418          */
 419         protected function getConnection( $key ) {
 420                 $candidates = array_keys( $this->serverTagMap );
 421
 422                 if ( count( $this->servers ) > 1 ) {
 423                         ArrayUtils::consistentHashSort( $candidates, $key, '/' );
 424                         if ( !$this->automaticFailover ) {
 425                                 $candidates = array_slice( $candidates, 0, 1 );
 426                         }
 427                 }
 428
 429                 while ( ( $tag = array_shift( $candidates ) ) !== null ) {
 430                         $server = $this->serverTagMap[$tag];
 431                         $conn = $this->redisPool->getConnection( $server, $this->logger );
 432                         if ( !$conn ) {
 433                                 continue;
 434                         }
 435
 436                         // If automatic failover is enabled, check that the server's link
 437                         // to its master (if any) is up -- but only if there are other
 438                         // viable candidates left to consider. Also, getMasterLinkStatus()
 439                         // does not work with twemproxy, though $candidates will be empty
 440                         // by now in such cases.
 441                         if ( $this->automaticFailover && $candidates ) {
 442                                 try {
 443                                         /** @var string[] $info */
 444                                         $info = $conn->info();
 445                                         if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
 446                                                 // If the master cannot be reached, fail-over to the next server.
 447                                                 // If masters are in data-center A, and replica DBs in data-center B,
 448                                                 // this helps avoid the case were fail-over happens in A but not
 449                                                 // to the corresponding server in B (e.g. read/write mismatch).
 450                                                 continue;
 451                                         }
 452                                 } catch ( RedisException $e ) {
 453                                         // Server is not accepting commands
 454                                         $this->redisPool->handleError( $conn, $e );
 455                                         continue;
 456                                 }
 457                         }
 458
 459                         return $conn;
 460                 }
 461
 462                 $this->setLastError( BagOStuff::ERR_UNREACHABLE );
 463
 464                 return null;
 465         }
 466
 467         /**
 468          * Log a fatal error
 469          * @param string $msg
 470          */
 471         protected function logError( $msg ) {
 472                 $this->logger->error( "Redis error: $msg" );
 473         }
 474
 475         /**
 476          * The redis extension throws an exception in response to various read, write
 477          * and protocol errors. Sometimes it also closes the connection, sometimes
 478          * not. The safest response for us is to explicitly destroy the connection
 479          * object and let it be reopened during the next request.
 480          * @param RedisConnRef $conn
 481          * @param RedisException $e
 482          */
 483         protected function handleException( RedisConnRef $conn, RedisException $e ) {
 484                 $this->setLastError( BagOStuff::ERR_UNEXPECTED );
 485                 $this->redisPool->handleError( $conn, $e );
 486         }
 487
 488         /**
 489          * Send information about a single request to the debug log
 490          * @param string $op
 491          * @param string $keys
 492          * @param string $server
 493          * @param Exception|bool|null $e
 494          */
 495         public function logRequest( $op, $keys, $server, $e = null ) {
 496                 $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
 497         }
 498 }