includes/libs/objectcache/RedisBagOStuff.php

   1 <?php
   2 /**
   3  * Object caching using Redis (http://redis.io/).
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 /**
  24  * Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4
  25  *
  26  * @see https://github.com/phpredis/phpredis/blob/d310ed7c8/Changelog.md
  27  * @note Avoid use of Redis::MULTI transactions for twemproxy support
  28  *
  29  * @ingroup Cache
  30  * @ingroup Redis
  31  */
  32 class RedisBagOStuff extends MediumSpecificBagOStuff {
  33         /** @var RedisConnectionPool */
  34         protected $redisPool;
  35         /** @var array List of server names */
  36         protected $servers;
  37         /** @var array Map of (tag => server name) */
  38         protected $serverTagMap;
  39         /** @var bool */
  40         protected $automaticFailover;
  41
  42         /**
  43          * Construct a RedisBagOStuff object. Parameters are:
  44          *
  45          *   - servers: An array of server names. A server name may be a hostname,
  46          *     a hostname/port combination or the absolute path of a UNIX socket.
  47          *     If a hostname is specified but no port, the standard port number
  48          *     6379 will be used. Arrays keys can be used to specify the tag to
  49          *     hash on in place of the host/port. Required.
  50          *
  51          *   - connectTimeout: The timeout for new connections, in seconds. Optional,
  52          *     default is 1 second.
  53          *
  54          *   - persistent: Set this to true to allow connections to persist across
  55          *     multiple web requests. False by default.
  56          *
  57          *   - password: The authentication password, will be sent to Redis in
  58          *     clear text. Optional, if it is unspecified, no AUTH command will be
  59          *     sent.
  60          *
  61          *   - automaticFailover: If this is false, then each key will be mapped to
  62          *     a single server, and if that server is down, any requests for that key
  63          *     will fail. If this is true, a connection failure will cause the client
  64          *     to immediately try the next server in the list (as determined by a
  65          *     consistent hashing algorithm). True by default. This has the
  66          *     potential to create consistency issues if a server is slow enough to
  67          *     flap, for example if it is in swap death.
  68          * @param array $params
  69          */
  70         function __construct( $params ) {
  71                 parent::__construct( $params );
  72                 $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
  73                 foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
  74                         if ( isset( $params[$opt] ) ) {
  75                                 $redisConf[$opt] = $params[$opt];
  76                         }
  77                 }
  78                 $this->redisPool = RedisConnectionPool::singleton( $redisConf );
  79
  80                 $this->servers = $params['servers'];
  81                 foreach ( $this->servers as $key => $server ) {
  82                         $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
  83                 }
  84
  85                 $this->automaticFailover = $params['automaticFailover'] ?? true;
  86
  87                 $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_NONE;
  88         }
  89
  90         protected function doGet( $key, $flags = 0, &$casToken = null ) {
  91                 $casToken = null;
  92
  93                 $conn = $this->getConnection( $key );
  94                 if ( !$conn ) {
  95                         return false;
  96                 }
  97
  98                 $e = null;
  99                 try {
 100                         $value = $conn->get( $key );
 101                         $casToken = $value;
 102                         $result = $this->unserialize( $value );
 103                 } catch ( RedisException $e ) {
 104                         $result = false;
 105                         $this->handleException( $conn, $e );
 106                 }
 107
 108                 $this->logRequest( 'get', $key, $conn->getServer(), $e );
 109
 110                 return $result;
 111         }
 112
 113         protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
 114                 $conn = $this->getConnection( $key );
 115                 if ( !$conn ) {
 116                         return false;
 117                 }
 118
 119                 $ttl = $this->convertToRelative( $exptime );
 120
 121                 $e = null;
 122                 try {
 123                         if ( $ttl ) {
 124                                 $result = $conn->setex( $key, $ttl, $this->serialize( $value ) );
 125                         } else {
 126                                 $result = $conn->set( $key, $this->serialize( $value ) );
 127                         }
 128                 } catch ( RedisException $e ) {
 129                         $result = false;
 130                         $this->handleException( $conn, $e );
 131                 }
 132
 133                 $this->logRequest( 'set', $key, $conn->getServer(), $e );
 134
 135                 return $result;
 136         }
 137
 138         protected function doDelete( $key, $flags = 0 ) {
 139                 $conn = $this->getConnection( $key );
 140                 if ( !$conn ) {
 141                         return false;
 142                 }
 143
 144                 $e = null;
 145                 try {
 146                         // Note that redis does not return false if the key was not there
 147                         $result = ( $conn->del( $key ) !== false );
 148                 } catch ( RedisException $e ) {
 149                         $result = false;
 150                         $this->handleException( $conn, $e );
 151                 }
 152
 153                 $this->logRequest( 'delete', $key, $conn->getServer(), $e );
 154
 155                 return $result;
 156         }
 157
 158         protected function doGetMulti( array $keys, $flags = 0 ) {
 159                 /** @var RedisConnRef[]|Redis[] $conns */
 160                 $conns = [];
 161                 $batches = [];
 162                 foreach ( $keys as $key ) {
 163                         $conn = $this->getConnection( $key );
 164                         if ( $conn ) {
 165                                 $server = $conn->getServer();
 166                                 $conns[$server] = $conn;
 167                                 $batches[$server][] = $key;
 168                         }
 169                 }
 170
 171                 $result = [];
 172                 foreach ( $batches as $server => $batchKeys ) {
 173                         $conn = $conns[$server];
 174
 175                         $e = null;
 176                         try {
 177                                 // Avoid mget() to reduce CPU hogging from a single request
 178                                 $conn->multi( Redis::PIPELINE );
 179                                 foreach ( $batchKeys as $key ) {
 180                                         $conn->get( $key );
 181                                 }
 182                                 $batchResult = $conn->exec();
 183                                 if ( $batchResult === false ) {
 184                                         $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
 185                                         continue;
 186                                 }
 187
 188                                 foreach ( $batchResult as $i => $value ) {
 189                                         if ( $value !== false ) {
 190                                                 $result[$batchKeys[$i]] = $this->unserialize( $value );
 191                                         }
 192                                 }
 193                         } catch ( RedisException $e ) {
 194                                 $this->handleException( $conn, $e );
 195                         }
 196
 197                         $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
 198                 }
 199
 200                 return $result;
 201         }
 202
 203         protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
 204                 /** @var RedisConnRef[]|Redis[] $conns */
 205                 $conns = [];
 206                 $batches = [];
 207                 foreach ( $data as $key => $value ) {
 208                         $conn = $this->getConnection( $key );
 209                         if ( $conn ) {
 210                                 $server = $conn->getServer();
 211                                 $conns[$server] = $conn;
 212                                 $batches[$server][] = $key;
 213                         }
 214                 }
 215
 216                 $ttl = $this->convertToRelative( $exptime );
 217                 $op = $ttl ? 'setex' : 'set';
 218
 219                 $result = true;
 220                 foreach ( $batches as $server => $batchKeys ) {
 221                         $conn = $conns[$server];
 222
 223                         $e = null;
 224                         try {
 225                                 // Avoid mset() to reduce CPU hogging from a single request
 226                                 $conn->multi( Redis::PIPELINE );
 227                                 foreach ( $batchKeys as $key ) {
 228                                         if ( $ttl ) {
 229                                                 $conn->setex( $key, $ttl, $this->serialize( $data[$key] ) );
 230                                         } else {
 231                                                 $conn->set( $key, $this->serialize( $data[$key] ) );
 232                                         }
 233                                 }
 234                                 $batchResult = $conn->exec();
 235                                 if ( $batchResult === false ) {
 236                                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
 237                                         continue;
 238                                 }
 239                                 $result = $result && !in_array( false, $batchResult, true );
 240                         } catch ( RedisException $e ) {
 241                                 $this->handleException( $conn, $e );
 242                                 $result = false;
 243                         }
 244
 245                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
 246                 }
 247
 248                 return $result;
 249         }
 250
 251         protected function doDeleteMulti( array $keys, $flags = 0 ) {
 252                 /** @var RedisConnRef[]|Redis[] $conns */
 253                 $conns = [];
 254                 $batches = [];
 255                 foreach ( $keys as $key ) {
 256                         $conn = $this->getConnection( $key );
 257                         if ( $conn ) {
 258                                 $server = $conn->getServer();
 259                                 $conns[$server] = $conn;
 260                                 $batches[$server][] = $key;
 261                         }
 262                 }
 263
 264                 $result = true;
 265                 foreach ( $batches as $server => $batchKeys ) {
 266                         $conn = $conns[$server];
 267
 268                         $e = null;
 269                         try {
 270                                 // Avoid delete() with array to reduce CPU hogging from a single request
 271                                 $conn->multi( Redis::PIPELINE );
 272                                 foreach ( $batchKeys as $key ) {
 273                                         $conn->del( $key );
 274                                 }
 275                                 $batchResult = $conn->exec();
 276                                 if ( $batchResult === false ) {
 277                                         $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
 278                                         continue;
 279                                 }
 280                                 // Note that redis does not return false if the key was not there
 281                                 $result = $result && !in_array( false, $batchResult, true );
 282                         } catch ( RedisException $e ) {
 283                                 $this->handleException( $conn, $e );
 284                                 $result = false;
 285                         }
 286
 287                         $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
 288                 }
 289
 290                 return $result;
 291         }
 292
 293         public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
 294                 /** @var RedisConnRef[]|Redis[] $conns */
 295                 $conns = [];
 296                 $batches = [];
 297                 foreach ( $keys as $key ) {
 298                         $conn = $this->getConnection( $key );
 299                         if ( $conn ) {
 300                                 $server = $conn->getServer();
 301                                 $conns[$server] = $conn;
 302                                 $batches[$server][] = $key;
 303                         }
 304                 }
 305
 306                 $relative = $this->expiryIsRelative( $exptime );
 307                 $op = ( $exptime == 0 ) ? 'persist' : ( $relative ? 'expire' : 'expireAt' );
 308
 309                 $result = true;
 310                 foreach ( $batches as $server => $batchKeys ) {
 311                         $conn = $conns[$server];
 312
 313                         $e = null;
 314                         try {
 315                                 $conn->multi( Redis::PIPELINE );
 316                                 foreach ( $batchKeys as $key ) {
 317                                         if ( $exptime == 0 ) {
 318                                                 $conn->persist( $key );
 319                                         } elseif ( $relative ) {
 320                                                 $conn->expire( $key, $this->convertToRelative( $exptime ) );
 321                                         } else {
 322                                                 $conn->expireAt( $key, $this->convertToExpiry( $exptime ) );
 323                                         }
 324                                 }
 325                                 $batchResult = $conn->exec();
 326                                 if ( $batchResult === false ) {
 327                                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
 328                                         continue;
 329                                 }
 330                                 $result = in_array( false, $batchResult, true ) ? false : $result;
 331                         } catch ( RedisException $e ) {
 332                                 $this->handleException( $conn, $e );
 333                                 $result = false;
 334                         }
 335
 336                         $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
 337                 }
 338
 339                 return $result;
 340         }
 341
 342         public function add( $key, $value, $expiry = 0, $flags = 0 ) {
 343                 $conn = $this->getConnection( $key );
 344                 if ( !$conn ) {
 345                         return false;
 346                 }
 347
 348                 $ttl = $this->convertToRelative( $expiry );
 349                 try {
 350                         $result = $conn->set(
 351                                 $key,
 352                                 $this->serialize( $value ),
 353                                 $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
 354                         );
 355                 } catch ( RedisException $e ) {
 356                         $result = false;
 357                         $this->handleException( $conn, $e );
 358                 }
 359
 360                 $this->logRequest( 'add', $key, $conn->getServer(), $result );
 361
 362                 return $result;
 363         }
 364
 365         public function incr( $key, $value = 1 ) {
 366                 $conn = $this->getConnection( $key );
 367                 if ( !$conn ) {
 368                         return false;
 369                 }
 370
 371                 try {
 372                         if ( !$conn->exists( $key ) ) {
 373                                 return false;
 374                         }
 375                         // @FIXME: on races, the key may have a 0 TTL
 376                         $result = $conn->incrBy( $key, $value );
 377                 } catch ( RedisException $e ) {
 378                         $result = false;
 379                         $this->handleException( $conn, $e );
 380                 }
 381
 382                 $this->logRequest( 'incr', $key, $conn->getServer(), $result );
 383
 384                 return $result;
 385         }
 386
 387         protected function doChangeTTL( $key, $exptime, $flags ) {
 388                 $conn = $this->getConnection( $key );
 389                 if ( !$conn ) {
 390                         return false;
 391                 }
 392
 393                 $relative = $this->expiryIsRelative( $exptime );
 394                 try {
 395                         if ( $exptime == 0 ) {
 396                                 $result = $conn->persist( $key );
 397                                 $this->logRequest( 'persist', $key, $conn->getServer(), $result );
 398                         } elseif ( $relative ) {
 399                                 $result = $conn->expire( $key, $this->convertToRelative( $exptime ) );
 400                                 $this->logRequest( 'expire', $key, $conn->getServer(), $result );
 401                         } else {
 402                                 $result = $conn->expireAt( $key, $this->convertToExpiry( $exptime ) );
 403                                 $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
 404                         }
 405                 } catch ( RedisException $e ) {
 406                         $result = false;
 407                         $this->handleException( $conn, $e );
 408                 }
 409
 410                 return $result;
 411         }
 412
 413         /**
 414          * @param string $key
 415          * @return RedisConnRef|Redis|null Redis handle wrapper for the key or null on failure
 416          */
 417         protected function getConnection( $key ) {
 418                 $candidates = array_keys( $this->serverTagMap );
 419
 420                 if ( count( $this->servers ) > 1 ) {
 421                         ArrayUtils::consistentHashSort( $candidates, $key, '/' );
 422                         if ( !$this->automaticFailover ) {
 423                                 $candidates = array_slice( $candidates, 0, 1 );
 424                         }
 425                 }
 426
 427                 while ( ( $tag = array_shift( $candidates ) ) !== null ) {
 428                         $server = $this->serverTagMap[$tag];
 429                         $conn = $this->redisPool->getConnection( $server, $this->logger );
 430                         if ( !$conn ) {
 431                                 continue;
 432                         }
 433
 434                         // If automatic failover is enabled, check that the server's link
 435                         // to its master (if any) is up -- but only if there are other
 436                         // viable candidates left to consider. Also, getMasterLinkStatus()
 437                         // does not work with twemproxy, though $candidates will be empty
 438                         // by now in such cases.
 439                         if ( $this->automaticFailover && $candidates ) {
 440                                 try {
 441                                         /** @var string[] $info */
 442                                         $info = $conn->info();
 443                                         if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
 444                                                 // If the master cannot be reached, fail-over to the next server.
 445                                                 // If masters are in data-center A, and replica DBs in data-center B,
 446                                                 // this helps avoid the case were fail-over happens in A but not
 447                                                 // to the corresponding server in B (e.g. read/write mismatch).
 448                                                 continue;
 449                                         }
 450                                 } catch ( RedisException $e ) {
 451                                         // Server is not accepting commands
 452                                         $this->redisPool->handleError( $conn, $e );
 453                                         continue;
 454                                 }
 455                         }
 456
 457                         return $conn;
 458                 }
 459
 460                 $this->setLastError( BagOStuff::ERR_UNREACHABLE );
 461
 462                 return null;
 463         }
 464
 465         /**
 466          * Log a fatal error
 467          * @param string $msg
 468          */
 469         protected function logError( $msg ) {
 470                 $this->logger->error( "Redis error: $msg" );
 471         }
 472
 473         /**
 474          * The redis extension throws an exception in response to various read, write
 475          * and protocol errors. Sometimes it also closes the connection, sometimes
 476          * not. The safest response for us is to explicitly destroy the connection
 477          * object and let it be reopened during the next request.
 478          * @param RedisConnRef $conn
 479          * @param RedisException $e
 480          */
 481         protected function handleException( RedisConnRef $conn, RedisException $e ) {
 482                 $this->setLastError( BagOStuff::ERR_UNEXPECTED );
 483                 $this->redisPool->handleError( $conn, $e );
 484         }
 485
 486         /**
 487          * Send information about a single request to the debug log
 488          * @param string $op
 489          * @param string $keys
 490          * @param string $server
 491          * @param Exception|bool|null $e
 492          */
 493         public function logRequest( $op, $keys, $server, $e = null ) {
 494                 $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
 495         }
 496 }