3 * Object caching using Redis (http://redis.io/).
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
24 * Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4
26 * @see https://github.com/phpredis/phpredis/blob/d310ed7c8/Changelog.md
27 * @note Avoid use of Redis::MULTI transactions for twemproxy support
32 class RedisBagOStuff
extends MediumSpecificBagOStuff
{
33 /** @var RedisConnectionPool */
35 /** @var array List of server names */
37 /** @var array Map of (tag => server name) */
38 protected $serverTagMap;
40 protected $automaticFailover;
43 * Construct a RedisBagOStuff object. Parameters are:
45 * - servers: An array of server names. A server name may be a hostname,
46 * a hostname/port combination or the absolute path of a UNIX socket.
47 * If a hostname is specified but no port, the standard port number
48 * 6379 will be used. Arrays keys can be used to specify the tag to
49 * hash on in place of the host/port. Required.
51 * - connectTimeout: The timeout for new connections, in seconds. Optional,
52 * default is 1 second.
54 * - persistent: Set this to true to allow connections to persist across
55 * multiple web requests. False by default.
57 * - password: The authentication password, will be sent to Redis in
58 * clear text. Optional, if it is unspecified, no AUTH command will be
61 * - automaticFailover: If this is false, then each key will be mapped to
62 * a single server, and if that server is down, any requests for that key
63 * will fail. If this is true, a connection failure will cause the client
64 * to immediately try the next server in the list (as determined by a
65 * consistent hashing algorithm). True by default. This has the
66 * potential to create consistency issues if a server is slow enough to
67 * flap, for example if it is in swap death.
68 * @param array $params
70 function __construct( $params ) {
71 parent
::__construct( $params );
72 $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
73 foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
74 if ( isset( $params[$opt] ) ) {
75 $redisConf[$opt] = $params[$opt];
78 $this->redisPool
= RedisConnectionPool
::singleton( $redisConf );
80 $this->servers
= $params['servers'];
81 foreach ( $this->servers
as $key => $server ) {
82 $this->serverTagMap
[is_int( $key ) ?
$server : $key] = $server;
85 $this->automaticFailover
= $params['automaticFailover'] ??
true;
87 $this->attrMap
[self
::ATTR_SYNCWRITES
] = self
::QOS_SYNCWRITES_NONE
;
90 protected function doGet( $key, $flags = 0, &$casToken = null ) {
93 $conn = $this->getConnection( $key );
100 $value = $conn->get( $key );
102 $result = $this->unserialize( $value );
103 } catch ( RedisException
$e ) {
105 $this->handleException( $conn, $e );
108 $this->logRequest( 'get', $key, $conn->getServer(), $e );
113 protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
114 $conn = $this->getConnection( $key );
119 $ttl = $this->convertToRelative( $exptime );
124 $result = $conn->setex( $key, $ttl, $this->serialize( $value ) );
126 $result = $conn->set( $key, $this->serialize( $value ) );
128 } catch ( RedisException
$e ) {
130 $this->handleException( $conn, $e );
133 $this->logRequest( 'set', $key, $conn->getServer(), $e );
138 protected function doDelete( $key, $flags = 0 ) {
139 $conn = $this->getConnection( $key );
146 // Note that redis does not return false if the key was not there
147 $result = ( $conn->del( $key ) !== false );
148 } catch ( RedisException
$e ) {
150 $this->handleException( $conn, $e );
153 $this->logRequest( 'delete', $key, $conn->getServer(), $e );
158 protected function doGetMulti( array $keys, $flags = 0 ) {
159 /** @var RedisConnRef[]|Redis[] $conns */
162 foreach ( $keys as $key ) {
163 $conn = $this->getConnection( $key );
165 $server = $conn->getServer();
166 $conns[$server] = $conn;
167 $batches[$server][] = $key;
172 foreach ( $batches as $server => $batchKeys ) {
173 $conn = $conns[$server];
177 // Avoid mget() to reduce CPU hogging from a single request
178 $conn->multi( Redis
::PIPELINE
);
179 foreach ( $batchKeys as $key ) {
182 $batchResult = $conn->exec();
183 if ( $batchResult === false ) {
184 $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
188 foreach ( $batchResult as $i => $value ) {
189 if ( $value !== false ) {
190 $result[$batchKeys[$i]] = $this->unserialize( $value );
193 } catch ( RedisException
$e ) {
194 $this->handleException( $conn, $e );
197 $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
203 protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
204 /** @var RedisConnRef[]|Redis[] $conns */
207 foreach ( $data as $key => $value ) {
208 $conn = $this->getConnection( $key );
210 $server = $conn->getServer();
211 $conns[$server] = $conn;
212 $batches[$server][] = $key;
216 $ttl = $this->convertToRelative( $exptime );
217 $op = $ttl ?
'setex' : 'set';
220 foreach ( $batches as $server => $batchKeys ) {
221 $conn = $conns[$server];
225 // Avoid mset() to reduce CPU hogging from a single request
226 $conn->multi( Redis
::PIPELINE
);
227 foreach ( $batchKeys as $key ) {
229 $conn->setex( $key, $ttl, $this->serialize( $data[$key] ) );
231 $conn->set( $key, $this->serialize( $data[$key] ) );
234 $batchResult = $conn->exec();
235 if ( $batchResult === false ) {
236 $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
239 $result = $result && !in_array( false, $batchResult, true );
240 } catch ( RedisException
$e ) {
241 $this->handleException( $conn, $e );
245 $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
251 protected function doDeleteMulti( array $keys, $flags = 0 ) {
252 /** @var RedisConnRef[]|Redis[] $conns */
255 foreach ( $keys as $key ) {
256 $conn = $this->getConnection( $key );
258 $server = $conn->getServer();
259 $conns[$server] = $conn;
260 $batches[$server][] = $key;
265 foreach ( $batches as $server => $batchKeys ) {
266 $conn = $conns[$server];
270 // Avoid delete() with array to reduce CPU hogging from a single request
271 $conn->multi( Redis
::PIPELINE
);
272 foreach ( $batchKeys as $key ) {
275 $batchResult = $conn->exec();
276 if ( $batchResult === false ) {
277 $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
280 // Note that redis does not return false if the key was not there
281 $result = $result && !in_array( false, $batchResult, true );
282 } catch ( RedisException
$e ) {
283 $this->handleException( $conn, $e );
287 $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
293 public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
294 /** @var RedisConnRef[]|Redis[] $conns */
297 foreach ( $keys as $key ) {
298 $conn = $this->getConnection( $key );
300 $server = $conn->getServer();
301 $conns[$server] = $conn;
302 $batches[$server][] = $key;
306 $relative = $this->expiryIsRelative( $exptime );
307 $op = ( $exptime == 0 ) ?
'persist' : ( $relative ?
'expire' : 'expireAt' );
310 foreach ( $batches as $server => $batchKeys ) {
311 $conn = $conns[$server];
315 $conn->multi( Redis
::PIPELINE
);
316 foreach ( $batchKeys as $key ) {
317 if ( $exptime == 0 ) {
318 $conn->persist( $key );
319 } elseif ( $relative ) {
320 $conn->expire( $key, $this->convertToRelative( $exptime ) );
322 $conn->expireAt( $key, $this->convertToExpiry( $exptime ) );
325 $batchResult = $conn->exec();
326 if ( $batchResult === false ) {
327 $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
330 $result = in_array( false, $batchResult, true ) ?
false : $result;
331 } catch ( RedisException
$e ) {
332 $this->handleException( $conn, $e );
336 $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
342 public function add( $key, $value, $expiry = 0, $flags = 0 ) {
343 $conn = $this->getConnection( $key );
348 $ttl = $this->convertToRelative( $expiry );
350 $result = $conn->set(
352 $this->serialize( $value ),
353 $ttl ?
[ 'nx', 'ex' => $ttl ] : [ 'nx' ]
355 } catch ( RedisException
$e ) {
357 $this->handleException( $conn, $e );
360 $this->logRequest( 'add', $key, $conn->getServer(), $result );
365 public function incr( $key, $value = 1 ) {
366 $conn = $this->getConnection( $key );
372 if ( !$conn->exists( $key ) ) {
375 // @FIXME: on races, the key may have a 0 TTL
376 $result = $conn->incrBy( $key, $value );
377 } catch ( RedisException
$e ) {
379 $this->handleException( $conn, $e );
382 $this->logRequest( 'incr', $key, $conn->getServer(), $result );
387 protected function doChangeTTL( $key, $exptime, $flags ) {
388 $conn = $this->getConnection( $key );
393 $relative = $this->expiryIsRelative( $exptime );
395 if ( $exptime == 0 ) {
396 $result = $conn->persist( $key );
397 $this->logRequest( 'persist', $key, $conn->getServer(), $result );
398 } elseif ( $relative ) {
399 $result = $conn->expire( $key, $this->convertToRelative( $exptime ) );
400 $this->logRequest( 'expire', $key, $conn->getServer(), $result );
402 $result = $conn->expireAt( $key, $this->convertToExpiry( $exptime ) );
403 $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
405 } catch ( RedisException
$e ) {
407 $this->handleException( $conn, $e );
415 * @return RedisConnRef|Redis|null Redis handle wrapper for the key or null on failure
417 protected function getConnection( $key ) {
418 $candidates = array_keys( $this->serverTagMap
);
420 if ( count( $this->servers
) > 1 ) {
421 ArrayUtils
::consistentHashSort( $candidates, $key, '/' );
422 if ( !$this->automaticFailover
) {
423 $candidates = array_slice( $candidates, 0, 1 );
427 while ( ( $tag = array_shift( $candidates ) ) !== null ) {
428 $server = $this->serverTagMap
[$tag];
429 $conn = $this->redisPool
->getConnection( $server, $this->logger
);
434 // If automatic failover is enabled, check that the server's link
435 // to its master (if any) is up -- but only if there are other
436 // viable candidates left to consider. Also, getMasterLinkStatus()
437 // does not work with twemproxy, though $candidates will be empty
438 // by now in such cases.
439 if ( $this->automaticFailover
&& $candidates ) {
441 /** @var string[] $info */
442 $info = $conn->info();
443 if ( ( $info['master_link_status'] ??
null ) === 'down' ) {
444 // If the master cannot be reached, fail-over to the next server.
445 // If masters are in data-center A, and replica DBs in data-center B,
446 // this helps avoid the case were fail-over happens in A but not
447 // to the corresponding server in B (e.g. read/write mismatch).
450 } catch ( RedisException
$e ) {
451 // Server is not accepting commands
452 $this->redisPool
->handleError( $conn, $e );
460 $this->setLastError( BagOStuff
::ERR_UNREACHABLE
);
469 protected function logError( $msg ) {
470 $this->logger
->error( "Redis error: $msg" );
474 * The redis extension throws an exception in response to various read, write
475 * and protocol errors. Sometimes it also closes the connection, sometimes
476 * not. The safest response for us is to explicitly destroy the connection
477 * object and let it be reopened during the next request.
478 * @param RedisConnRef $conn
479 * @param RedisException $e
481 protected function handleException( RedisConnRef
$conn, RedisException
$e ) {
482 $this->setLastError( BagOStuff
::ERR_UNEXPECTED
);
483 $this->redisPool
->handleError( $conn, $e );
487 * Send information about a single request to the debug log
489 * @param string $keys
490 * @param string $server
491 * @param Exception|bool|null $e
493 public function logRequest( $op, $keys, $server, $e = null ) {
494 $this->debug( "$op($keys) on $server: " . ( $e ?
"failure" : "success" ) );