dépôts / lhc / web / wiklou.git / blob
? search:


2d1ed059b2d0de58ccb6b25b26e7560d779d5829
[lhc/web/wiklou.git] / includes / libs / objectcache / RedisBagOStuff.php
1 <?php
2 /**
3 * Object caching using Redis (http://redis.io/).
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 */
22
23 /**
24 * Redis-based caching module for redis server >= 2.6.12
25 *
26 * @note Avoid use of Redis::MULTI transactions for twemproxy support
27 *
28 * @ingroup Cache
29 * @ingroup Redis
30 */
31 class RedisBagOStuff extends BagOStuff {
32 /** @var RedisConnectionPool */
33 protected $redisPool;
34 /** @var array List of server names */
35 protected $servers;
36 /** @var array Map of (tag => server name) */
37 protected $serverTagMap;
38 /** @var bool */
39 protected $automaticFailover;
40
41 /**
42 * Construct a RedisBagOStuff object. Parameters are:
43 *
44 * - servers: An array of server names. A server name may be a hostname,
45 * a hostname/port combination or the absolute path of a UNIX socket.
46 * If a hostname is specified but no port, the standard port number
47 * 6379 will be used. Arrays keys can be used to specify the tag to
48 * hash on in place of the host/port. Required.
49 *
50 * - connectTimeout: The timeout for new connections, in seconds. Optional,
51 * default is 1 second.
52 *
53 * - persistent: Set this to true to allow connections to persist across
54 * multiple web requests. False by default.
55 *
56 * - password: The authentication password, will be sent to Redis in
57 * clear text. Optional, if it is unspecified, no AUTH command will be
58 * sent.
59 *
60 * - automaticFailover: If this is false, then each key will be mapped to
61 * a single server, and if that server is down, any requests for that key
62 * will fail. If this is true, a connection failure will cause the client
63 * to immediately try the next server in the list (as determined by a
64 * consistent hashing algorithm). True by default. This has the
65 * potential to create consistency issues if a server is slow enough to
66 * flap, for example if it is in swap death.
67 * @param array $params
68 */
69 function __construct( $params ) {
70 parent::__construct( $params );
71 $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
72 foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
73 if ( isset( $params[$opt] ) ) {
74 $redisConf[$opt] = $params[$opt];
75 }
76 }
77 $this->redisPool = RedisConnectionPool::singleton( $redisConf );
78
79 $this->servers = $params['servers'];
80 foreach ( $this->servers as $key => $server ) {
81 $this->serverTagMap[is_int( $key ) ? $server : $key] = $server;
82 }
83
84 $this->automaticFailover = $params['automaticFailover'] ?? true;
85
86 $this->attrMap[self::ATTR_SYNCWRITES] = self::QOS_SYNCWRITES_NONE;
87 }
88
89 protected function doGet( $key, $flags = 0, &$casToken = null ) {
90 $casToken = null;
91
92 $conn = $this->getConnection( $key );
93 if ( !$conn ) {
94 return false;
95 }
96
97 $e = null;
98 try {
99 $value = $conn->get( $key );
100 $casToken = $value;
101 $result = $this->unserialize( $value );
102 } catch ( RedisException $e ) {
103 $result = false;
104 $this->handleException( $conn, $e );
105 }
106
107 $this->logRequest( 'get', $key, $conn->getServer(), $e );
108
109 return $result;
110 }
111
112 protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
113 $conn = $this->getConnection( $key );
114 if ( !$conn ) {
115 return false;
116 }
117
118 $ttl = $this->convertToRelative( $exptime );
119
120 $e = null;
121 try {
122 if ( $ttl ) {
123 $result = $conn->setex( $key, $ttl, $this->serialize( $value ) );
124 } else {
125 $result = $conn->set( $key, $this->serialize( $value ) );
126 }
127 } catch ( RedisException $e ) {
128 $result = false;
129 $this->handleException( $conn, $e );
130 }
131
132 $this->logRequest( 'set', $key, $conn->getServer(), $e );
133
134 return $result;
135 }
136
137 protected function doDelete( $key, $flags = 0 ) {
138 $conn = $this->getConnection( $key );
139 if ( !$conn ) {
140 return false;
141 }
142
143 $e = null;
144 try {
145 // Note that redis does not return false if the key was not there
146 $result = ( $conn->delete( $key ) !== false );
147 } catch ( RedisException $e ) {
148 $result = false;
149 $this->handleException( $conn, $e );
150 }
151
152 $this->logRequest( 'delete', $key, $conn->getServer(), $e );
153
154 return $result;
155 }
156
157 protected function doGetMulti( array $keys, $flags = 0 ) {
158 /** @var RedisConnRef[]|Redis[] $conns */
159 $conns = [];
160 $batches = [];
161 foreach ( $keys as $key ) {
162 $conn = $this->getConnection( $key );
163 if ( $conn ) {
164 $server = $conn->getServer();
165 $conns[$server] = $conn;
166 $batches[$server][] = $key;
167 }
168 }
169
170 $result = [];
171 foreach ( $batches as $server => $batchKeys ) {
172 $conn = $conns[$server];
173
174 $e = null;
175 try {
176 // Avoid mget() to reduce CPU hogging from a single request
177 $conn->multi( Redis::PIPELINE );
178 foreach ( $batchKeys as $key ) {
179 $conn->get( $key );
180 }
181 $batchResult = $conn->exec();
182 if ( $batchResult === false ) {
183 $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
184 continue;
185 }
186
187 foreach ( $batchResult as $i => $value ) {
188 if ( $value !== false ) {
189 $result[$batchKeys[$i]] = $this->unserialize( $value );
190 }
191 }
192 } catch ( RedisException $e ) {
193 $this->handleException( $conn, $e );
194 }
195
196 $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
197 }
198
199 return $result;
200 }
201
202 protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
203 /** @var RedisConnRef[]|Redis[] $conns */
204 $conns = [];
205 $batches = [];
206 foreach ( $data as $key => $value ) {
207 $conn = $this->getConnection( $key );
208 if ( $conn ) {
209 $server = $conn->getServer();
210 $conns[$server] = $conn;
211 $batches[$server][] = $key;
212 }
213 }
214
215 $ttl = $this->convertToRelative( $exptime );
216 $op = $ttl ? 'setex' : 'set';
217
218 $result = true;
219 foreach ( $batches as $server => $batchKeys ) {
220 $conn = $conns[$server];
221
222 $e = null;
223 try {
224 // Avoid mset() to reduce CPU hogging from a single request
225 $conn->multi( Redis::PIPELINE );
226 foreach ( $batchKeys as $key ) {
227 if ( $ttl ) {
228 $conn->setex( $key, $ttl, $this->serialize( $data[$key] ) );
229 } else {
230 $conn->set( $key, $this->serialize( $data[$key] ) );
231 }
232 }
233 $batchResult = $conn->exec();
234 if ( $batchResult === false ) {
235 $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
236 continue;
237 }
238 $result = $result && !in_array( false, $batchResult, true );
239 } catch ( RedisException $e ) {
240 $this->handleException( $conn, $e );
241 $result = false;
242 }
243
244 $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
245 }
246
247 return $result;
248 }
249
250 protected function doDeleteMulti( array $keys, $flags = 0 ) {
251 /** @var RedisConnRef[]|Redis[] $conns */
252 $conns = [];
253 $batches = [];
254 foreach ( $keys as $key ) {
255 $conn = $this->getConnection( $key );
256 if ( $conn ) {
257 $server = $conn->getServer();
258 $conns[$server] = $conn;
259 $batches[$server][] = $key;
260 }
261 }
262
263 $result = true;
264 foreach ( $batches as $server => $batchKeys ) {
265 $conn = $conns[$server];
266
267 $e = null;
268 try {
269 // Avoid delete() with array to reduce CPU hogging from a single request
270 $conn->multi( Redis::PIPELINE );
271 foreach ( $batchKeys as $key ) {
272 $conn->delete( $key );
273 }
274 $batchResult = $conn->exec();
275 if ( $batchResult === false ) {
276 $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
277 continue;
278 }
279 // Note that redis does not return false if the key was not there
280 $result = $result && !in_array( false, $batchResult, true );
281 } catch ( RedisException $e ) {
282 $this->handleException( $conn, $e );
283 $result = false;
284 }
285
286 $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
287 }
288
289 return $result;
290 }
291
292 public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
293 /** @var RedisConnRef[]|Redis[] $conns */
294 $conns = [];
295 $batches = [];
296 foreach ( $keys as $key ) {
297 $conn = $this->getConnection( $key );
298 if ( $conn ) {
299 $server = $conn->getServer();
300 $conns[$server] = $conn;
301 $batches[$server][] = $key;
302 }
303 }
304
305 $relative = $this->expiryIsRelative( $exptime );
306 $op = ( $exptime == 0 ) ? 'persist' : ( $relative ? 'expire' : 'expireAt' );
307
308 $result = true;
309 foreach ( $batches as $server => $batchKeys ) {
310 $conn = $conns[$server];
311
312 $e = null;
313 try {
314 $conn->multi( Redis::PIPELINE );
315 foreach ( $batchKeys as $key ) {
316 if ( $exptime == 0 ) {
317 $conn->persist( $key );
318 } elseif ( $relative ) {
319 $conn->expire( $key, $this->convertToRelative( $exptime ) );
320 } else {
321 $conn->expireAt( $key, $this->convertToExpiry( $exptime ) );
322 }
323 }
324 $batchResult = $conn->exec();
325 if ( $batchResult === false ) {
326 $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
327 continue;
328 }
329 $result = in_array( false, $batchResult, true ) ? false : $result;
330 } catch ( RedisException $e ) {
331 $this->handleException( $conn, $e );
332 $result = false;
333 }
334
335 $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
336 }
337
338 return $result;
339 }
340
341 public function add( $key, $value, $expiry = 0, $flags = 0 ) {
342 $conn = $this->getConnection( $key );
343 if ( !$conn ) {
344 return false;
345 }
346
347 $ttl = $this->convertToRelative( $expiry );
348 try {
349 $result = $conn->set(
350 $key,
351 $this->serialize( $value ),
352 $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ]
353 );
354 } catch ( RedisException $e ) {
355 $result = false;
356 $this->handleException( $conn, $e );
357 }
358
359 $this->logRequest( 'add', $key, $conn->getServer(), $result );
360
361 return $result;
362 }
363
364 public function incr( $key, $value = 1 ) {
365 $conn = $this->getConnection( $key );
366 if ( !$conn ) {
367 return false;
368 }
369
370 try {
371 $conn->watch( $key );
372 if ( $conn->exists( $key ) ) {
373 $conn->multi( Redis::MULTI );
374 $conn->incrBy( $key, $value );
375 $batchResult = $conn->exec();
376 if ( $batchResult === false ) {
377 $result = false;
378 } else {
379 $result = end( $batchResult );
380 }
381 } else {
382 $result = false;
383 $conn->unwatch();
384 }
385 } catch ( RedisException $e ) {
386 try {
387 $conn->unwatch(); // sanity
388 } catch ( RedisException $ex ) {
389 // already errored
390 }
391 $result = false;
392 $this->handleException( $conn, $e );
393 }
394
395 $this->logRequest( 'incr', $key, $conn->getServer(), $result );
396
397 return $result;
398 }
399
400 public function incrWithInit( $key, $exptime, $value = 1, $init = 1 ) {
401 $conn = $this->getConnection( $key );
402 if ( !$conn ) {
403 return false;
404 }
405
406 $ttl = $this->convertToRelative( $exptime );
407 $preIncrInit = $init - $value;
408 try {
409 $conn->multi( Redis::MULTI );
410 $conn->set( $key, $preIncrInit, $ttl ? [ 'nx', 'ex' => $ttl ] : [ 'nx' ] );
411 $conn->incrBy( $key, $value );
412 $batchResult = $conn->exec();
413 if ( $batchResult === false ) {
414 $result = false;
415 $this->debug( "incrWithInit request to {$conn->getServer()} failed" );
416 } else {
417 $result = end( $batchResult );
418 }
419 } catch ( RedisException $e ) {
420 $result = false;
421 $this->handleException( $conn, $e );
422 }
423
424 $this->logRequest( 'incr', $key, $conn->getServer(), $result );
425
426 return $result;
427 }
428
429 protected function doChangeTTL( $key, $exptime, $flags ) {
430 $conn = $this->getConnection( $key );
431 if ( !$conn ) {
432 return false;
433 }
434
435 $relative = $this->expiryIsRelative( $exptime );
436 try {
437 if ( $exptime == 0 ) {
438 $result = $conn->persist( $key );
439 $this->logRequest( 'persist', $key, $conn->getServer(), $result );
440 } elseif ( $relative ) {
441 $result = $conn->expire( $key, $this->convertToRelative( $exptime ) );
442 $this->logRequest( 'expire', $key, $conn->getServer(), $result );
443 } else {
444 $result = $conn->expireAt( $key, $this->convertToExpiry( $exptime ) );
445 $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
446 }
447 } catch ( RedisException $e ) {
448 $result = false;
449 $this->handleException( $conn, $e );
450 }
451
452 return $result;
453 }
454
455 /**
456 * @param string $key
457 * @return RedisConnRef|Redis|null Redis handle wrapper for the key or null on failure
458 */
459 protected function getConnection( $key ) {
460 $candidates = array_keys( $this->serverTagMap );
461
462 if ( count( $this->servers ) > 1 ) {
463 ArrayUtils::consistentHashSort( $candidates, $key, '/' );
464 if ( !$this->automaticFailover ) {
465 $candidates = array_slice( $candidates, 0, 1 );
466 }
467 }
468
469 while ( ( $tag = array_shift( $candidates ) ) !== null ) {
470 $server = $this->serverTagMap[$tag];
471 $conn = $this->redisPool->getConnection( $server, $this->logger );
472 if ( !$conn ) {
473 continue;
474 }
475
476 // If automatic failover is enabled, check that the server's link
477 // to its master (if any) is up -- but only if there are other
478 // viable candidates left to consider. Also, getMasterLinkStatus()
479 // does not work with twemproxy, though $candidates will be empty
480 // by now in such cases.
481 if ( $this->automaticFailover && $candidates ) {
482 try {
483 /** @var string[] $info */
484 $info = $conn->info();
485 if ( ( $info['master_link_status'] ?? null ) === 'down' ) {
486 // If the master cannot be reached, fail-over to the next server.
487 // If masters are in data-center A, and replica DBs in data-center B,
488 // this helps avoid the case were fail-over happens in A but not
489 // to the corresponding server in B (e.g. read/write mismatch).
490 continue;
491 }
492 } catch ( RedisException $e ) {
493 // Server is not accepting commands
494 $this->redisPool->handleError( $conn, $e );
495 continue;
496 }
497 }
498
499 return $conn;
500 }
501
502 $this->setLastError( BagOStuff::ERR_UNREACHABLE );
503
504 return null;
505 }
506
507 /**
508 * Log a fatal error
509 * @param string $msg
510 */
511 protected function logError( $msg ) {
512 $this->logger->error( "Redis error: $msg" );
513 }
514
515 /**
516 * The redis extension throws an exception in response to various read, write
517 * and protocol errors. Sometimes it also closes the connection, sometimes
518 * not. The safest response for us is to explicitly destroy the connection
519 * object and let it be reopened during the next request.
520 * @param RedisConnRef $conn
521 * @param RedisException $e
522 */
523 protected function handleException( RedisConnRef $conn, RedisException $e ) {
524 $this->setLastError( BagOStuff::ERR_UNEXPECTED );
525 $this->redisPool->handleError( $conn, $e );
526 }
527
528 /**
529 * Send information about a single request to the debug log
530 * @param string $op
531 * @param string $keys
532 * @param string $server
533 * @param Exception|bool|null $e
534 */
535 public function logRequest( $op, $keys, $server, $e = null ) {
536 $this->debug( "$op($keys) on $server: " . ( $e ? "failure" : "success" ) );
537 }
538 }
Wiklou, le Wiki du Wiklou