includes/libs/rdbms/ChronologyProtector.php

   1 <?php
   2 /**
   3  * Generator of database load balancing objects.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Database
  22  */
  23
  24 namespace Wikimedia\Rdbms;
  25
  26 use Psr\Log\LoggerAwareInterface;
  27 use Psr\Log\LoggerInterface;
  28 use Psr\Log\NullLogger;
  29 use Wikimedia\WaitConditionLoop;
  30 use BagOStuff;
  31
  32 /**
  33  * Class for ensuring a consistent ordering of events as seen by the user, despite replication.
  34  * Kind of like Hawking's [[Chronology Protection Agency]].
  35  */
  36 class ChronologyProtector implements LoggerAwareInterface {
  37         /** @var BagOStuff */
  38         protected $store;
  39         /** @var LoggerInterface */
  40         protected $logger;
  41
  42         /** @var string Storage key name */
  43         protected $key;
  44         /** @var string Hash of client parameters */
  45         protected $clientId;
  46         /** @var string[] Map of client information fields for logging */
  47         protected $clientLogInfo;
  48         /** @var int|null Expected minimum index of the last write to the position store */
  49         protected $waitForPosIndex;
  50         /** @var int Max seconds to wait on positions to appear */
  51         protected $waitForPosStoreTimeout = self::POS_STORE_WAIT_TIMEOUT;
  52         /** @var bool Whether to no-op all method calls */
  53         protected $enabled = true;
  54         /** @var bool Whether to check and wait on positions */
  55         protected $wait = true;
  56
  57         /** @var bool Whether the client data was loaded */
  58         protected $initialized = false;
  59         /** @var DBMasterPos[] Map of (DB master name => position) */
  60         protected $startupPositions = [];
  61         /** @var DBMasterPos[] Map of (DB master name => position) */
  62         protected $shutdownPositions = [];
  63         /** @var float[] Map of (DB master name => 1) */
  64         protected $shutdownTouchDBs = [];
  65
  66         /** @var int Seconds to store positions */
  67         const POSITION_TTL = 60;
  68         /** @var int Seconds to store position write index cookies (safely less than POSITION_TTL) */
  69         const POSITION_COOKIE_TTL = 10;
  70         /** @var int Max time to wait for positions to appear */
  71         const POS_STORE_WAIT_TIMEOUT = 5;
  72
  73         /**
  74          * @param BagOStuff $store
  75          * @param array $client Map of (ip: <IP>, agent: <user-agent> [, clientId: <hash>] )
  76          * @param int|null $posIndex Write counter index
  77          * @param string $secret Secret string for HMAC hashing [optional]
  78          * @since 1.27
  79          */
  80         public function __construct( BagOStuff $store, array $client, $posIndex, $secret = '' ) {
  81                 $this->store = $store;
  82                 if ( isset( $client['clientId'] ) ) {
  83                         $this->clientId = $client['clientId'];
  84                 } else {
  85                         $this->clientId = strlen( $secret )
  86                                 ? hash_hmac( 'md5', $client['ip'] . "\n" . $client['agent'], $secret )
  87                                 : md5( $client['ip'] . "\n" . $client['agent'] );
  88                 }
  89                 $this->key = $store->makeGlobalKey( __CLASS__, $this->clientId, 'v2' );
  90                 $this->waitForPosIndex = $posIndex;
  91
  92                 $this->clientLogInfo = [
  93                         'clientIP' => $client['ip'],
  94                         'clientAgent' => $client['agent'],
  95                         'clientId' => $client['clientId'] ?? null
  96                 ];
  97
  98                 $this->logger = new NullLogger();
  99         }
 100
 101         public function setLogger( LoggerInterface $logger ) {
 102                 $this->logger = $logger;
 103         }
 104
 105         /**
 106          * @return string Client ID hash
 107          * @since 1.32
 108          */
 109         public function getClientId() {
 110                 return $this->clientId;
 111         }
 112
 113         /**
 114          * @param bool $enabled Whether to no-op all method calls
 115          * @since 1.27
 116          */
 117         public function setEnabled( $enabled ) {
 118                 $this->enabled = $enabled;
 119         }
 120
 121         /**
 122          * @param bool $enabled Whether to check and wait on positions
 123          * @since 1.27
 124          */
 125         public function setWaitEnabled( $enabled ) {
 126                 $this->wait = $enabled;
 127         }
 128
 129         /**
 130          * Initialise a ILoadBalancer to give it appropriate chronology protection.
 131          *
 132          * If the stash has a previous master position recorded, this will try to
 133          * make sure that the next query to a replica DB of that master will see changes up
 134          * to that position by delaying execution. The delay may timeout and allow stale
 135          * data if no non-lagged replica DBs are available.
 136          *
 137          * @param ILoadBalancer $lb
 138          * @return void
 139          */
 140         public function initLB( ILoadBalancer $lb ) {
 141                 if ( !$this->enabled || $lb->getServerCount() <= 1 ) {
 142                         return; // non-replicated setup or disabled
 143                 }
 144
 145                 $this->initPositions();
 146
 147                 $masterName = $lb->getServerName( $lb->getWriterIndex() );
 148                 if (
 149                         isset( $this->startupPositions[$masterName] ) &&
 150                         $this->startupPositions[$masterName] instanceof DBMasterPos
 151                 ) {
 152                         $pos = $this->startupPositions[$masterName];
 153                         $this->logger->debug( __METHOD__ . ": LB for '$masterName' set to pos $pos\n" );
 154                         $lb->waitFor( $pos );
 155                 }
 156         }
 157
 158         /**
 159          * Notify the ChronologyProtector that the ILoadBalancer is about to shut
 160          * down. Saves replication positions.
 161          *
 162          * @param ILoadBalancer $lb
 163          * @return void
 164          */
 165         public function shutdownLB( ILoadBalancer $lb ) {
 166                 if ( !$this->enabled ) {
 167                         return; // not enabled
 168                 } elseif ( !$lb->hasOrMadeRecentMasterChanges( INF ) ) {
 169                         // Only save the position if writes have been done on the connection
 170                         return;
 171                 }
 172
 173                 $masterName = $lb->getServerName( $lb->getWriterIndex() );
 174                 if ( $lb->getServerCount() > 1 ) {
 175                         $pos = $lb->getMasterPos();
 176                         if ( $pos ) {
 177                                 $this->logger->debug( __METHOD__ . ": LB for '$masterName' has pos $pos\n" );
 178                                 $this->shutdownPositions[$masterName] = $pos;
 179                         }
 180                 } else {
 181                         $this->logger->debug( __METHOD__ . ": DB '$masterName' touched\n" );
 182                 }
 183                 $this->shutdownTouchDBs[$masterName] = 1;
 184         }
 185
 186         /**
 187          * Notify the ChronologyProtector that the LBFactory is done calling shutdownLB() for now.
 188          * May commit chronology data to persistent storage.
 189          *
 190          * @param callable|null $workCallback Work to do instead of waiting on syncing positions
 191          * @param string $mode One of (sync, async); whether to wait on remote datacenters
 192          * @param int|null &$cpIndex DB position key write counter; incremented on update
 193          * @return DBMasterPos[] Empty on success; returns the (db name => position) map on failure
 194          */
 195         public function shutdown( callable $workCallback = null, $mode = 'sync', &$cpIndex = null ) {
 196                 if ( !$this->enabled ) {
 197                         return [];
 198                 }
 199
 200                 $store = $this->store;
 201                 // Some callers might want to know if a user recently touched a DB.
 202                 // These writes do not need to block on all datacenters receiving them.
 203                 foreach ( $this->shutdownTouchDBs as $dbName => $unused ) {
 204                         $store->set(
 205                                 $this->getTouchedKey( $this->store, $dbName ),
 206                                 microtime( true ),
 207                                 $store::TTL_DAY
 208                         );
 209                 }
 210
 211                 if ( $this->shutdownPositions === [] ) {
 212                         return []; // nothing to save
 213                 }
 214
 215                 $this->logger->debug( __METHOD__ . ": saving master pos for " .
 216                         implode( ', ', array_keys( $this->shutdownPositions ) ) . "\n"
 217                 );
 218
 219                 // CP-protected writes should overwhelmingly go to the master datacenter, so use a
 220                 // DC-local lock to merge the values. Use a DC-local get() and a synchronous all-DC
 221                 // set(). This makes it possible for the BagOStuff class to write in parallel to all
 222                 // DCs with one RTT. The use of WRITE_SYNC avoids needing READ_LATEST for the get().
 223                 if ( $store->lock( $this->key, 3 ) ) {
 224                         if ( $workCallback ) {
 225                                 // Let the store run the work before blocking on a replication sync barrier.
 226                                 // If replication caught up while the work finished, the barrier will be fast.
 227                                 $store->addBusyCallback( $workCallback );
 228                         }
 229                         $ok = $store->set(
 230                                 $this->key,
 231                                 $this->mergePositions(
 232                                         $store->get( $this->key ),
 233                                         $this->shutdownPositions,
 234                                         $cpIndex
 235                                 ),
 236                                 self::POSITION_TTL,
 237                                 ( $mode === 'sync' ) ? $store::WRITE_SYNC : 0
 238                         );
 239                         $store->unlock( $this->key );
 240                 } else {
 241                         $ok = false;
 242                 }
 243
 244                 if ( !$ok ) {
 245                         $cpIndex = null; // nothing saved
 246                         $bouncedPositions = $this->shutdownPositions;
 247                         // Raced out too many times or stash is down
 248                         $this->logger->warning( __METHOD__ . ": failed to save master pos for " .
 249                                 implode( ', ', array_keys( $this->shutdownPositions ) ) . "\n"
 250                         );
 251                 } elseif ( $mode === 'sync' &&
 252                         $store->getQoS( $store::ATTR_SYNCWRITES ) < $store::QOS_SYNCWRITES_BE
 253                 ) {
 254                         // Positions may not be in all datacenters, force LBFactory to play it safe
 255                         $this->logger->info( __METHOD__ . ": store may not support synchronous writes." );
 256                         $bouncedPositions = $this->shutdownPositions;
 257                 } else {
 258                         $bouncedPositions = [];
 259                 }
 260
 261                 return $bouncedPositions;
 262         }
 263
 264         /**
 265          * @param string $dbName DB master name (e.g. "db1052")
 266          * @return float|bool UNIX timestamp when client last touched the DB; false if not on record
 267          * @since 1.28
 268          */
 269         public function getTouched( $dbName ) {
 270                 return $this->store->get( $this->getTouchedKey( $this->store, $dbName ) );
 271         }
 272
 273         /**
 274          * @param BagOStuff $store
 275          * @param string $dbName
 276          * @return string
 277          */
 278         private function getTouchedKey( BagOStuff $store, $dbName ) {
 279                 return $store->makeGlobalKey( __CLASS__, 'mtime', $this->clientId, $dbName );
 280         }
 281
 282         /**
 283          * Load in previous master positions for the client
 284          */
 285         protected function initPositions() {
 286                 if ( $this->initialized ) {
 287                         return;
 288                 }
 289
 290                 $this->initialized = true;
 291                 if ( $this->wait ) {
 292                         // If there is an expectation to see master positions from a certain write
 293                         // index or higher, then block until it appears, or until a timeout is reached.
 294                         // Since the write index restarts each time the key is created, it is possible that
 295                         // a lagged store has a matching key write index. However, in that case, it should
 296                         // already be expired and thus treated as non-existing, maintaining correctness.
 297                         if ( $this->waitForPosIndex > 0 ) {
 298                                 $data = null;
 299                                 $indexReached = null; // highest index reached in the position store
 300                                 $loop = new WaitConditionLoop(
 301                                         function () use ( &$data, &$indexReached ) {
 302                                                 $data = $this->store->get( $this->key );
 303                                                 if ( !is_array( $data ) ) {
 304                                                         return WaitConditionLoop::CONDITION_CONTINUE; // not found yet
 305                                                 } elseif ( !isset( $data['writeIndex'] ) ) {
 306                                                         return WaitConditionLoop::CONDITION_REACHED; // b/c
 307                                                 }
 308                                                 $indexReached = max( $data['writeIndex'], $indexReached );
 309
 310                                                 return ( $data['writeIndex'] >= $this->waitForPosIndex )
 311                                                         ? WaitConditionLoop::CONDITION_REACHED
 312                                                         : WaitConditionLoop::CONDITION_CONTINUE;
 313                                         },
 314                                         $this->waitForPosStoreTimeout
 315                                 );
 316                                 $result = $loop->invoke();
 317                                 $waitedMs = $loop->getLastWaitTime() * 1e3;
 318
 319                                 if ( $result == $loop::CONDITION_REACHED ) {
 320                                         $this->logger->debug(
 321                                                 __METHOD__ . ": expected and found position index.",
 322                                                 [
 323                                                         'cpPosIndex' => $this->waitForPosIndex,
 324                                                         'waitTimeMs' => $waitedMs
 325                                                 ] + $this->clientLogInfo
 326                                         );
 327                                 } else {
 328                                         $this->logger->warning(
 329                                                 __METHOD__ . ": expected but failed to find position index.",
 330                                                 [
 331                                                         'cpPosIndex' => $this->waitForPosIndex,
 332                                                         'indexReached' => $indexReached,
 333                                                         'waitTimeMs' => $waitedMs
 334                                                 ] + $this->clientLogInfo
 335                                         );
 336                                 }
 337                         } else {
 338                                 $data = $this->store->get( $this->key );
 339                         }
 340
 341                         $this->startupPositions = $data ? $data['positions'] : [];
 342                         $this->logger->debug( __METHOD__ . ": key is {$this->key} (read)\n" );
 343                 } else {
 344                         $this->startupPositions = [];
 345                         $this->logger->debug( __METHOD__ . ": key is {$this->key} (unread)\n" );
 346                 }
 347         }
 348
 349         /**
 350          * @param array|bool $curValue
 351          * @param DBMasterPos[] $shutdownPositions
 352          * @param int|null &$cpIndex
 353          * @return array
 354          */
 355         protected function mergePositions( $curValue, array $shutdownPositions, &$cpIndex = null ) {
 356                 /** @var DBMasterPos[] $curPositions */
 357                 $curPositions = $curValue['positions'] ?? [];
 358                 // Use the newest positions for each DB master
 359                 foreach ( $shutdownPositions as $db => $pos ) {
 360                         if (
 361                                 !isset( $curPositions[$db] ) ||
 362                                 !( $curPositions[$db] instanceof DBMasterPos ) ||
 363                                 $pos->asOfTime() > $curPositions[$db]->asOfTime()
 364                         ) {
 365                                 $curPositions[$db] = $pos;
 366                         }
 367                 }
 368
 369                 $cpIndex = $curValue['writeIndex'] ?? 0;
 370
 371                 return [
 372                         'positions' => $curPositions,
 373                         'writeIndex' => ++$cpIndex
 374                 ];
 375         }
 376 }