includes/libs/rdbms/ChronologyProtector.php

   1 <?php
   2 /**
   3  * Generator of database load balancing objects.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Database
  22  */
  23
  24 namespace Wikimedia\Rdbms;
  25
  26 use Psr\Log\LoggerAwareInterface;
  27 use Psr\Log\LoggerInterface;
  28 use Psr\Log\NullLogger;
  29 use Wikimedia\WaitConditionLoop;
  30 use BagOStuff;
  31 use DBMasterPos;
  32
  33 /**
  34  * Class for ensuring a consistent ordering of events as seen by the user, despite replication.
  35  * Kind of like Hawking's [[Chronology Protection Agency]].
  36  */
  37 class ChronologyProtector implements LoggerAwareInterface {
  38         /** @var BagOStuff */
  39         protected $store;
  40         /** @var LoggerInterface */
  41         protected $logger;
  42
  43         /** @var string Storage key name */
  44         protected $key;
  45         /** @var string Hash of client parameters */
  46         protected $clientId;
  47         /** @var float|null Minimum UNIX timestamp of 1+ expected startup positions */
  48         protected $waitForPosTime;
  49         /** @var int Max seconds to wait on positions to appear */
  50         protected $waitForPosTimeout = self::POS_WAIT_TIMEOUT;
  51         /** @var bool Whether to no-op all method calls */
  52         protected $enabled = true;
  53         /** @var bool Whether to check and wait on positions */
  54         protected $wait = true;
  55
  56         /** @var bool Whether the client data was loaded */
  57         protected $initialized = false;
  58         /** @var DBMasterPos[] Map of (DB master name => position) */
  59         protected $startupPositions = [];
  60         /** @var DBMasterPos[] Map of (DB master name => position) */
  61         protected $shutdownPositions = [];
  62         /** @var float[] Map of (DB master name => 1) */
  63         protected $shutdownTouchDBs = [];
  64
  65         /** @var integer Seconds to store positions */
  66         const POSITION_TTL = 60;
  67         /** @var integer Max time to wait for positions to appear */
  68         const POS_WAIT_TIMEOUT = 5;
  69
  70         /**
  71          * @param BagOStuff $store
  72          * @param array $client Map of (ip: <IP>, agent: <user-agent>)
  73          * @param float $posTime UNIX timestamp
  74          * @since 1.27
  75          */
  76         public function __construct( BagOStuff $store, array $client, $posTime = null ) {
  77                 $this->store = $store;
  78                 $this->clientId = md5( $client['ip'] . "\n" . $client['agent'] );
  79                 $this->key = $store->makeGlobalKey( __CLASS__, $this->clientId );
  80                 $this->waitForPosTime = $posTime;
  81                 $this->logger = new NullLogger();
  82         }
  83
  84         public function setLogger( LoggerInterface $logger ) {
  85                 $this->logger = $logger;
  86         }
  87
  88         /**
  89          * @param bool $enabled Whether to no-op all method calls
  90          * @since 1.27
  91          */
  92         public function setEnabled( $enabled ) {
  93                 $this->enabled = $enabled;
  94         }
  95
  96         /**
  97          * @param bool $enabled Whether to check and wait on positions
  98          * @since 1.27
  99          */
 100         public function setWaitEnabled( $enabled ) {
 101                 $this->wait = $enabled;
 102         }
 103
 104         /**
 105          * Initialise a ILoadBalancer to give it appropriate chronology protection.
 106          *
 107          * If the stash has a previous master position recorded, this will try to
 108          * make sure that the next query to a replica DB of that master will see changes up
 109          * to that position by delaying execution. The delay may timeout and allow stale
 110          * data if no non-lagged replica DBs are available.
 111          *
 112          * @param ILoadBalancer $lb
 113          * @return void
 114          */
 115         public function initLB( ILoadBalancer $lb ) {
 116                 if ( !$this->enabled || $lb->getServerCount() <= 1 ) {
 117                         return; // non-replicated setup or disabled
 118                 }
 119
 120                 $this->initPositions();
 121
 122                 $masterName = $lb->getServerName( $lb->getWriterIndex() );
 123                 if ( !empty( $this->startupPositions[$masterName] ) ) {
 124                         $pos = $this->startupPositions[$masterName];
 125                         $this->logger->info( __METHOD__ . ": LB for '$masterName' set to pos $pos\n" );
 126                         $lb->waitFor( $pos );
 127                 }
 128         }
 129
 130         /**
 131          * Notify the ChronologyProtector that the ILoadBalancer is about to shut
 132          * down. Saves replication positions.
 133          *
 134          * @param ILoadBalancer $lb
 135          * @return void
 136          */
 137         public function shutdownLB( ILoadBalancer $lb ) {
 138                 if ( !$this->enabled ) {
 139                         return; // not enabled
 140                 } elseif ( !$lb->hasOrMadeRecentMasterChanges( INF ) ) {
 141                         // Only save the position if writes have been done on the connection
 142                         return;
 143                 }
 144
 145                 $masterName = $lb->getServerName( $lb->getWriterIndex() );
 146                 if ( $lb->getServerCount() > 1 ) {
 147                         $pos = $lb->getMasterPos();
 148                         $this->logger->info( __METHOD__ . ": LB for '$masterName' has pos $pos\n" );
 149                         $this->shutdownPositions[$masterName] = $pos;
 150                 } else {
 151                         $this->logger->info( __METHOD__ . ": DB '$masterName' touched\n" );
 152                 }
 153                 $this->shutdownTouchDBs[$masterName] = 1;
 154         }
 155
 156         /**
 157          * Notify the ChronologyProtector that the LBFactory is done calling shutdownLB() for now.
 158          * May commit chronology data to persistent storage.
 159          *
 160          * @param callable|null $workCallback Work to do instead of waiting on syncing positions
 161          * @param string $mode One of (sync, async); whether to wait on remote datacenters
 162          * @return DBMasterPos[] Empty on success; returns the (db name => position) map on failure
 163          */
 164         public function shutdown( callable $workCallback = null, $mode = 'sync' ) {
 165                 if ( !$this->enabled ) {
 166                         return [];
 167                 }
 168
 169                 $store = $this->store;
 170                 // Some callers might want to know if a user recently touched a DB.
 171                 // These writes do not need to block on all datacenters receiving them.
 172                 foreach ( $this->shutdownTouchDBs as $dbName => $unused ) {
 173                         $store->set(
 174                                 $this->getTouchedKey( $this->store, $dbName ),
 175                                 microtime( true ),
 176                                 $store::TTL_DAY
 177                         );
 178                 }
 179
 180                 if ( !count( $this->shutdownPositions ) ) {
 181                         return []; // nothing to save
 182                 }
 183
 184                 $this->logger->info( __METHOD__ . ": saving master pos for " .
 185                         implode( ', ', array_keys( $this->shutdownPositions ) ) . "\n"
 186                 );
 187
 188                 // CP-protected writes should overwhemingly go to the master datacenter, so get DC-local
 189                 // lock to merge the values. Use a DC-local get() and a synchronous all-DC set(). This
 190                 // makes it possible for the BagOStuff class to write in parallel to all DCs with one RTT.
 191                 if ( $store->lock( $this->key, 3 ) ) {
 192                         if ( $workCallback ) {
 193                                 // Let the store run the work before blocking on a replication sync barrier. By the
 194                                 // time it's done with the work, the barrier should be fast if replication caught up.
 195                                 $store->addBusyCallback( $workCallback );
 196                         }
 197                         $ok = $store->set(
 198                                 $this->key,
 199                                 self::mergePositions( $store->get( $this->key ), $this->shutdownPositions ),
 200                                 self::POSITION_TTL,
 201                                 ( $mode === 'sync' ) ? $store::WRITE_SYNC : 0
 202                         );
 203                         $store->unlock( $this->key );
 204                 } else {
 205                         $ok = false;
 206                 }
 207
 208                 if ( !$ok ) {
 209                         $bouncedPositions = $this->shutdownPositions;
 210                         // Raced out too many times or stash is down
 211                         $this->logger->warning( __METHOD__ . ": failed to save master pos for " .
 212                                 implode( ', ', array_keys( $this->shutdownPositions ) ) . "\n"
 213                         );
 214                 } elseif ( $mode === 'sync' &&
 215                         $store->getQoS( $store::ATTR_SYNCWRITES ) < $store::QOS_SYNCWRITES_BE
 216                 ) {
 217                         // Positions may not be in all datacenters, force LBFactory to play it safe
 218                         $this->logger->info( __METHOD__ . ": store may not support synchronous writes." );
 219                         $bouncedPositions = $this->shutdownPositions;
 220                 } else {
 221                         $bouncedPositions = [];
 222                 }
 223
 224                 return $bouncedPositions;
 225         }
 226
 227         /**
 228          * @param string $dbName DB master name (e.g. "db1052")
 229          * @return float|bool UNIX timestamp when client last touched the DB; false if not on record
 230          * @since 1.28
 231          */
 232         public function getTouched( $dbName ) {
 233                 return $this->store->get( $this->getTouchedKey( $this->store, $dbName ) );
 234         }
 235
 236         /**
 237          * @param BagOStuff $store
 238          * @param string $dbName
 239          * @return string
 240          */
 241         private function getTouchedKey( BagOStuff $store, $dbName ) {
 242                 return $store->makeGlobalKey( __CLASS__, 'mtime', $this->clientId, $dbName );
 243         }
 244
 245         /**
 246          * Load in previous master positions for the client
 247          */
 248         protected function initPositions() {
 249                 if ( $this->initialized ) {
 250                         return;
 251                 }
 252
 253                 $this->initialized = true;
 254                 if ( $this->wait ) {
 255                         // If there is an expectation to see master positions with a certain min
 256                         // timestamp, then block until they appear, or until a timeout is reached.
 257                         if ( $this->waitForPosTime > 0.0 ) {
 258                                 $data = null;
 259                                 $loop = new WaitConditionLoop(
 260                                         function () use ( &$data ) {
 261                                                 $data = $this->store->get( $this->key );
 262
 263                                                 return ( self::minPosTime( $data ) >= $this->waitForPosTime )
 264                                                         ? WaitConditionLoop::CONDITION_REACHED
 265                                                         : WaitConditionLoop::CONDITION_CONTINUE;
 266                                         },
 267                                         $this->waitForPosTimeout
 268                                 );
 269                                 $result = $loop->invoke();
 270                                 $waitedMs = $loop->getLastWaitTime() * 1e3;
 271
 272                                 if ( $result == $loop::CONDITION_REACHED ) {
 273                                         $msg = "expected and found pos time {$this->waitForPosTime} ({$waitedMs}ms)";
 274                                         $this->logger->debug( $msg );
 275                                 } else {
 276                                         $msg = "expected but missed pos time {$this->waitForPosTime} ({$waitedMs}ms)";
 277                                         $this->logger->info( $msg );
 278                                 }
 279                         } else {
 280                                 $data = $this->store->get( $this->key );
 281                         }
 282
 283                         $this->startupPositions = $data ? $data['positions'] : [];
 284                         $this->logger->info( __METHOD__ . ": key is {$this->key} (read)\n" );
 285                 } else {
 286                         $this->startupPositions = [];
 287                         $this->logger->info( __METHOD__ . ": key is {$this->key} (unread)\n" );
 288                 }
 289         }
 290
 291         /**
 292          * @param array|bool $data
 293          * @return float|null
 294          */
 295         private static function minPosTime( $data ) {
 296                 if ( !isset( $data['positions'] ) ) {
 297                         return null;
 298                 }
 299
 300                 $min = null;
 301                 foreach ( $data['positions'] as $pos ) {
 302                         /** @var DBMasterPos $pos */
 303                         $min = $min ? min( $pos->asOfTime(), $min ) : $pos->asOfTime();
 304                 }
 305
 306                 return $min;
 307         }
 308
 309         /**
 310          * @param array|bool $curValue
 311          * @param DBMasterPos[] $shutdownPositions
 312          * @return array
 313          */
 314         private static function mergePositions( $curValue, array $shutdownPositions ) {
 315                 /** @var $curPositions DBMasterPos[] */
 316                 if ( $curValue === false ) {
 317                         $curPositions = $shutdownPositions;
 318                 } else {
 319                         $curPositions = $curValue['positions'];
 320                         // Use the newest positions for each DB master
 321                         foreach ( $shutdownPositions as $db => $pos ) {
 322                                 if ( !isset( $curPositions[$db] )
 323                                         || $pos->asOfTime() > $curPositions[$db]->asOfTime()
 324                                 ) {
 325                                         $curPositions[$db] = $pos;
 326                                 }
 327                         }
 328                 }
 329
 330                 return [ 'positions' => $curPositions ];
 331         }
 332 }