includes/libs/rdbms/lbfactory/LBFactory.php

   1 <?php
   2 /**
   3  * Generator and manager of database load balancing objects
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Database
  22  */
  23
  24 namespace Wikimedia\Rdbms;
  25
  26 use Psr\Log\LoggerInterface;
  27 use Wikimedia\ScopedCallback;
  28 use BagOStuff;
  29 use EmptyBagOStuff;
  30 use WANObjectCache;
  31 use Exception;
  32 use RuntimeException;
  33
  34 /**
  35  * An interface for generating database load balancers
  36  * @ingroup Database
  37  */
  38 abstract class LBFactory implements ILBFactory {
  39         /** @var ChronologyProtector */
  40         protected $chronProt;
  41         /** @var object|string Class name or object With profileIn/profileOut methods */
  42         protected $profiler;
  43         /** @var TransactionProfiler */
  44         protected $trxProfiler;
  45         /** @var LoggerInterface */
  46         protected $replLogger;
  47         /** @var LoggerInterface */
  48         protected $connLogger;
  49         /** @var LoggerInterface */
  50         protected $queryLogger;
  51         /** @var LoggerInterface */
  52         protected $perfLogger;
  53         /** @var callable Error logger */
  54         protected $errorLogger;
  55         /** @var BagOStuff */
  56         protected $srvCache;
  57         /** @var BagOStuff */
  58         protected $memStash;
  59         /** @var WANObjectCache */
  60         protected $wanCache;
  61
  62         /** @var DatabaseDomain Local domain */
  63         protected $localDomain;
  64         /** @var string Local hostname of the app server */
  65         protected $hostname;
  66         /** @var array Web request information about the client */
  67         protected $requestInfo;
  68
  69         /** @var mixed */
  70         protected $ticket;
  71         /** @var string|bool String if a requested DBO_TRX transaction round is active */
  72         protected $trxRoundId = false;
  73         /** @var string|bool Reason all LBs are read-only or false if not */
  74         protected $readOnlyReason = false;
  75         /** @var callable[] */
  76         protected $replicationWaitCallbacks = [];
  77
  78         /** @var bool Whether this PHP instance is for a CLI script */
  79         protected $cliMode;
  80         /** @var string Agent name for query profiling */
  81         protected $agent;
  82
  83         private static $loggerFields =
  84                 [ 'replLogger', 'connLogger', 'queryLogger', 'perfLogger' ];
  85
  86         public function __construct( array $conf ) {
  87                 $this->localDomain = isset( $conf['localDomain'] )
  88                         ? DatabaseDomain::newFromId( $conf['localDomain'] )
  89                         : DatabaseDomain::newUnspecified();
  90
  91                 if ( isset( $conf['readOnlyReason'] ) && is_string( $conf['readOnlyReason'] ) ) {
  92                         $this->readOnlyReason = $conf['readOnlyReason'];
  93                 }
  94
  95                 $this->srvCache = isset( $conf['srvCache'] ) ? $conf['srvCache'] : new EmptyBagOStuff();
  96                 $this->memStash = isset( $conf['memStash'] ) ? $conf['memStash'] : new EmptyBagOStuff();
  97                 $this->wanCache = isset( $conf['wanCache'] )
  98                         ? $conf['wanCache']
  99                         : WANObjectCache::newEmpty();
 100
 101                 foreach ( self::$loggerFields as $key ) {
 102                         $this->$key = isset( $conf[$key] ) ? $conf[$key] : new \Psr\Log\NullLogger();
 103                 }
 104                 $this->errorLogger = isset( $conf['errorLogger'] )
 105                         ? $conf['errorLogger']
 106                         : function ( Exception $e ) {
 107                                 trigger_error( E_USER_WARNING, get_class( $e ) . ': ' . $e->getMessage() );
 108                         };
 109
 110                 $this->profiler = isset( $conf['profiler'] ) ? $conf['profiler'] : null;
 111                 $this->trxProfiler = isset( $conf['trxProfiler'] )
 112                         ? $conf['trxProfiler']
 113                         : new TransactionProfiler();
 114
 115                 $this->requestInfo = [
 116                         'IPAddress' => isset( $_SERVER[ 'REMOTE_ADDR' ] ) ? $_SERVER[ 'REMOTE_ADDR' ] : '',
 117                         'UserAgent' => isset( $_SERVER['HTTP_USER_AGENT'] ) ? $_SERVER['HTTP_USER_AGENT'] : '',
 118                         'ChronologyProtection' => 'true',
 119                         'ChronologyPositionIndex' => isset( $_GET['cpPosIndex'] ) ? $_GET['cpPosIndex'] : null
 120                 ];
 121
 122                 $this->cliMode = isset( $conf['cliMode'] ) ? $conf['cliMode'] : PHP_SAPI === 'cli';
 123                 $this->hostname = isset( $conf['hostname'] ) ? $conf['hostname'] : gethostname();
 124                 $this->agent = isset( $conf['agent'] ) ? $conf['agent'] : '';
 125
 126                 $this->ticket = mt_rand();
 127         }
 128
 129         public function destroy() {
 130                 $this->shutdown( self::SHUTDOWN_NO_CHRONPROT );
 131                 $this->forEachLBCallMethod( 'disable' );
 132         }
 133
 134         public function shutdown(
 135                 $mode = self::SHUTDOWN_CHRONPROT_SYNC, callable $workCallback = null, &$cpIndex = null
 136         ) {
 137                 $chronProt = $this->getChronologyProtector();
 138                 if ( $mode === self::SHUTDOWN_CHRONPROT_SYNC ) {
 139                         $this->shutdownChronologyProtector( $chronProt, $workCallback, 'sync', $cpIndex );
 140                 } elseif ( $mode === self::SHUTDOWN_CHRONPROT_ASYNC ) {
 141                         $this->shutdownChronologyProtector( $chronProt, null, 'async', $cpIndex );
 142                 }
 143
 144                 $this->commitMasterChanges( __METHOD__ ); // sanity
 145         }
 146
 147         /**
 148          * @see ILBFactory::newMainLB()
 149          * @param bool $domain
 150          * @return LoadBalancer
 151          */
 152         abstract public function newMainLB( $domain = false );
 153
 154         /**
 155          * @see ILBFactory::getMainLB()
 156          * @param bool $domain
 157          * @return LoadBalancer
 158          */
 159         abstract public function getMainLB( $domain = false );
 160
 161         /**
 162          * @see ILBFactory::newExternalLB()
 163          * @param string $cluster
 164          * @return LoadBalancer
 165          */
 166         abstract public function newExternalLB( $cluster );
 167
 168         /**
 169          * @see ILBFactory::getExternalLB()
 170          * @param string $cluster
 171          * @return LoadBalancer
 172          */
 173         abstract public function getExternalLB( $cluster );
 174
 175         /**
 176          * Call a method of each tracked load balancer
 177          *
 178          * @param string $methodName
 179          * @param array $args
 180          */
 181         protected function forEachLBCallMethod( $methodName, array $args = [] ) {
 182                 $this->forEachLB(
 183                         function ( ILoadBalancer $loadBalancer, $methodName, array $args ) {
 184                                 call_user_func_array( [ $loadBalancer, $methodName ], $args );
 185                         },
 186                         [ $methodName, $args ]
 187                 );
 188         }
 189
 190         public function flushReplicaSnapshots( $fname = __METHOD__ ) {
 191                 $this->forEachLBCallMethod( 'flushReplicaSnapshots', [ $fname ] );
 192         }
 193
 194         public function commitAll( $fname = __METHOD__, array $options = [] ) {
 195                 $this->commitMasterChanges( $fname, $options );
 196                 $this->forEachLBCallMethod( 'commitAll', [ $fname ] );
 197         }
 198
 199         public function beginMasterChanges( $fname = __METHOD__ ) {
 200                 if ( $this->trxRoundId !== false ) {
 201                         throw new DBTransactionError(
 202                                 null,
 203                                 "$fname: transaction round '{$this->trxRoundId}' already started."
 204                         );
 205                 }
 206                 $this->trxRoundId = $fname;
 207                 // Set DBO_TRX flags on all appropriate DBs
 208                 $this->forEachLBCallMethod( 'beginMasterChanges', [ $fname ] );
 209         }
 210
 211         public function commitMasterChanges( $fname = __METHOD__, array $options = [] ) {
 212                 if ( $this->trxRoundId !== false && $this->trxRoundId !== $fname ) {
 213                         throw new DBTransactionError(
 214                                 null,
 215                                 "$fname: transaction round '{$this->trxRoundId}' still running."
 216                         );
 217                 }
 218                 /** @noinspection PhpUnusedLocalVariableInspection */
 219                 $scope = $this->getScopedPHPBehaviorForCommit(); // try to ignore client aborts
 220                 // Run pre-commit callbacks and suppress post-commit callbacks, aborting on failure
 221                 $this->forEachLBCallMethod( 'finalizeMasterChanges' );
 222                 $this->trxRoundId = false;
 223                 // Perform pre-commit checks, aborting on failure
 224                 $this->forEachLBCallMethod( 'approveMasterChanges', [ $options ] );
 225                 // Log the DBs and methods involved in multi-DB transactions
 226                 $this->logIfMultiDbTransaction();
 227                 // Actually perform the commit on all master DB connections and revert DBO_TRX
 228                 $this->forEachLBCallMethod( 'commitMasterChanges', [ $fname ] );
 229                 // Run all post-commit callbacks
 230                 /** @var Exception $e */
 231                 $e = null; // first callback exception
 232                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( &$e ) {
 233                         $ex = $lb->runMasterPostTrxCallbacks( IDatabase::TRIGGER_COMMIT );
 234                         $e = $e ?: $ex;
 235                 } );
 236                 // Commit any dangling DBO_TRX transactions from callbacks on one DB to another DB
 237                 $this->forEachLBCallMethod( 'commitMasterChanges', [ $fname ] );
 238                 // Throw any last post-commit callback error
 239                 if ( $e instanceof Exception ) {
 240                         throw $e;
 241                 }
 242         }
 243
 244         public function rollbackMasterChanges( $fname = __METHOD__ ) {
 245                 $this->trxRoundId = false;
 246                 $this->forEachLBCallMethod( 'suppressTransactionEndCallbacks' );
 247                 $this->forEachLBCallMethod( 'rollbackMasterChanges', [ $fname ] );
 248                 // Run all post-rollback callbacks
 249                 $this->forEachLB( function ( ILoadBalancer $lb ) {
 250                         $lb->runMasterPostTrxCallbacks( IDatabase::TRIGGER_ROLLBACK );
 251                 } );
 252         }
 253
 254         public function hasTransactionRound() {
 255                 return ( $this->trxRoundId !== false );
 256         }
 257
 258         /**
 259          * Log query info if multi DB transactions are going to be committed now
 260          */
 261         private function logIfMultiDbTransaction() {
 262                 $callersByDB = [];
 263                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( &$callersByDB ) {
 264                         $masterName = $lb->getServerName( $lb->getWriterIndex() );
 265                         $callers = $lb->pendingMasterChangeCallers();
 266                         if ( $callers ) {
 267                                 $callersByDB[$masterName] = $callers;
 268                         }
 269                 } );
 270
 271                 if ( count( $callersByDB ) >= 2 ) {
 272                         $dbs = implode( ', ', array_keys( $callersByDB ) );
 273                         $msg = "Multi-DB transaction [{$dbs}]:\n";
 274                         foreach ( $callersByDB as $db => $callers ) {
 275                                 $msg .= "$db: " . implode( '; ', $callers ) . "\n";
 276                         }
 277                         $this->queryLogger->info( $msg );
 278                 }
 279         }
 280
 281         public function hasMasterChanges() {
 282                 $ret = false;
 283                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( &$ret ) {
 284                         $ret = $ret || $lb->hasMasterChanges();
 285                 } );
 286
 287                 return $ret;
 288         }
 289
 290         public function laggedReplicaUsed() {
 291                 $ret = false;
 292                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( &$ret ) {
 293                         $ret = $ret || $lb->laggedReplicaUsed();
 294                 } );
 295
 296                 return $ret;
 297         }
 298
 299         public function hasOrMadeRecentMasterChanges( $age = null ) {
 300                 $ret = false;
 301                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( $age, &$ret ) {
 302                         $ret = $ret || $lb->hasOrMadeRecentMasterChanges( $age );
 303                 } );
 304                 return $ret;
 305         }
 306
 307         public function waitForReplication( array $opts = [] ) {
 308                 $opts += [
 309                         'domain' => false,
 310                         'cluster' => false,
 311                         'timeout' => 60,
 312                         'ifWritesSince' => null
 313                 ];
 314
 315                 if ( $opts['domain'] === false && isset( $opts['wiki'] ) ) {
 316                         $opts['domain'] = $opts['wiki']; // b/c
 317                 }
 318
 319                 // Figure out which clusters need to be checked
 320                 /** @var ILoadBalancer[] $lbs */
 321                 $lbs = [];
 322                 if ( $opts['cluster'] !== false ) {
 323                         $lbs[] = $this->getExternalLB( $opts['cluster'] );
 324                 } elseif ( $opts['domain'] !== false ) {
 325                         $lbs[] = $this->getMainLB( $opts['domain'] );
 326                 } else {
 327                         $this->forEachLB( function ( ILoadBalancer $lb ) use ( &$lbs ) {
 328                                 $lbs[] = $lb;
 329                         } );
 330                         if ( !$lbs ) {
 331                                 return; // nothing actually used
 332                         }
 333                 }
 334
 335                 // Get all the master positions of applicable DBs right now.
 336                 // This can be faster since waiting on one cluster reduces the
 337                 // time needed to wait on the next clusters.
 338                 $masterPositions = array_fill( 0, count( $lbs ), false );
 339                 foreach ( $lbs as $i => $lb ) {
 340                         if ( $lb->getServerCount() <= 1 ) {
 341                                 // T29975 - Don't try to wait for replica DBs if there are none
 342                                 // Prevents permission error when getting master position
 343                                 continue;
 344                         } elseif ( $opts['ifWritesSince']
 345                                 && $lb->lastMasterChangeTimestamp() < $opts['ifWritesSince']
 346                         ) {
 347                                 continue; // no writes since the last wait
 348                         }
 349                         $masterPositions[$i] = $lb->getMasterPos();
 350                 }
 351
 352                 // Run any listener callbacks *after* getting the DB positions. The more
 353                 // time spent in the callbacks, the less time is spent in waitForAll().
 354                 foreach ( $this->replicationWaitCallbacks as $callback ) {
 355                         $callback();
 356                 }
 357
 358                 $failed = [];
 359                 foreach ( $lbs as $i => $lb ) {
 360                         if ( $masterPositions[$i] ) {
 361                                 // The RDBMS may not support getMasterPos()
 362                                 if ( !$lb->waitForAll( $masterPositions[$i], $opts['timeout'] ) ) {
 363                                         $failed[] = $lb->getServerName( $lb->getWriterIndex() );
 364                                 }
 365                         }
 366                 }
 367
 368                 if ( $failed ) {
 369                         throw new DBReplicationWaitError(
 370                                 null,
 371                                 "Could not wait for replica DBs to catch up to " .
 372                                 implode( ', ', $failed )
 373                         );
 374                 }
 375         }
 376
 377         public function setWaitForReplicationListener( $name, callable $callback = null ) {
 378                 if ( $callback ) {
 379                         $this->replicationWaitCallbacks[$name] = $callback;
 380                 } else {
 381                         unset( $this->replicationWaitCallbacks[$name] );
 382                 }
 383         }
 384
 385         public function getEmptyTransactionTicket( $fname ) {
 386                 if ( $this->hasMasterChanges() ) {
 387                         $this->queryLogger->error( __METHOD__ . ": $fname does not have outer scope.\n" .
 388                                 ( new RuntimeException() )->getTraceAsString() );
 389
 390                         return null;
 391                 }
 392
 393                 return $this->ticket;
 394         }
 395
 396         public function commitAndWaitForReplication( $fname, $ticket, array $opts = [] ) {
 397                 if ( $ticket !== $this->ticket ) {
 398                         $this->perfLogger->error( __METHOD__ . ": $fname does not have outer scope.\n" .
 399                                 ( new RuntimeException() )->getTraceAsString() );
 400
 401                         return;
 402                 }
 403
 404                 // The transaction owner and any caller with the empty transaction ticket can commit
 405                 // so that getEmptyTransactionTicket() callers don't risk seeing DBTransactionError.
 406                 if ( $this->trxRoundId !== false && $fname !== $this->trxRoundId ) {
 407                         $this->queryLogger->info( "$fname: committing on behalf of {$this->trxRoundId}." );
 408                         $fnameEffective = $this->trxRoundId;
 409                 } else {
 410                         $fnameEffective = $fname;
 411                 }
 412
 413                 $this->commitMasterChanges( $fnameEffective );
 414                 $this->waitForReplication( $opts );
 415                 // If a nested caller committed on behalf of $fname, start another empty $fname
 416                 // transaction, leaving the caller with the same empty transaction state as before.
 417                 if ( $fnameEffective !== $fname ) {
 418                         $this->beginMasterChanges( $fnameEffective );
 419                 }
 420         }
 421
 422         public function getChronologyProtectorTouched( $dbName ) {
 423                 return $this->getChronologyProtector()->getTouched( $dbName );
 424         }
 425
 426         public function disableChronologyProtection() {
 427                 $this->getChronologyProtector()->setEnabled( false );
 428         }
 429
 430         /**
 431          * @return ChronologyProtector
 432          */
 433         protected function getChronologyProtector() {
 434                 if ( $this->chronProt ) {
 435                         return $this->chronProt;
 436                 }
 437
 438                 $this->chronProt = new ChronologyProtector(
 439                         $this->memStash,
 440                         [
 441                                 'ip' => $this->requestInfo['IPAddress'],
 442                                 'agent' => $this->requestInfo['UserAgent'],
 443                         ],
 444                         $this->requestInfo['ChronologyPositionIndex']
 445                 );
 446                 $this->chronProt->setLogger( $this->replLogger );
 447
 448                 if ( $this->cliMode ) {
 449                         $this->chronProt->setEnabled( false );
 450                 } elseif ( $this->requestInfo['ChronologyProtection'] === 'false' ) {
 451                         // Request opted out of using position wait logic. This is useful for requests
 452                         // done by the job queue or background ETL that do not have a meaningful session.
 453                         $this->chronProt->setWaitEnabled( false );
 454                 }
 455
 456                 $this->replLogger->debug( __METHOD__ . ': using request info ' .
 457                         json_encode( $this->requestInfo, JSON_PRETTY_PRINT ) );
 458
 459                 return $this->chronProt;
 460         }
 461
 462         /**
 463          * Get and record all of the staged DB positions into persistent memory storage
 464          *
 465          * @param ChronologyProtector $cp
 466          * @param callable|null $workCallback Work to do instead of waiting on syncing positions
 467          * @param string $mode One of (sync, async); whether to wait on remote datacenters
 468          * @param int|null &$cpIndex DB position key write counter; incremented on update
 469          */
 470         protected function shutdownChronologyProtector(
 471                 ChronologyProtector $cp, $workCallback, $mode, &$cpIndex = null
 472         ) {
 473                 // Record all the master positions needed
 474                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( $cp ) {
 475                         $cp->shutdownLB( $lb );
 476                 } );
 477                 // Write them to the persistent stash. Try to do something useful by running $work
 478                 // while ChronologyProtector waits for the stash write to replicate to all DCs.
 479                 $unsavedPositions = $cp->shutdown( $workCallback, $mode, $cpIndex );
 480                 if ( $unsavedPositions && $workCallback ) {
 481                         // Invoke callback in case it did not cache the result yet
 482                         $workCallback(); // work now to block for less time in waitForAll()
 483                 }
 484                 // If the positions failed to write to the stash, at least wait on local datacenter
 485                 // replica DBs to catch up before responding. Even if there are several DCs, this increases
 486                 // the chance that the user will see their own changes immediately afterwards. As long
 487                 // as the sticky DC cookie applies (same domain), this is not even an issue.
 488                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( $unsavedPositions ) {
 489                         $masterName = $lb->getServerName( $lb->getWriterIndex() );
 490                         if ( isset( $unsavedPositions[$masterName] ) ) {
 491                                 $lb->waitForAll( $unsavedPositions[$masterName] );
 492                         }
 493                 } );
 494         }
 495
 496         /**
 497          * Base parameters to LoadBalancer::__construct()
 498          * @return array
 499          */
 500         final protected function baseLoadBalancerParams() {
 501                 return [
 502                         'localDomain' => $this->localDomain,
 503                         'readOnlyReason' => $this->readOnlyReason,
 504                         'srvCache' => $this->srvCache,
 505                         'wanCache' => $this->wanCache,
 506                         'profiler' => $this->profiler,
 507                         'trxProfiler' => $this->trxProfiler,
 508                         'queryLogger' => $this->queryLogger,
 509                         'connLogger' => $this->connLogger,
 510                         'replLogger' => $this->replLogger,
 511                         'errorLogger' => $this->errorLogger,
 512                         'hostname' => $this->hostname,
 513                         'cliMode' => $this->cliMode,
 514                         'agent' => $this->agent,
 515                         'chronologyProtector' => $this->getChronologyProtector()
 516                 ];
 517         }
 518
 519         /**
 520          * @param ILoadBalancer $lb
 521          */
 522         protected function initLoadBalancer( ILoadBalancer $lb ) {
 523                 if ( $this->trxRoundId !== false ) {
 524                         $lb->beginMasterChanges( $this->trxRoundId ); // set DBO_TRX
 525                 }
 526         }
 527
 528         public function setDomainPrefix( $prefix ) {
 529                 $this->localDomain = new DatabaseDomain(
 530                         $this->localDomain->getDatabase(),
 531                         null,
 532                         $prefix
 533                 );
 534
 535                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( $prefix ) {
 536                         $lb->setDomainPrefix( $prefix );
 537                 } );
 538         }
 539
 540         public function closeAll() {
 541                 $this->forEachLBCallMethod( 'closeAll', [] );
 542         }
 543
 544         public function setAgentName( $agent ) {
 545                 $this->agent = $agent;
 546         }
 547
 548         public function appendShutdownCPIndexAsQuery( $url, $index ) {
 549                 $usedCluster = 0;
 550                 $this->forEachLB( function ( ILoadBalancer $lb ) use ( &$usedCluster ) {
 551                         $usedCluster |= ( $lb->getServerCount() > 1 );
 552                 } );
 553
 554                 if ( !$usedCluster ) {
 555                         return $url; // no master/replica clusters touched
 556                 }
 557
 558                 return strpos( $url, '?' ) === false ? "$url?cpPosIndex=$index" : "$url&cpPosIndex=$index";
 559         }
 560
 561         public function setRequestInfo( array $info ) {
 562                 $this->requestInfo = $info + $this->requestInfo;
 563         }
 564
 565         /**
 566          * Make PHP ignore user aborts/disconnects until the returned
 567          * value leaves scope. This returns null and does nothing in CLI mode.
 568          *
 569          * @return ScopedCallback|null
 570          */
 571         final protected function getScopedPHPBehaviorForCommit() {
 572                 if ( PHP_SAPI != 'cli' ) { // https://bugs.php.net/bug.php?id=47540
 573                         $old = ignore_user_abort( true ); // avoid half-finished operations
 574                         return new ScopedCallback( function () use ( $old ) {
 575                                 ignore_user_abort( $old );
 576                         } );
 577                 }
 578
 579                 return null;
 580         }
 581
 582         function __destruct() {
 583                 $this->destroy();
 584         }
 585 }
 586
 587 class_alias( LBFactory::class, 'LBFactory' );