includes/libs/rdbms/loadmonitor/LoadMonitor.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup Database
  20  */
  21
  22 namespace Wikimedia\Rdbms;
  23
  24 use Psr\Log\LoggerInterface;
  25 use Psr\Log\NullLogger;
  26 use Wikimedia\ScopedCallback;
  27 use BagOStuff;
  28 use WANObjectCache;
  29
  30 /**
  31  * Basic DB load monitor with no external dependencies
  32  * Uses memcached to cache the replication lag for a short time
  33  *
  34  * @ingroup Database
  35  */
  36 class LoadMonitor implements ILoadMonitor {
  37         /** @var ILoadBalancer */
  38         protected $lb;
  39         /** @var BagOStuff */
  40         protected $srvCache;
  41         /** @var WANObjectCache */
  42         protected $wanCache;
  43         /** @var LoggerInterface */
  44         protected $replLogger;
  45
  46         /** @var float Moving average ratio (e.g. 0.1 for 10% weight to new weight) */
  47         private $movingAveRatio;
  48         /** @var int Amount of replication lag in seconds before warnings are logged */
  49         private $lagWarnThreshold;
  50
  51         /** @var int cache key version */
  52         const VERSION = 1;
  53         /** @var int Default 'max lag' in seconds when unspecified */
  54         const LAG_WARN_THRESHOLD = 10;
  55
  56         /**
  57          * @param ILoadBalancer $lb
  58          * @param BagOStuff $srvCache
  59          * @param WANObjectCache $wCache
  60          * @param array $options
  61          *   - movingAveRatio: moving average constant for server weight updates based on lag
  62          *   - lagWarnThreshold: how many seconds of lag trigger warnings
  63          */
  64         public function __construct(
  65                 ILoadBalancer $lb, BagOStuff $srvCache, WANObjectCache $wCache, array $options = []
  66         ) {
  67                 $this->lb = $lb;
  68                 $this->srvCache = $srvCache;
  69                 $this->wanCache = $wCache;
  70                 $this->replLogger = new NullLogger();
  71
  72                 $this->movingAveRatio = $options['movingAveRatio'] ?? 0.1;
  73                 $this->lagWarnThreshold = $options['lagWarnThreshold'] ?? self::LAG_WARN_THRESHOLD;
  74         }
  75
  76         public function setLogger( LoggerInterface $logger ) {
  77                 $this->replLogger = $logger;
  78         }
  79
  80         final public function scaleLoads( array &$weightByServer, $domain ) {
  81                 $serverIndexes = array_keys( $weightByServer );
  82                 $states = $this->getServerStates( $serverIndexes, $domain );
  83                 $newScalesByServer = $states['weightScales'];
  84                 foreach ( $weightByServer as $i => $weight ) {
  85                         if ( isset( $newScalesByServer[$i] ) ) {
  86                                 $weightByServer[$i] = $weight * $newScalesByServer[$i];
  87                         } else { // server recently added to config?
  88                                 $host = $this->lb->getServerName( $i );
  89                                 $this->replLogger->error( __METHOD__ . ": host $host not in cache" );
  90                         }
  91                 }
  92         }
  93
  94         final public function getLagTimes( array $serverIndexes, $domain ) {
  95                 return $this->getServerStates( $serverIndexes, $domain )['lagTimes'];
  96         }
  97
  98         protected function getServerStates( array $serverIndexes, $domain ) {
  99                 $writerIndex = $this->lb->getWriterIndex();
 100                 if ( count( $serverIndexes ) == 1 && reset( $serverIndexes ) == $writerIndex ) {
 101                         # Single server only, just return zero without caching
 102                         return [
 103                                 'lagTimes' => [ $writerIndex => 0 ],
 104                                 'weightScales' => [ $writerIndex => 1.0 ]
 105                         ];
 106                 }
 107
 108                 $key = $this->getCacheKey( $serverIndexes );
 109                 # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
 110                 // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
 111                 $ttl = mt_rand( 4e6, 5e6 ) / 1e6;
 112                 # Keep keys around longer as fallbacks
 113                 $staleTTL = 60;
 114
 115                 # (a) Check the local APC cache
 116                 $value = $this->srvCache->get( $key );
 117                 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
 118                         $this->replLogger->debug( __METHOD__ . ": got lag times ($key) from local cache" );
 119                         return $value; // cache hit
 120                 }
 121                 $staleValue = $value ?: false;
 122
 123                 # (b) Check the shared cache and backfill APC
 124                 $value = $this->wanCache->get( $key );
 125                 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
 126                         $this->srvCache->set( $key, $value, $staleTTL );
 127                         $this->replLogger->debug( __METHOD__ . ": got lag times ($key) from main cache" );
 128
 129                         return $value; // cache hit
 130                 }
 131                 $staleValue = $value ?: $staleValue;
 132
 133                 # (c) Cache key missing or expired; regenerate and backfill
 134                 if ( $this->srvCache->lock( $key, 0, 10 ) ) {
 135                         # Let only this process update the cache value on this server
 136                         $sCache = $this->srvCache;
 137                         /** @noinspection PhpUnusedLocalVariableInspection */
 138                         $unlocker = new ScopedCallback( function () use ( $sCache, $key ) {
 139                                 $sCache->unlock( $key );
 140                         } );
 141                 } elseif ( $staleValue ) {
 142                         # Could not acquire lock but an old cache exists, so use it
 143                         return $staleValue;
 144                 }
 145
 146                 $lagTimes = [];
 147                 $weightScales = [];
 148                 $movAveRatio = $this->movingAveRatio;
 149                 foreach ( $serverIndexes as $i ) {
 150                         if ( $i == $this->lb->getWriterIndex() ) {
 151                                 $lagTimes[$i] = 0; // master always has no lag
 152                                 $weightScales[$i] = 1.0; // nominal weight
 153                                 continue;
 154                         }
 155
 156                         # Handles with open transactions are avoided since they might be subject
 157                         # to REPEATABLE-READ snapshots, which could affect the lag estimate query.
 158                         $flags = ILoadBalancer::CONN_TRX_AUTOCOMMIT | ILoadBalancer::CONN_SILENCE_ERRORS;
 159                         $conn = $this->lb->getAnyOpenConnection( $i, $flags );
 160                         if ( $conn ) {
 161                                 $close = false; // already open
 162                         } else {
 163                                 // Get a connection to this server without triggering other server connections
 164                                 $conn = $this->lb->getServerConnection( $i, ILoadBalancer::DOMAIN_ANY, $flags );
 165                                 $close = true; // new connection
 166                         }
 167
 168                         $lastWeight = $staleValue['weightScales'][$i] ?? 1.0;
 169                         $coefficient = $this->getWeightScale( $i, $conn ?: null );
 170                         $newWeight = $movAveRatio * $coefficient + ( 1 - $movAveRatio ) * $lastWeight;
 171
 172                         // Scale from 10% to 100% of nominal weight
 173                         $weightScales[$i] = max( $newWeight, 0.10 );
 174
 175                         $host = $this->lb->getServerName( $i );
 176
 177                         if ( !$conn ) {
 178                                 $lagTimes[$i] = false;
 179                                 $this->replLogger->error(
 180                                         __METHOD__ . ": host {db_server} is unreachable",
 181                                         [ 'db_server' => $host ]
 182                                 );
 183                                 continue;
 184                         }
 185
 186                         $lagTimes[$i] = $conn->getLag();
 187                         if ( $lagTimes[$i] === false ) {
 188                                 $this->replLogger->error(
 189                                         __METHOD__ . ": host {db_server} is not replicating?",
 190                                         [ 'db_server' => $host ]
 191                                 );
 192                         } elseif ( $lagTimes[$i] > $this->lagWarnThreshold ) {
 193                                 $this->replLogger->warning(
 194                                         "Server {host} has {lag} seconds of lag (>= {maxlag})",
 195                                         [
 196                                                 'host' => $host,
 197                                                 'lag' => $lagTimes[$i],
 198                                                 'maxlag' => $this->lagWarnThreshold
 199                                         ]
 200                                 );
 201                         }
 202
 203                         if ( $close ) {
 204                                 # Close the connection to avoid sleeper connections piling up.
 205                                 # Note that the caller will pick one of these DBs and reconnect,
 206                                 # which is slightly inefficient, but this only matters for the lag
 207                                 # time cache miss cache, which is far less common that cache hits.
 208                                 $this->lb->closeConnection( $conn );
 209                         }
 210                 }
 211
 212                 # Add a timestamp key so we know when it was cached
 213                 $value = [
 214                         'lagTimes' => $lagTimes,
 215                         'weightScales' => $weightScales,
 216                         'timestamp' => microtime( true )
 217                 ];
 218                 $this->wanCache->set( $key, $value, $staleTTL );
 219                 $this->srvCache->set( $key, $value, $staleTTL );
 220                 $this->replLogger->info( __METHOD__ . ": re-calculated lag times ($key)" );
 221
 222                 return $value;
 223         }
 224
 225         /**
 226          * @param int $index Server index
 227          * @param IDatabase|null $conn Connection handle or null on connection failure
 228          * @return float
 229          */
 230         protected function getWeightScale( $index, IDatabase $conn = null ) {
 231                 return $conn ? 1.0 : 0.0;
 232         }
 233
 234         private function getCacheKey( array $serverIndexes ) {
 235                 sort( $serverIndexes );
 236                 // Lag is per-server, not per-DB, so key on the master DB name
 237                 return $this->srvCache->makeGlobalKey(
 238                         'lag-times',
 239                         self::VERSION,
 240                         $this->lb->getServerName( $this->lb->getWriterIndex() ),
 241                         implode( '-', $serverIndexes )
 242                 );
 243         }
 244 }