includes/db/loadbalancer/LBFactoryMulti.php

   1 <?php
   2 /**
   3  * Advanced generator of database load balancing objects for wiki farms.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Database
  22  */
  23
  24 /**
  25  * A multi-wiki, multi-master factory for Wikimedia and similar installations.
  26  * Ignores the old configuration globals
  27  *
  28  * Configuration:
  29  *     sectionsByDB                A map of database names to section names.
  30  *
  31  *     sectionLoads                A 2-d map. For each section, gives a map of server names to
  32  *                                 load ratios. For example:
  33  *                                 array(
  34  *                                     'section1' => array(
  35  *                                         'db1' => 100,
  36  *                                         'db2' => 100
  37  *                                     )
  38  *                                 )
  39  *
  40  *     serverTemplate              A server info associative array as documented for $wgDBservers.
  41  *                                 The host, hostName and load entries will be overridden.
  42  *
  43  *     groupLoadsBySection         A 3-d map giving server load ratios for each section and group.
  44  *                                 For example:
  45  *                                 array(
  46  *                                     'section1' => array(
  47  *                                         'group1' => array(
  48  *                                             'db1' => 100,
  49  *                                             'db2' => 100
  50  *                                         )
  51  *                                     )
  52  *                                 )
  53  *
  54  *     groupLoadsByDB              A 3-d map giving server load ratios by DB name.
  55  *
  56  *     hostsByName                 A map of hostname to IP address.
  57  *
  58  *     externalLoads               A map of external storage cluster name to server load map.
  59  *
  60  *     externalTemplateOverrides   A set of server info keys overriding serverTemplate for external
  61  *                                 storage.
  62  *
  63  *     templateOverridesByServer   A 2-d map overriding serverTemplate and
  64  *                                 externalTemplateOverrides on a server-by-server basis. Applies
  65  *                                 to both core and external storage.
  66  *
  67  *     templateOverridesByCluster  A 2-d map overriding the server info by external storage cluster.
  68  *
  69  *     masterTemplateOverrides     An override array for all master servers.
  70  *
  71  *     loadMonitorClass            Name of the LoadMonitor class to always use.
  72  *
  73  *     readOnlyBySection           A map of section name to read-only message.
  74  *                                 Missing or false for read/write.
  75  *
  76  * @ingroup Database
  77  */
  78 class LBFactoryMulti extends LBFactory {
  79         /** @var array A map of database names to section names */
  80         private $sectionsByDB;
  81
  82         /**
  83          * @var array A 2-d map. For each section, gives a map of server names to
  84          * load ratios
  85          */
  86         private $sectionLoads;
  87
  88         /**
  89          * @var array A server info associative array as documented for
  90          * $wgDBservers. The host, hostName and load entries will be
  91          * overridden
  92          */
  93         private $serverTemplate;
  94
  95         // Optional settings
  96
  97         /** @var array A 3-d map giving server load ratios for each section and group */
  98         private $groupLoadsBySection = [];
  99
 100         /** @var array A 3-d map giving server load ratios by DB name */
 101         private $groupLoadsByDB = [];
 102
 103         /** @var array A map of hostname to IP address */
 104         private $hostsByName = [];
 105
 106         /** @var array A map of external storage cluster name to server load map */
 107         private $externalLoads = [];
 108
 109         /**
 110          * @var array A set of server info keys overriding serverTemplate for
 111          * external storage
 112          */
 113         private $externalTemplateOverrides;
 114
 115         /**
 116          * @var array A 2-d map overriding serverTemplate and
 117          * externalTemplateOverrides on a server-by-server basis. Applies to both
 118          * core and external storage
 119          */
 120         private $templateOverridesByServer;
 121
 122         /** @var array A 2-d map overriding the server info by external storage cluster */
 123         private $templateOverridesByCluster;
 124
 125         /** @var array An override array for all master servers */
 126         private $masterTemplateOverrides;
 127
 128         /**
 129          * @var array|bool A map of section name to read-only message. Missing or
 130          * false for read/write
 131          */
 132         private $readOnlyBySection = [];
 133
 134         // Other stuff
 135
 136         /** @var array Load balancer factory configuration */
 137         private $conf;
 138
 139         /** @var LoadBalancer[] */
 140         private $mainLBs = [];
 141
 142         /** @var LoadBalancer[] */
 143         private $extLBs = [];
 144
 145         /** @var string */
 146         private $loadMonitorClass;
 147
 148         /** @var string */
 149         private $lastWiki;
 150
 151         /** @var string */
 152         private $lastSection;
 153
 154         /**
 155          * @param array $conf
 156          * @throws MWException
 157          */
 158         public function __construct( array $conf ) {
 159                 parent::__construct( $conf );
 160
 161                 $this->conf = $conf;
 162                 $required = [ 'sectionsByDB', 'sectionLoads', 'serverTemplate' ];
 163                 $optional = [ 'groupLoadsBySection', 'groupLoadsByDB', 'hostsByName',
 164                         'externalLoads', 'externalTemplateOverrides', 'templateOverridesByServer',
 165                         'templateOverridesByCluster', 'masterTemplateOverrides',
 166                         'readOnlyBySection', 'loadMonitorClass' ];
 167
 168                 foreach ( $required as $key ) {
 169                         if ( !isset( $conf[$key] ) ) {
 170                                 throw new MWException( __CLASS__ . ": $key is required in configuration" );
 171                         }
 172                         $this->$key = $conf[$key];
 173                 }
 174
 175                 foreach ( $optional as $key ) {
 176                         if ( isset( $conf[$key] ) ) {
 177                                 $this->$key = $conf[$key];
 178                         }
 179                 }
 180         }
 181
 182         /**
 183          * @param bool|string $wiki
 184          * @return string
 185          */
 186         private function getSectionForWiki( $wiki = false ) {
 187                 if ( $this->lastWiki === $wiki ) {
 188                         return $this->lastSection;
 189                 }
 190                 list( $dbName, ) = $this->getDBNameAndPrefix( $wiki );
 191                 if ( isset( $this->sectionsByDB[$dbName] ) ) {
 192                         $section = $this->sectionsByDB[$dbName];
 193                 } else {
 194                         $section = 'DEFAULT';
 195                 }
 196                 $this->lastSection = $section;
 197                 $this->lastWiki = $wiki;
 198
 199                 return $section;
 200         }
 201
 202         /**
 203          * @param bool|string $wiki
 204          * @return LoadBalancer
 205          */
 206         public function newMainLB( $wiki = false ) {
 207                 list( $dbName, ) = $this->getDBNameAndPrefix( $wiki );
 208                 $section = $this->getSectionForWiki( $wiki );
 209                 if ( isset( $this->groupLoadsByDB[$dbName] ) ) {
 210                         $groupLoads = $this->groupLoadsByDB[$dbName];
 211                 } else {
 212                         $groupLoads = [];
 213                 }
 214
 215                 if ( isset( $this->groupLoadsBySection[$section] ) ) {
 216                         $groupLoads = array_merge_recursive( $groupLoads, $this->groupLoadsBySection[$section] );
 217                 }
 218
 219                 $readOnlyReason = $this->readOnlyReason;
 220                 // Use the LB-specific read-only reason if everything isn't already read-only
 221                 if ( $readOnlyReason === false && isset( $this->readOnlyBySection[$section] ) ) {
 222                         $readOnlyReason = $this->readOnlyBySection[$section];
 223                 }
 224
 225                 return $this->newLoadBalancer(
 226                         $this->serverTemplate,
 227                         $this->sectionLoads[$section],
 228                         $groupLoads,
 229                         $readOnlyReason
 230                 );
 231         }
 232
 233         /**
 234          * @param bool|string $wiki
 235          * @return LoadBalancer
 236          */
 237         public function getMainLB( $wiki = false ) {
 238                 $section = $this->getSectionForWiki( $wiki );
 239                 if ( !isset( $this->mainLBs[$section] ) ) {
 240                         $lb = $this->newMainLB( $wiki );
 241                         $lb->parentInfo( [ 'id' => "main-$section" ] );
 242                         $this->chronProt->initLB( $lb );
 243                         $this->mainLBs[$section] = $lb;
 244                 }
 245
 246                 return $this->mainLBs[$section];
 247         }
 248
 249         /**
 250          * @param string $cluster
 251          * @param bool|string $wiki
 252          * @throws MWException
 253          * @return LoadBalancer
 254          */
 255         protected function newExternalLB( $cluster, $wiki = false ) {
 256                 if ( !isset( $this->externalLoads[$cluster] ) ) {
 257                         throw new MWException( __METHOD__ . ": Unknown cluster \"$cluster\"" );
 258                 }
 259                 $template = $this->serverTemplate;
 260                 if ( isset( $this->externalTemplateOverrides ) ) {
 261                         $template = $this->externalTemplateOverrides + $template;
 262                 }
 263                 if ( isset( $this->templateOverridesByCluster[$cluster] ) ) {
 264                         $template = $this->templateOverridesByCluster[$cluster] + $template;
 265                 }
 266
 267                 return $this->newLoadBalancer(
 268                         $template,
 269                         $this->externalLoads[$cluster],
 270                         [],
 271                         $this->readOnlyReason
 272                 );
 273         }
 274
 275         /**
 276          * @param string $cluster External storage cluster, or false for core
 277          * @param bool|string $wiki Wiki ID, or false for the current wiki
 278          * @return LoadBalancer
 279          */
 280         public function &getExternalLB( $cluster, $wiki = false ) {
 281                 if ( !isset( $this->extLBs[$cluster] ) ) {
 282                         $this->extLBs[$cluster] = $this->newExternalLB( $cluster, $wiki );
 283                         $this->extLBs[$cluster]->parentInfo( [ 'id' => "ext-$cluster" ] );
 284                         $this->chronProt->initLB( $this->extLBs[$cluster] );
 285                 }
 286
 287                 return $this->extLBs[$cluster];
 288         }
 289
 290         /**
 291          * Make a new load balancer object based on template and load array
 292          *
 293          * @param array $template
 294          * @param array $loads
 295          * @param array $groupLoads
 296          * @param string|bool $readOnlyReason
 297          * @return LoadBalancer
 298          */
 299         private function newLoadBalancer( $template, $loads, $groupLoads, $readOnlyReason ) {
 300                 return new LoadBalancer( [
 301                         'servers' => $this->makeServerArray( $template, $loads, $groupLoads ),
 302                         'loadMonitor' => $this->loadMonitorClass,
 303                         'readOnlyReason' => $readOnlyReason,
 304                         'trxProfiler' => $this->trxProfiler
 305                 ] );
 306         }
 307
 308         /**
 309          * Make a server array as expected by LoadBalancer::__construct, using a template and load array
 310          *
 311          * @param array $template
 312          * @param array $loads
 313          * @param array $groupLoads
 314          * @return array
 315          */
 316         private function makeServerArray( $template, $loads, $groupLoads ) {
 317                 $servers = [];
 318                 $master = true;
 319                 $groupLoadsByServer = $this->reindexGroupLoads( $groupLoads );
 320                 foreach ( $groupLoadsByServer as $server => $stuff ) {
 321                         if ( !isset( $loads[$server] ) ) {
 322                                 $loads[$server] = 0;
 323                         }
 324                 }
 325                 foreach ( $loads as $serverName => $load ) {
 326                         $serverInfo = $template;
 327                         if ( $master ) {
 328                                 $serverInfo['master'] = true;
 329                                 if ( isset( $this->masterTemplateOverrides ) ) {
 330                                         $serverInfo = $this->masterTemplateOverrides + $serverInfo;
 331                                 }
 332                                 $master = false;
 333                         } else {
 334                                 $serverInfo['slave'] = true;
 335                         }
 336                         if ( isset( $this->templateOverridesByServer[$serverName] ) ) {
 337                                 $serverInfo = $this->templateOverridesByServer[$serverName] + $serverInfo;
 338                         }
 339                         if ( isset( $groupLoadsByServer[$serverName] ) ) {
 340                                 $serverInfo['groupLoads'] = $groupLoadsByServer[$serverName];
 341                         }
 342                         if ( isset( $this->hostsByName[$serverName] ) ) {
 343                                 $serverInfo['host'] = $this->hostsByName[$serverName];
 344                         } else {
 345                                 $serverInfo['host'] = $serverName;
 346                         }
 347                         $serverInfo['hostName'] = $serverName;
 348                         $serverInfo['load'] = $load;
 349                         $servers[] = $serverInfo;
 350                 }
 351
 352                 return $servers;
 353         }
 354
 355         /**
 356          * Take a group load array indexed by group then server, and reindex it by server then group
 357          * @param array $groupLoads
 358          * @return array
 359          */
 360         private function reindexGroupLoads( $groupLoads ) {
 361                 $reindexed = [];
 362                 foreach ( $groupLoads as $group => $loads ) {
 363                         foreach ( $loads as $server => $load ) {
 364                                 $reindexed[$server][$group] = $load;
 365                         }
 366                 }
 367
 368                 return $reindexed;
 369         }
 370
 371         /**
 372          * Get the database name and prefix based on the wiki ID
 373          * @param bool|string $wiki
 374          * @return array
 375          */
 376         private function getDBNameAndPrefix( $wiki = false ) {
 377                 if ( $wiki === false ) {
 378                         global $wgDBname, $wgDBprefix;
 379
 380                         return [ $wgDBname, $wgDBprefix ];
 381                 } else {
 382                         return wfSplitWikiID( $wiki );
 383                 }
 384         }
 385
 386         /**
 387          * Execute a function for each tracked load balancer
 388          * The callback is called with the load balancer as the first parameter,
 389          * and $params passed as the subsequent parameters.
 390          * @param callable $callback
 391          * @param array $params
 392          */
 393         public function forEachLB( $callback, array $params = [] ) {
 394                 foreach ( $this->mainLBs as $lb ) {
 395                         call_user_func_array( $callback, array_merge( [ $lb ], $params ) );
 396                 }
 397                 foreach ( $this->extLBs as $lb ) {
 398                         call_user_func_array( $callback, array_merge( [ $lb ], $params ) );
 399                 }
 400         }
 401
 402         public function shutdown( $flags = 0 ) {
 403                 if ( !( $flags & self::SHUTDOWN_NO_CHRONPROT ) ) {
 404                         $this->shutdownChronologyProtector( $this->chronProt );
 405                 }
 406                 $this->commitMasterChanges( __METHOD__ ); // sanity
 407         }
 408 }