includes/externalstore/ExternalStoreDB.php

   1 <?php
   2 /**
   3  * External storage in SQL database.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 use Wikimedia\Rdbms\LBFactory;
  24 use Wikimedia\Rdbms\ILoadBalancer;
  25 use Wikimedia\Rdbms\IDatabase;
  26 use Wikimedia\Rdbms\DBConnRef;
  27 use Wikimedia\Rdbms\MaintainableDBConnRef;
  28 use Wikimedia\Rdbms\DatabaseDomain;
  29 use Wikimedia\Rdbms\DBUnexpectedError;
  30
  31 /**
  32  * DB accessible external objects.
  33  *
  34  * In this system, each store "location" maps to a database "cluster".
  35  * The clusters must be defined in the normal LBFactory configuration.
  36  *
  37  * @ingroup ExternalStorage
  38  */
  39 class ExternalStoreDB extends ExternalStoreMedium {
  40         /** @var LBFactory */
  41         private $lbFactory;
  42
  43         /**
  44          * @see ExternalStoreMedium::__construct()
  45          * @param array $params Additional parameters include:
  46          *   - lbFactory: an LBFactory instance
  47          */
  48         public function __construct( array $params ) {
  49                 parent::__construct( $params );
  50                 if ( !isset( $params['lbFactory'] ) || !( $params['lbFactory'] instanceof LBFactory ) ) {
  51                         throw new InvalidArgumentException( "LBFactory required in 'lbFactory' field." );
  52                 }
  53                 $this->lbFactory = $params['lbFactory'];
  54         }
  55
  56         /**
  57          * The provided URL is in the form of DB://cluster/id
  58          * or DB://cluster/id/itemid for concatened storage.
  59          *
  60          * @param string $url
  61          * @return string|bool False if missing
  62          * @see ExternalStoreMedium::fetchFromURL()
  63          */
  64         public function fetchFromURL( $url ) {
  65                 list( $cluster, $id, $itemID ) = $this->parseURL( $url );
  66                 $ret = $this->fetchBlob( $cluster, $id, $itemID );
  67
  68                 if ( $itemID !== false && $ret !== false ) {
  69                         return $ret->getItem( $itemID );
  70                 }
  71
  72                 return $ret;
  73         }
  74
  75         /**
  76          * Fetch data from given external store URLs.
  77          * The provided URLs are in the form of DB://cluster/id
  78          * or DB://cluster/id/itemid for concatened storage.
  79          *
  80          * @param array $urls An array of external store URLs
  81          * @return array A map from url to stored content. Failed results
  82          *     are not represented.
  83          */
  84         public function batchFetchFromURLs( array $urls ) {
  85                 $batched = $inverseUrlMap = [];
  86                 foreach ( $urls as $url ) {
  87                         list( $cluster, $id, $itemID ) = $this->parseURL( $url );
  88                         $batched[$cluster][$id][] = $itemID;
  89                         // false $itemID gets cast to int, but should be ok
  90                         // since we do === from the $itemID in $batched
  91                         $inverseUrlMap[$cluster][$id][$itemID] = $url;
  92                 }
  93                 $ret = [];
  94                 foreach ( $batched as $cluster => $batchByCluster ) {
  95                         $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
  96                         /** @var HistoryBlob $blob */
  97                         foreach ( $res as $id => $blob ) {
  98                                 foreach ( $batchByCluster[$id] as $itemID ) {
  99                                         $url = $inverseUrlMap[$cluster][$id][$itemID];
 100                                         if ( $itemID === false ) {
 101                                                 $ret[$url] = $blob;
 102                                         } else {
 103                                                 $ret[$url] = $blob->getItem( $itemID );
 104                                         }
 105                                 }
 106                         }
 107                 }
 108
 109                 return $ret;
 110         }
 111
 112         /**
 113          * @inheritDoc
 114          */
 115         public function store( $location, $data ) {
 116                 $dbw = $this->getMaster( $location );
 117                 $dbw->insert(
 118                         $this->getTable( $dbw, $location ),
 119                         [ 'blob_text' => $data ],
 120                         __METHOD__
 121                 );
 122                 $id = $dbw->insertId();
 123                 if ( !$id ) {
 124                         throw new MWException( __METHOD__ . ': no insert ID' );
 125                 }
 126
 127                 return "DB://$location/$id";
 128         }
 129
 130         /**
 131          * @inheritDoc
 132          */
 133         public function isReadOnly( $location ) {
 134                 if ( parent::isReadOnly( $location ) ) {
 135                         return true;
 136                 }
 137
 138                 $lb = $this->getLoadBalancer( $location );
 139                 $domainId = $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) );
 140
 141                 return ( $lb->getReadOnlyReason( $domainId ) !== false );
 142         }
 143
 144         /**
 145          * Get a LoadBalancer for the specified cluster
 146          *
 147          * @param string $cluster Cluster name
 148          * @return ILoadBalancer
 149          */
 150         private function getLoadBalancer( $cluster ) {
 151                 return $this->lbFactory->getExternalLB( $cluster );
 152         }
 153
 154         /**
 155          * Get a replica DB connection for the specified cluster
 156          *
 157          * @since 1.34
 158          * @param string $cluster Cluster name
 159          * @return DBConnRef
 160          */
 161         public function getReplica( $cluster ) {
 162                 $lb = $this->getLoadBalancer( $cluster );
 163
 164                 return $lb->getConnectionRef(
 165                         DB_REPLICA,
 166                         [],
 167                         $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
 168                         $lb::CONN_TRX_AUTOCOMMIT
 169                 );
 170         }
 171
 172         /**
 173          * Get a replica DB connection for the specified cluster
 174          *
 175          * @param string $cluster Cluster name
 176          * @return DBConnRef
 177          * @deprecated since 1.34
 178          */
 179         public function getSlave( $cluster ) {
 180                 return $this->getReplica( $cluster );
 181         }
 182
 183         /**
 184          * Get a master database connection for the specified cluster
 185          *
 186          * @param string $cluster Cluster name
 187          * @return MaintainableDBConnRef
 188          */
 189         public function getMaster( $cluster ) {
 190                 $lb = $this->getLoadBalancer( $cluster );
 191
 192                 return $lb->getMaintenanceConnectionRef(
 193                         DB_MASTER,
 194                         [],
 195                         $this->getDomainId( $lb->getServerInfo( $lb->getWriterIndex() ) ),
 196                         $lb::CONN_TRX_AUTOCOMMIT
 197                 );
 198         }
 199
 200         /**
 201          * @param array $server Master DB server configuration array for LoadBalancer
 202          * @return string|bool Database domain ID or false
 203          */
 204         private function getDomainId( array $server ) {
 205                 if ( $this->isDbDomainExplicit ) {
 206                         return $this->dbDomain; // explicit foreign domain
 207                 }
 208
 209                 if ( isset( $server['dbname'] ) ) {
 210                         // T200471: for b/c, treat any "dbname" field as forcing which database to use.
 211                         // MediaWiki/LoadBalancer previously did not enforce any concept of a local DB
 212                         // domain, but rather assumed that the LB server configuration matched $wgDBname.
 213                         // This check is useful when the external storage DB for this cluster does not use
 214                         // the same name as the corresponding "main" DB(s) for wikis.
 215                         $domain = new DatabaseDomain(
 216                                 $server['dbname'],
 217                                 $server['schema'] ?? null,
 218                                 $server['tablePrefix'] ?? ''
 219                         );
 220
 221                         return $domain->getId();
 222                 }
 223
 224                 return false; // local LB domain
 225         }
 226
 227         /**
 228          * Get the 'blobs' table name for this database
 229          *
 230          * @param IDatabase $db
 231          * @param string|null $cluster Cluster name
 232          * @return string Table name ('blobs' by default)
 233          */
 234         public function getTable( $db, $cluster = null ) {
 235                 if ( $cluster !== null ) {
 236                         $lb = $this->getLoadBalancer( $cluster );
 237                         $info = $lb->getServerInfo( $lb->getWriterIndex() );
 238                         if ( isset( $info['blobs table'] ) ) {
 239                                 return $info['blobs table'];
 240                         }
 241                 }
 242
 243                 return $db->getLBInfo( 'blobs table' ) ?? 'blobs'; // b/c
 244         }
 245
 246         /**
 247          * Create the appropriate blobs table on this cluster
 248          *
 249          * @see getTable()
 250          * @since 1.34
 251          * @param string $cluster
 252          */
 253         public function initializeTable( $cluster ) {
 254                 global $IP;
 255
 256                 static $supportedTypes = [ 'mysql', 'sqlite' ];
 257
 258                 $dbw = $this->getMaster( $cluster );
 259                 if ( !in_array( $dbw->getType(), $supportedTypes, true ) ) {
 260                         throw new DBUnexpectedError( $dbw, "RDBMS type '{$dbw->getType()}' not supported." );
 261                 }
 262
 263                 $sqlFilePath = "$IP/maintenance/storage/blobs.sql";
 264                 $sql = file_get_contents( $sqlFilePath );
 265                 if ( $sql === false ) {
 266                         throw new RuntimeException( "Failed to read '$sqlFilePath'." );
 267                 }
 268
 269                 $rawTable = $this->getTable( $dbw, $cluster ); // e.g. "blobs_cluster23"
 270                 $encTable = $dbw->tableName( $rawTable );
 271                 $dbw->query(
 272                         str_replace(
 273                                 [ '/*$wgDBprefix*/blobs', '/*_*/blobs' ],
 274                                 [ $encTable, $encTable ],
 275                                 $sql
 276                         ),
 277                         __METHOD__,
 278                         $dbw::QUERY_IGNORE_DBO_TRX
 279                 );
 280         }
 281
 282         /**
 283          * Fetch a blob item out of the database; a cache of the last-loaded
 284          * blob will be kept so that multiple loads out of a multi-item blob
 285          * can avoid redundant database access and decompression.
 286          * @param string $cluster
 287          * @param string $id
 288          * @param string $itemID
 289          * @return HistoryBlob|bool Returns false if missing
 290          */
 291         private function fetchBlob( $cluster, $id, $itemID ) {
 292                 /**
 293                  * One-step cache variable to hold base blobs; operations that
 294                  * pull multiple revisions may often pull multiple times from
 295                  * the same blob. By keeping the last-used one open, we avoid
 296                  * redundant unserialization and decompression overhead.
 297                  */
 298                 static $externalBlobCache = [];
 299
 300                 $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
 301                 $cacheID = "$cacheID@{$this->dbDomain}";
 302
 303                 if ( isset( $externalBlobCache[$cacheID] ) ) {
 304                         $this->logger->debug( "ExternalStoreDB::fetchBlob cache hit on $cacheID" );
 305
 306                         return $externalBlobCache[$cacheID];
 307                 }
 308
 309                 $this->logger->debug( "ExternalStoreDB::fetchBlob cache miss on $cacheID" );
 310
 311                 $dbr = $this->getReplica( $cluster );
 312                 $ret = $dbr->selectField(
 313                         $this->getTable( $dbr, $cluster ),
 314                         'blob_text',
 315                         [ 'blob_id' => $id ],
 316                         __METHOD__
 317                 );
 318                 if ( $ret === false ) {
 319                         $this->logger->info( "ExternalStoreDB::fetchBlob master fallback on $cacheID" );
 320                         // Try the master
 321                         $dbw = $this->getMaster( $cluster );
 322                         $ret = $dbw->selectField(
 323                                 $this->getTable( $dbw, $cluster ),
 324                                 'blob_text',
 325                                 [ 'blob_id' => $id ],
 326                                 __METHOD__
 327                         );
 328                         if ( $ret === false ) {
 329                                 $this->logger->error( "ExternalStoreDB::fetchBlob master failed to find $cacheID" );
 330                         }
 331                 }
 332                 if ( $itemID !== false && $ret !== false ) {
 333                         // Unserialise object; caller extracts item
 334                         $ret = unserialize( $ret );
 335                 }
 336
 337                 $externalBlobCache = [ $cacheID => $ret ];
 338
 339                 return $ret;
 340         }
 341
 342         /**
 343          * Fetch multiple blob items out of the database
 344          *
 345          * @param string $cluster A cluster name valid for use with LBFactory
 346          * @param array $ids A map from the blob_id's to look for to the requested itemIDs in the blobs
 347          * @return array A map from the blob_id's requested to their content.
 348          *   Unlocated ids are not represented
 349          */
 350         private function batchFetchBlobs( $cluster, array $ids ) {
 351                 $dbr = $this->getReplica( $cluster );
 352                 $res = $dbr->select(
 353                         $this->getTable( $dbr, $cluster ),
 354                         [ 'blob_id', 'blob_text' ],
 355                         [ 'blob_id' => array_keys( $ids ) ],
 356                         __METHOD__
 357                 );
 358
 359                 $ret = [];
 360                 if ( $res !== false ) {
 361                         $this->mergeBatchResult( $ret, $ids, $res );
 362                 }
 363                 if ( $ids ) {
 364                         $this->logger->info(
 365                                 __METHOD__ . ": master fallback on '$cluster' for: " .
 366                                 implode( ',', array_keys( $ids ) )
 367                         );
 368                         // Try the master
 369                         $dbw = $this->getMaster( $cluster );
 370                         $res = $dbw->select(
 371                                 $this->getTable( $dbr, $cluster ),
 372                                 [ 'blob_id', 'blob_text' ],
 373                                 [ 'blob_id' => array_keys( $ids ) ],
 374                                 __METHOD__ );
 375                         if ( $res === false ) {
 376                                 $this->logger->error( __METHOD__ . ": master failed on '$cluster'" );
 377                         } else {
 378                                 $this->mergeBatchResult( $ret, $ids, $res );
 379                         }
 380                 }
 381                 if ( $ids ) {
 382                         $this->logger->error(
 383                                 __METHOD__ . ": master on '$cluster' failed locating items: " .
 384                                 implode( ',', array_keys( $ids ) )
 385                         );
 386                 }
 387
 388                 return $ret;
 389         }
 390
 391         /**
 392          * Helper function for self::batchFetchBlobs for merging master/replica DB results
 393          * @param array &$ret Current self::batchFetchBlobs return value
 394          * @param array &$ids Map from blob_id to requested itemIDs
 395          * @param mixed $res DB result from Database::select
 396          */
 397         private function mergeBatchResult( array &$ret, array &$ids, $res ) {
 398                 foreach ( $res as $row ) {
 399                         $id = $row->blob_id;
 400                         $itemIDs = $ids[$id];
 401                         unset( $ids[$id] ); // to track if everything is found
 402                         if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
 403                                 // single result stored per blob
 404                                 $ret[$id] = $row->blob_text;
 405                         } else {
 406                                 // multi result stored per blob
 407                                 $ret[$id] = unserialize( $row->blob_text );
 408                         }
 409                 }
 410         }
 411
 412         /**
 413          * @param string $url
 414          * @return array
 415          */
 416         protected function parseURL( $url ) {
 417                 $path = explode( '/', $url );
 418
 419                 return [
 420                         $path[2], // cluster
 421                         $path[3], // id
 422                         $path[4] ?? false // itemID
 423                 ];
 424         }
 425 }