includes/externalstore/ExternalStoreDB.php

   1 <?php
   2 /**
   3  * External storage in SQL database.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 use MediaWiki\MediaWikiServices;
  24 use Wikimedia\Rdbms\LoadBalancer;
  25 use Wikimedia\Rdbms\IDatabase;
  26 use Wikimedia\Rdbms\DBConnRef;
  27 use Wikimedia\Rdbms\MaintainableDBConnRef;
  28
  29 /**
  30  * DB accessible external objects.
  31  *
  32  * In this system, each store "location" maps to a database "cluster".
  33  * The clusters must be defined in the normal LBFactory configuration.
  34  *
  35  * @ingroup ExternalStorage
  36  */
  37 class ExternalStoreDB extends ExternalStoreMedium {
  38         /**
  39          * The provided URL is in the form of DB://cluster/id
  40          * or DB://cluster/id/itemid for concatened storage.
  41          *
  42          * @param string $url
  43          * @return string|bool False if missing
  44          * @see ExternalStoreMedium::fetchFromURL()
  45          */
  46         public function fetchFromURL( $url ) {
  47                 list( $cluster, $id, $itemID ) = $this->parseURL( $url );
  48                 $ret = $this->fetchBlob( $cluster, $id, $itemID );
  49
  50                 if ( $itemID !== false && $ret !== false ) {
  51                         return $ret->getItem( $itemID );
  52                 }
  53
  54                 return $ret;
  55         }
  56
  57         /**
  58          * Fetch data from given external store URLs.
  59          * The provided URLs are in the form of DB://cluster/id
  60          * or DB://cluster/id/itemid for concatened storage.
  61          *
  62          * @param array $urls An array of external store URLs
  63          * @return array A map from url to stored content. Failed results
  64          *     are not represented.
  65          */
  66         public function batchFetchFromURLs( array $urls ) {
  67                 $batched = $inverseUrlMap = [];
  68                 foreach ( $urls as $url ) {
  69                         list( $cluster, $id, $itemID ) = $this->parseURL( $url );
  70                         $batched[$cluster][$id][] = $itemID;
  71                         // false $itemID gets cast to int, but should be ok
  72                         // since we do === from the $itemID in $batched
  73                         $inverseUrlMap[$cluster][$id][$itemID] = $url;
  74                 }
  75                 $ret = [];
  76                 foreach ( $batched as $cluster => $batchByCluster ) {
  77                         $res = $this->batchFetchBlobs( $cluster, $batchByCluster );
  78                         /** @var HistoryBlob $blob */
  79                         foreach ( $res as $id => $blob ) {
  80                                 foreach ( $batchByCluster[$id] as $itemID ) {
  81                                         $url = $inverseUrlMap[$cluster][$id][$itemID];
  82                                         if ( $itemID === false ) {
  83                                                 $ret[$url] = $blob;
  84                                         } else {
  85                                                 $ret[$url] = $blob->getItem( $itemID );
  86                                         }
  87                                 }
  88                         }
  89                 }
  90
  91                 return $ret;
  92         }
  93
  94         public function store( $location, $data ) {
  95                 $dbw = $this->getMaster( $location );
  96                 $dbw->insert( $this->getTable( $dbw ),
  97                         [ 'blob_text' => $data ],
  98                         __METHOD__ );
  99                 $id = $dbw->insertId();
 100                 if ( !$id ) {
 101                         throw new MWException( __METHOD__ . ': no insert ID' );
 102                 }
 103
 104                 return "DB://$location/$id";
 105         }
 106
 107         public function isReadOnly( $location ) {
 108                 return ( $this->getLoadBalancer( $location )->getReadOnlyReason() !== false );
 109         }
 110
 111         /**
 112          * Get a LoadBalancer for the specified cluster
 113          *
 114          * @param string $cluster Cluster name
 115          * @return LoadBalancer
 116          */
 117         private function getLoadBalancer( $cluster ) {
 118                 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
 119                 return $lbFactory->getExternalLB( $cluster );
 120         }
 121
 122         /**
 123          * Get a replica DB connection for the specified cluster
 124          *
 125          * @param string $cluster Cluster name
 126          * @return DBConnRef
 127          */
 128         public function getSlave( $cluster ) {
 129                 global $wgDefaultExternalStore;
 130
 131                 $wiki = $this->params['wiki'] ?? false;
 132                 $lb = $this->getLoadBalancer( $cluster );
 133
 134                 if ( !in_array( "DB://" . $cluster, (array)$wgDefaultExternalStore ) ) {
 135                         wfDebug( "read only external store\n" );
 136                         $lb->allowLagged( true );
 137                 } else {
 138                         wfDebug( "writable external store\n" );
 139                 }
 140
 141                 $db = $lb->getConnectionRef( DB_REPLICA, [], $wiki );
 142                 $db->clearFlag( DBO_TRX ); // sanity
 143
 144                 return $db;
 145         }
 146
 147         /**
 148          * Get a master database connection for the specified cluster
 149          *
 150          * @param string $cluster Cluster name
 151          * @return MaintainableDBConnRef
 152          */
 153         public function getMaster( $cluster ) {
 154                 $wiki = $this->params['wiki'] ?? false;
 155                 $lb = $this->getLoadBalancer( $cluster );
 156
 157                 $db = $lb->getMaintenanceConnectionRef( DB_MASTER, [], $wiki );
 158                 $db->clearFlag( DBO_TRX ); // sanity
 159
 160                 return $db;
 161         }
 162
 163         /**
 164          * Get the 'blobs' table name for this database
 165          *
 166          * @param IDatabase $db
 167          * @return string Table name ('blobs' by default)
 168          */
 169         public function getTable( $db ) {
 170                 $table = $db->getLBInfo( 'blobs table' );
 171                 if ( is_null( $table ) ) {
 172                         $table = 'blobs';
 173                 }
 174
 175                 return $table;
 176         }
 177
 178         /**
 179          * Fetch a blob item out of the database; a cache of the last-loaded
 180          * blob will be kept so that multiple loads out of a multi-item blob
 181          * can avoid redundant database access and decompression.
 182          * @param string $cluster
 183          * @param string $id
 184          * @param string $itemID
 185          * @return HistoryBlob|bool Returns false if missing
 186          */
 187         private function fetchBlob( $cluster, $id, $itemID ) {
 188                 /**
 189                  * One-step cache variable to hold base blobs; operations that
 190                  * pull multiple revisions may often pull multiple times from
 191                  * the same blob. By keeping the last-used one open, we avoid
 192                  * redundant unserialization and decompression overhead.
 193                  */
 194                 static $externalBlobCache = [];
 195
 196                 $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/";
 197
 198                 $wiki = $this->params['wiki'] ?? false;
 199                 $cacheID = ( $wiki === false ) ? $cacheID : "$cacheID@$wiki";
 200
 201                 if ( isset( $externalBlobCache[$cacheID] ) ) {
 202                         wfDebugLog( 'ExternalStoreDB-cache',
 203                                 "ExternalStoreDB::fetchBlob cache hit on $cacheID" );
 204
 205                         return $externalBlobCache[$cacheID];
 206                 }
 207
 208                 wfDebugLog( 'ExternalStoreDB-cache',
 209                         "ExternalStoreDB::fetchBlob cache miss on $cacheID" );
 210
 211                 $dbr = $this->getSlave( $cluster );
 212                 $ret = $dbr->selectField( $this->getTable( $dbr ),
 213                         'blob_text', [ 'blob_id' => $id ], __METHOD__ );
 214                 if ( $ret === false ) {
 215                         wfDebugLog( 'ExternalStoreDB',
 216                                 "ExternalStoreDB::fetchBlob master fallback on $cacheID" );
 217                         // Try the master
 218                         $dbw = $this->getMaster( $cluster );
 219                         $ret = $dbw->selectField( $this->getTable( $dbw ),
 220                                 'blob_text', [ 'blob_id' => $id ], __METHOD__ );
 221                         if ( $ret === false ) {
 222                                 wfDebugLog( 'ExternalStoreDB',
 223                                         "ExternalStoreDB::fetchBlob master failed to find $cacheID" );
 224                         }
 225                 }
 226                 if ( $itemID !== false && $ret !== false ) {
 227                         // Unserialise object; caller extracts item
 228                         $ret = unserialize( $ret );
 229                 }
 230
 231                 $externalBlobCache = [ $cacheID => $ret ];
 232
 233                 return $ret;
 234         }
 235
 236         /**
 237          * Fetch multiple blob items out of the database
 238          *
 239          * @param string $cluster A cluster name valid for use with LBFactory
 240          * @param array $ids A map from the blob_id's to look for to the requested itemIDs in the blobs
 241          * @return array A map from the blob_id's requested to their content.
 242          *   Unlocated ids are not represented
 243          */
 244         private function batchFetchBlobs( $cluster, array $ids ) {
 245                 $dbr = $this->getSlave( $cluster );
 246                 $res = $dbr->select( $this->getTable( $dbr ),
 247                         [ 'blob_id', 'blob_text' ], [ 'blob_id' => array_keys( $ids ) ], __METHOD__ );
 248                 $ret = [];
 249                 if ( $res !== false ) {
 250                         $this->mergeBatchResult( $ret, $ids, $res );
 251                 }
 252                 if ( $ids ) {
 253                         wfDebugLog( __CLASS__, __METHOD__ .
 254                                 " master fallback on '$cluster' for: " .
 255                                 implode( ',', array_keys( $ids ) ) );
 256                         // Try the master
 257                         $dbw = $this->getMaster( $cluster );
 258                         $res = $dbw->select( $this->getTable( $dbr ),
 259                                 [ 'blob_id', 'blob_text' ],
 260                                 [ 'blob_id' => array_keys( $ids ) ],
 261                                 __METHOD__ );
 262                         if ( $res === false ) {
 263                                 wfDebugLog( __CLASS__, __METHOD__ . " master failed on '$cluster'" );
 264                         } else {
 265                                 $this->mergeBatchResult( $ret, $ids, $res );
 266                         }
 267                 }
 268                 if ( $ids ) {
 269                         wfDebugLog( __CLASS__, __METHOD__ .
 270                                 " master on '$cluster' failed locating items: " .
 271                                 implode( ',', array_keys( $ids ) ) );
 272                 }
 273
 274                 return $ret;
 275         }
 276
 277         /**
 278          * Helper function for self::batchFetchBlobs for merging master/replica DB results
 279          * @param array &$ret Current self::batchFetchBlobs return value
 280          * @param array &$ids Map from blob_id to requested itemIDs
 281          * @param mixed $res DB result from Database::select
 282          */
 283         private function mergeBatchResult( array &$ret, array &$ids, $res ) {
 284                 foreach ( $res as $row ) {
 285                         $id = $row->blob_id;
 286                         $itemIDs = $ids[$id];
 287                         unset( $ids[$id] ); // to track if everything is found
 288                         if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) {
 289                                 // single result stored per blob
 290                                 $ret[$id] = $row->blob_text;
 291                         } else {
 292                                 // multi result stored per blob
 293                                 $ret[$id] = unserialize( $row->blob_text );
 294                         }
 295                 }
 296         }
 297
 298         /**
 299          * @param string $url
 300          * @return array
 301          */
 302         protected function parseURL( $url ) {
 303                 $path = explode( '/', $url );
 304
 305                 return [
 306                         $path[2], // cluster
 307                         $path[3], // id
 308                         $path[4] ?? false // itemID
 309                 ];
 310         }
 311 }