From: Aaron Schulz Date: Tue, 27 Aug 2019 07:42:44 +0000 (-0700) Subject: externalstore: deprecate getLBInfo() dependency in ExternalStoreDB X-Git-Tag: 1.34.0-rc.0~286 X-Git-Url: https://git.heureux-cyclage.org/?p=lhc%2Fweb%2Fwiklou.git;a=commitdiff_plain;h=176e08484f79be7d6d5960c286bd02b2439e6f72 externalstore: deprecate getLBInfo() dependency in ExternalStoreDB Use the cluster and master server info array instead, when possible. Also add initializeTable() method based on the one in addWiki.php. This lets some internals be hidden from that script a bit more. Bug: T231162 Change-Id: I694f12cc800684bcc13351021d9abc9a01bddfe9 --- diff --git a/includes/externalstore/ExternalStoreDB.php b/includes/externalstore/ExternalStoreDB.php index 4db351b25b..264eabcd44 100644 --- a/includes/externalstore/ExternalStoreDB.php +++ b/includes/externalstore/ExternalStoreDB.php @@ -26,6 +26,7 @@ use Wikimedia\Rdbms\IDatabase; use Wikimedia\Rdbms\DBConnRef; use Wikimedia\Rdbms\MaintainableDBConnRef; use Wikimedia\Rdbms\DatabaseDomain; +use Wikimedia\Rdbms\DBUnexpectedError; /** * DB accessible external objects. @@ -113,7 +114,11 @@ class ExternalStoreDB extends ExternalStoreMedium { */ public function store( $location, $data ) { $dbw = $this->getMaster( $location ); - $dbw->insert( $this->getTable( $dbw ), [ 'blob_text' => $data ], __METHOD__ ); + $dbw->insert( + $this->getTable( $dbw, $location ), + [ 'blob_text' => $data ], + __METHOD__ + ); $id = $dbw->insertId(); if ( !$id ) { throw new MWException( __METHOD__ . ': no insert ID' ); @@ -149,10 +154,11 @@ class ExternalStoreDB extends ExternalStoreMedium { /** * Get a replica DB connection for the specified cluster * + * @since 1.34 * @param string $cluster Cluster name * @return DBConnRef */ - public function getSlave( $cluster ) { + public function getReplica( $cluster ) { $lb = $this->getLoadBalancer( $cluster ); return $lb->getConnectionRef( @@ -163,6 +169,17 @@ class ExternalStoreDB extends ExternalStoreMedium { ); } + /** + * Get a replica DB connection for the specified cluster + * + * @param string $cluster Cluster name + * @return DBConnRef + * @deprecated since 1.34 + */ + public function getSlave( $cluster ) { + return $this->getReplica( $cluster ); + } + /** * Get a master database connection for the specified cluster * @@ -211,15 +228,55 @@ class ExternalStoreDB extends ExternalStoreMedium { * Get the 'blobs' table name for this database * * @param IDatabase $db + * @param string|null $cluster Cluster name * @return string Table name ('blobs' by default) */ - public function getTable( $db ) { - $table = $db->getLBInfo( 'blobs table' ); - if ( is_null( $table ) ) { - $table = 'blobs'; + public function getTable( $db, $cluster = null ) { + if ( $cluster !== null ) { + $lb = $this->getLoadBalancer( $cluster ); + $info = $lb->getServerInfo( $lb->getWriterIndex() ); + if ( isset( $info['blobs table'] ) ) { + return $info['blobs table']; + } } - return $table; + return $db->getLBInfo( 'blobs table' ) ?? 'blobs'; // b/c + } + + /** + * Create the appropriate blobs table on this cluster + * + * @see getTable() + * @since 1.34 + * @param string $cluster + */ + public function initializeTable( $cluster ) { + global $IP; + + static $supportedTypes = [ 'mysql', 'sqlite' ]; + + $dbw = $this->getMaster( $cluster ); + if ( !in_array( $dbw->getType(), $supportedTypes, true ) ) { + throw new DBUnexpectedError( $dbw, "RDBMS type '{$dbw->getType()}' not supported." ); + } + + $sqlFilePath = "$IP/maintenance/storage/blobs.sql"; + $sql = file_get_contents( $sqlFilePath ); + if ( $sql === false ) { + throw new RuntimeException( "Failed to read '$sqlFilePath'." ); + } + + $rawTable = $this->getTable( $dbw, $cluster ); // e.g. "blobs_cluster23" + $encTable = $dbw->tableName( $rawTable ); + $dbw->query( + str_replace( + [ '/*$wgDBprefix*/blobs', '/*_*/blobs' ], + [ $encTable, $encTable ], + $sql + ), + __METHOD__, + $dbw::QUERY_IGNORE_DBO_TRX + ); } /** @@ -251,15 +308,23 @@ class ExternalStoreDB extends ExternalStoreMedium { $this->logger->debug( "ExternalStoreDB::fetchBlob cache miss on $cacheID" ); - $dbr = $this->getSlave( $cluster ); - $ret = $dbr->selectField( $this->getTable( $dbr ), - 'blob_text', [ 'blob_id' => $id ], __METHOD__ ); + $dbr = $this->getReplica( $cluster ); + $ret = $dbr->selectField( + $this->getTable( $dbr, $cluster ), + 'blob_text', + [ 'blob_id' => $id ], + __METHOD__ + ); if ( $ret === false ) { $this->logger->info( "ExternalStoreDB::fetchBlob master fallback on $cacheID" ); // Try the master $dbw = $this->getMaster( $cluster ); - $ret = $dbw->selectField( $this->getTable( $dbw ), - 'blob_text', [ 'blob_id' => $id ], __METHOD__ ); + $ret = $dbw->selectField( + $this->getTable( $dbw, $cluster ), + 'blob_text', + [ 'blob_id' => $id ], + __METHOD__ + ); if ( $ret === false ) { $this->logger->error( "ExternalStoreDB::fetchBlob master failed to find $cacheID" ); } @@ -283,9 +348,9 @@ class ExternalStoreDB extends ExternalStoreMedium { * Unlocated ids are not represented */ private function batchFetchBlobs( $cluster, array $ids ) { - $dbr = $this->getSlave( $cluster ); + $dbr = $this->getReplica( $cluster ); $res = $dbr->select( - $this->getTable( $dbr ), + $this->getTable( $dbr, $cluster ), [ 'blob_id', 'blob_text' ], [ 'blob_id' => array_keys( $ids ) ], __METHOD__ @@ -302,7 +367,8 @@ class ExternalStoreDB extends ExternalStoreMedium { ); // Try the master $dbw = $this->getMaster( $cluster ); - $res = $dbw->select( $this->getTable( $dbr ), + $res = $dbw->select( + $this->getTable( $dbr, $cluster ), [ 'blob_id', 'blob_text' ], [ 'blob_id' => array_keys( $ids ) ], __METHOD__ ); diff --git a/maintenance/storage/blobs.sql b/maintenance/storage/blobs.sql index 979e68a94e..e3c6c02bc3 100644 --- a/maintenance/storage/blobs.sql +++ b/maintenance/storage/blobs.sql @@ -1,6 +1,6 @@ -- Blobs table for external storage -CREATE TABLE /*$wgDBprefix*/blobs ( +CREATE TABLE IF NOT EXISTS /*$wgDBprefix*/blobs ( blob_id integer UNSIGNED NOT NULL AUTO_INCREMENT, blob_text longblob, PRIMARY KEY (blob_id) diff --git a/maintenance/storage/checkStorage.php b/maintenance/storage/checkStorage.php index 060f32768d..37625cf2f0 100644 --- a/maintenance/storage/checkStorage.php +++ b/maintenance/storage/checkStorage.php @@ -232,7 +232,7 @@ class CheckStorage { } foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) { $blobIds = array_keys( $xBlobIds ); - $extDb =& $this->dbStore->getSlave( $cluster ); + $extDb =& $this->dbStore->getReplica( $cluster ); $blobsTable = $this->dbStore->getTable( $extDb ); $res = $extDb->select( $blobsTable, [ 'blob_id' ], @@ -433,7 +433,7 @@ class CheckStorage { foreach ( $externalConcatBlobs as $cluster => $oldIds ) { $blobIds = array_keys( $oldIds ); - $extDb =& $this->dbStore->getSlave( $cluster ); + $extDb =& $this->dbStore->getReplica( $cluster ); $blobsTable = $this->dbStore->getTable( $extDb ); $headerLength = strlen( self::CONCAT_HEADER ); $res = $extDb->select( $blobsTable,