includes/Storage/NameTableStore.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20
  21 namespace MediaWiki\Storage;
  22
  23 use IExpiringStore;
  24 use Psr\Log\LoggerInterface;
  25 use WANObjectCache;
  26 use Wikimedia\Assert\Assert;
  27 use Wikimedia\Rdbms\Database;
  28 use Wikimedia\Rdbms\IDatabase;
  29 use Wikimedia\Rdbms\ILoadBalancer;
  30
  31 /**
  32  * @author Addshore
  33  * @since 1.31
  34  */
  35 class NameTableStore {
  36
  37         /** @var ILoadBalancer */
  38         private $loadBalancer;
  39
  40         /** @var WANObjectCache */
  41         private $cache;
  42
  43         /** @var LoggerInterface */
  44         private $logger;
  45
  46         /** @var string[] */
  47         private $tableCache = null;
  48
  49         /** @var bool|string */
  50         private $wikiId = false;
  51
  52         /** @var int */
  53         private $cacheTTL;
  54
  55         /** @var string */
  56         private $table;
  57         /** @var string */
  58         private $idField;
  59         /** @var string */
  60         private $nameField;
  61         /** @var null|callable */
  62         private $normalizationCallback = null;
  63         /** @var null|callable */
  64         private $insertCallback = null;
  65
  66         /**
  67          * @param ILoadBalancer $dbLoadBalancer A load balancer for acquiring database connections
  68          * @param WANObjectCache $cache A cache manager for caching data. This can be the local
  69          *        wiki's default instance even if $wikiId refers to a different wiki, since
  70          *        makeGlobalKey() is used to constructed a key that allows cached names from
  71          *        the same database to be re-used between wikis. For example, enwiki and frwiki will
  72          *        use the same cache keys for names from the wikidatawiki database, regardless
  73          *        of the cache's default key space.
  74          * @param LoggerInterface $logger
  75          * @param string $table
  76          * @param string $idField
  77          * @param string $nameField
  78          * @param callable|null $normalizationCallback Normalization to be applied to names before being
  79          * saved or queried. This should be a callback that accepts and returns a single string.
  80          * @param bool|string $wikiId The ID of the target wiki database. Use false for the local wiki.
  81          * @param callable|null $insertCallback Callback to change insert fields accordingly.
  82          * This parameter was introduced in 1.32
  83          */
  84         public function __construct(
  85                 ILoadBalancer $dbLoadBalancer,
  86                 WANObjectCache $cache,
  87                 LoggerInterface $logger,
  88                 $table,
  89                 $idField,
  90                 $nameField,
  91                 callable $normalizationCallback = null,
  92                 $wikiId = false,
  93                 callable $insertCallback = null
  94         ) {
  95                 $this->loadBalancer = $dbLoadBalancer;
  96                 $this->cache = $cache;
  97                 $this->logger = $logger;
  98                 $this->table = $table;
  99                 $this->idField = $idField;
 100                 $this->nameField = $nameField;
 101                 $this->normalizationCallback = $normalizationCallback;
 102                 $this->wikiId = $wikiId;
 103                 $this->cacheTTL = IExpiringStore::TTL_MONTH;
 104                 $this->insertCallback = $insertCallback;
 105         }
 106
 107         /**
 108          * @param int $index A database index, like DB_MASTER or DB_REPLICA
 109          * @param int $flags Database connection flags
 110          *
 111          * @return IDatabase
 112          */
 113         private function getDBConnection( $index, $flags = 0 ) {
 114                 return $this->loadBalancer->getConnection( $index, [], $this->wikiId, $flags );
 115         }
 116
 117         /**
 118          * Gets the cache key for names.
 119          *
 120          * The cache key is constructed based on the wiki ID passed to the constructor, and allows
 121          * sharing of name tables cached for a specific database between wikis.
 122          *
 123          * @return string
 124          */
 125         private function getCacheKey() {
 126                 return $this->cache->makeGlobalKey(
 127                         'NameTableSqlStore',
 128                         $this->table,
 129                         $this->loadBalancer->resolveDomainID( $this->wikiId )
 130                 );
 131         }
 132
 133         /**
 134          * @param string $name
 135          * @return string
 136          */
 137         private function normalizeName( $name ) {
 138                 if ( $this->normalizationCallback === null ) {
 139                         return $name;
 140                 }
 141                 return call_user_func( $this->normalizationCallback, $name );
 142         }
 143
 144         /**
 145          * Acquire the id of the given name.
 146          * This creates a row in the table if it doesn't already exist.
 147          *
 148          * @param string $name
 149          * @throws NameTableAccessException
 150          * @return int
 151          */
 152         public function acquireId( $name ) {
 153                 Assert::parameterType( 'string', $name, '$name' );
 154                 $name = $this->normalizeName( $name );
 155
 156                 $table = $this->getTableFromCachesOrReplica();
 157                 $searchResult = array_search( $name, $table, true );
 158                 if ( $searchResult === false ) {
 159                         $id = $this->store( $name );
 160                         if ( $id === null ) {
 161                                 // RACE: $name was already in the db, probably just inserted, so load from master.
 162                                 // Use DBO_TRX to avoid missing inserts due to other threads or REPEATABLE-READs.
 163                                 // ...but not during unit tests, because we need the fake DB tables of the default
 164                                 // connection.
 165                                 $connFlags = defined( 'MW_PHPUNIT_TEST' ) ? 0 : ILoadBalancer::CONN_TRX_AUTOCOMMIT;
 166                                 $table = $this->reloadMap( $connFlags );
 167
 168                                 $searchResult = array_search( $name, $table, true );
 169                                 if ( $searchResult === false ) {
 170                                         // Insert failed due to IGNORE flag, but DB_MASTER didn't give us the data
 171                                         $m = "No insert possible but master didn't give us a record for " .
 172                                                 "'{$name}' in '{$this->table}'";
 173                                         $this->logger->error( $m );
 174                                         throw new NameTableAccessException( $m );
 175                                 }
 176                         } elseif ( isset( $table[$id] ) ) {
 177                                 throw new NameTableAccessException(
 178                                         "Expected unused ID from database insert for '$name' "
 179                                         . " into '{$this->table}', but ID $id is already associated with"
 180                                         . " the name '{$table[$id]}'! This may indicate database corruption!" );
 181                         } else {
 182                                 $table[$id] = $name;
 183                                 $searchResult = $id;
 184
 185                                 // As store returned an ID we know we inserted so delete from WAN cache
 186                                 $this->purgeWANCache(
 187                                         function () {
 188                                                 $this->cache->delete( $this->getCacheKey() );
 189                                         }
 190                                 );
 191                         }
 192                         $this->tableCache = $table;
 193                 }
 194
 195                 return $searchResult;
 196         }
 197
 198         /**
 199          * Reloads the name table from the master database, and purges the WAN cache entry.
 200          *
 201          * @note This should only be called in situations where the local cache has been detected
 202          * to be out of sync with the database. There should be no reason to call this method
 203          * from outside the NameTabelStore during normal operation. This method may however be
 204          * useful in unit tests.
 205          *
 206          * @param int $connFlags ILoadBalancer::CONN_XXX flags. Optional.
 207          *
 208          * @return \string[] The freshly reloaded name map
 209          */
 210         public function reloadMap( $connFlags = 0 ) {
 211                 $this->tableCache = $this->loadTable(
 212                         $this->getDBConnection( DB_MASTER, $connFlags )
 213                 );
 214                 $this->purgeWANCache(
 215                         function () {
 216                                 $this->cache->reap( $this->getCacheKey(), INF );
 217                         }
 218                 );
 219
 220                 return $this->tableCache;
 221         }
 222
 223         /**
 224          * Get the id of the given name.
 225          * If the name doesn't exist this will throw.
 226          * This should be used in cases where we believe the name already exists or want to check for
 227          * existence.
 228          *
 229          * @param string $name
 230          * @throws NameTableAccessException The name does not exist
 231          * @return int Id
 232          */
 233         public function getId( $name ) {
 234                 Assert::parameterType( 'string', $name, '$name' );
 235                 $name = $this->normalizeName( $name );
 236
 237                 $table = $this->getTableFromCachesOrReplica();
 238                 $searchResult = array_search( $name, $table, true );
 239
 240                 if ( $searchResult !== false ) {
 241                         return $searchResult;
 242                 }
 243
 244                 throw NameTableAccessException::newFromDetails( $this->table, 'name', $name );
 245         }
 246
 247         /**
 248          * Get the name of the given id.
 249          * If the id doesn't exist this will throw.
 250          * This should be used in cases where we believe the id already exists.
 251          *
 252          * Note: Calls to this method will result in a master select for non existing IDs.
 253          *
 254          * @param int $id
 255          * @throws NameTableAccessException The id does not exist
 256          * @return string name
 257          */
 258         public function getName( $id ) {
 259                 Assert::parameterType( 'integer', $id, '$id' );
 260
 261                 $table = $this->getTableFromCachesOrReplica();
 262                 if ( array_key_exists( $id, $table ) ) {
 263                         return $table[$id];
 264                 }
 265
 266                 $table = $this->cache->getWithSetCallback(
 267                         $this->getCacheKey(),
 268                         $this->cacheTTL,
 269                         function ( $oldValue, &$ttl, &$setOpts ) use ( $id ) {
 270                                 // Check if cached value is up-to-date enough to have $id
 271                                 if ( is_array( $oldValue ) && array_key_exists( $id, $oldValue ) ) {
 272                                         // Completely leave the cache key alone
 273                                         $ttl = WANObjectCache::TTL_UNCACHEABLE;
 274                                         // Use the old value
 275                                         return $oldValue;
 276                                 }
 277                                 // Regenerate from replica DB, and master DB if needed
 278                                 foreach ( [ DB_REPLICA, DB_MASTER ] as $source ) {
 279                                         // Log a fallback to master
 280                                         if ( $source === DB_MASTER ) {
 281                                                 $this->logger->info(
 282                                                         __METHOD__ . 'falling back to master select from ' .
 283                                                         $this->table . ' with id ' . $id
 284                                                 );
 285                                         }
 286                                         $db = $this->getDBConnection( $source );
 287                                         $cacheSetOpts = Database::getCacheSetOptions( $db );
 288                                         $table = $this->loadTable( $db );
 289                                         if ( array_key_exists( $id, $table ) ) {
 290                                                 break; // found it
 291                                         }
 292                                 }
 293                                 // Use the value from last source checked
 294                                 $setOpts += $cacheSetOpts;
 295
 296                                 return $table;
 297                         },
 298                         [ 'minAsOf' => INF ] // force callback run
 299                 );
 300
 301                 $this->tableCache = $table;
 302
 303                 if ( array_key_exists( $id, $table ) ) {
 304                         return $table[$id];
 305                 }
 306
 307                 throw NameTableAccessException::newFromDetails( $this->table, 'id', $id );
 308         }
 309
 310         /**
 311          * Get the whole table, in no particular order as a map of ids to names.
 312          * This method could be subject to DB or cache lag.
 313          *
 314          * @return string[] keys are the name ids, values are the names themselves
 315          *  Example: [ 1 => 'foo', 3 => 'bar' ]
 316          */
 317         public function getMap() {
 318                 return $this->getTableFromCachesOrReplica();
 319         }
 320
 321         /**
 322          * @return string[]
 323          */
 324         private function getTableFromCachesOrReplica() {
 325                 if ( $this->tableCache !== null ) {
 326                         return $this->tableCache;
 327                 }
 328
 329                 $table = $this->cache->getWithSetCallback(
 330                         $this->getCacheKey(),
 331                         $this->cacheTTL,
 332                         function ( $oldValue, &$ttl, &$setOpts ) {
 333                                 $dbr = $this->getDBConnection( DB_REPLICA );
 334                                 $setOpts += Database::getCacheSetOptions( $dbr );
 335                                 return $this->loadTable( $dbr );
 336                         }
 337                 );
 338
 339                 $this->tableCache = $table;
 340
 341                 return $table;
 342         }
 343
 344         /**
 345          * Reap the WANCache entry for this table.
 346          *
 347          * @param callable $purgeCallback callback to 'purge' the WAN cache
 348          */
 349         private function purgeWANCache( $purgeCallback ) {
 350                 // If the LB has no DB changes don't both with onTransactionPreCommitOrIdle
 351                 if ( !$this->loadBalancer->hasOrMadeRecentMasterChanges() ) {
 352                         $purgeCallback();
 353                         return;
 354                 }
 355
 356                 $this->getDBConnection( DB_MASTER )
 357                         ->onTransactionPreCommitOrIdle( $purgeCallback, __METHOD__ );
 358         }
 359
 360         /**
 361          * Gets the table from the db
 362          *
 363          * @param IDatabase $db
 364          *
 365          * @return string[]
 366          */
 367         private function loadTable( IDatabase $db ) {
 368                 $result = $db->select(
 369                         $this->table,
 370                         [
 371                                 'id' => $this->idField,
 372                                 'name' => $this->nameField
 373                         ],
 374                         [],
 375                         __METHOD__,
 376                         [ 'ORDER BY' => 'id' ]
 377                 );
 378
 379                 $assocArray = [];
 380                 foreach ( $result as $row ) {
 381                         $assocArray[$row->id] = $row->name;
 382                 }
 383
 384                 return $assocArray;
 385         }
 386
 387         /**
 388          * Stores the given name in the DB, returning the ID when an insert occurs.
 389          *
 390          * @param string $name
 391          * @return int|null int if we know the ID, null if we don't
 392          */
 393         private function store( $name ) {
 394                 Assert::parameterType( 'string', $name, '$name' );
 395                 Assert::parameter( $name !== '', '$name', 'should not be an empty string' );
 396                 // Note: this is only called internally so normalization of $name has already occurred.
 397
 398                 $dbw = $this->getDBConnection( DB_MASTER );
 399
 400                 $dbw->insert(
 401                         $this->table,
 402                         $this->getFieldsToStore( $name ),
 403                         __METHOD__,
 404                         [ 'IGNORE' ]
 405                 );
 406
 407                 if ( $dbw->affectedRows() === 0 ) {
 408                         $this->logger->info(
 409                                 'Tried to insert name into table ' . $this->table . ', but value already existed.'
 410                         );
 411                         return null;
 412                 }
 413
 414                 return $dbw->insertId();
 415         }
 416
 417         /**
 418          * @param string $name
 419          * @return array
 420          */
 421         private function getFieldsToStore( $name ) {
 422                 $fields = [ $this->nameField => $name ];
 423                 if ( $this->insertCallback !== null ) {
 424                         $fields = call_user_func( $this->insertCallback, $fields );
 425                 }
 426                 return $fields;
 427         }
 428
 429 }