includes/Storage/NameTableStore.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20
  21 namespace MediaWiki\Storage;
  22
  23 use IExpiringStore;
  24 use Psr\Log\LoggerInterface;
  25 use WANObjectCache;
  26 use Wikimedia\Assert\Assert;
  27 use Wikimedia\Rdbms\Database;
  28 use Wikimedia\Rdbms\IDatabase;
  29 use Wikimedia\Rdbms\LoadBalancer;
  30
  31 /**
  32  * @author Addshore
  33  * @since 1.31
  34  */
  35 class NameTableStore {
  36
  37         /** @var LoadBalancer */
  38         private $loadBalancer;
  39
  40         /** @var WANObjectCache */
  41         private $cache;
  42
  43         /** @var LoggerInterface */
  44         private $logger;
  45
  46         /** @var string[] */
  47         private $tableCache = null;
  48
  49         /** @var bool|string */
  50         private $wikiId = false;
  51
  52         /** @var int */
  53         private $cacheTTL;
  54
  55         /** @var string */
  56         private $table;
  57         /** @var string */
  58         private $idField;
  59         /** @var string */
  60         private $nameField;
  61         /** @var null|callable */
  62         private $normalizationCallback = null;
  63         /** @var null|callable */
  64         private $insertCallback = null;
  65
  66         /**
  67          * @param LoadBalancer $dbLoadBalancer A load balancer for acquiring database connections
  68          * @param WANObjectCache $cache A cache manager for caching data
  69          * @param LoggerInterface $logger
  70          * @param string $table
  71          * @param string $idField
  72          * @param string $nameField
  73          * @param callable $normalizationCallback Normalization to be applied to names before being
  74          * saved or queried. This should be a callback that accepts and returns a single string.
  75          * @param bool|string $wikiId The ID of the target wiki database. Use false for the local wiki.
  76          * @param callable $insertCallback Callback to change insert fields accordingly.
  77          * This parameter was introduced in 1.32
  78          */
  79         public function __construct(
  80                 LoadBalancer $dbLoadBalancer,
  81                 WANObjectCache $cache,
  82                 LoggerInterface $logger,
  83                 $table,
  84                 $idField,
  85                 $nameField,
  86                 callable $normalizationCallback = null,
  87                 $wikiId = false,
  88                 callable $insertCallback = null
  89         ) {
  90                 $this->loadBalancer = $dbLoadBalancer;
  91                 $this->cache = $cache;
  92                 $this->logger = $logger;
  93                 $this->table = $table;
  94                 $this->idField = $idField;
  95                 $this->nameField = $nameField;
  96                 $this->normalizationCallback = $normalizationCallback;
  97                 $this->wikiId = $wikiId;
  98                 $this->cacheTTL = IExpiringStore::TTL_MONTH;
  99                 $this->insertCallback = $insertCallback;
 100         }
 101
 102         /**
 103          * @param int $index A database index, like DB_MASTER or DB_REPLICA
 104          * @param int $flags Database connection flags
 105          *
 106          * @return IDatabase
 107          */
 108         private function getDBConnection( $index, $flags = 0 ) {
 109                 return $this->loadBalancer->getConnection( $index, [], $this->wikiId, $flags );
 110         }
 111
 112         private function getCacheKey() {
 113                 return $this->cache->makeKey( 'NameTableSqlStore', $this->table, $this->wikiId );
 114         }
 115
 116         /**
 117          * @param string $name
 118          * @return string
 119          */
 120         private function normalizeName( $name ) {
 121                 if ( $this->normalizationCallback === null ) {
 122                         return $name;
 123                 }
 124                 return call_user_func( $this->normalizationCallback, $name );
 125         }
 126
 127         /**
 128          * Acquire the id of the given name.
 129          * This creates a row in the table if it doesn't already exist.
 130          *
 131          * @param string $name
 132          * @throws NameTableAccessException
 133          * @return int
 134          */
 135         public function acquireId( $name ) {
 136                 Assert::parameterType( 'string', $name, '$name' );
 137                 $name = $this->normalizeName( $name );
 138
 139                 $table = $this->getTableFromCachesOrReplica();
 140                 $searchResult = array_search( $name, $table, true );
 141                 if ( $searchResult === false ) {
 142                         $id = $this->store( $name );
 143                         if ( $id === null ) {
 144                                 // RACE: $name was already in the db, probably just inserted, so load from master
 145                                 // Use DBO_TRX to avoid missing inserts due to other threads or REPEATABLE-READs
 146                                 $table = $this->loadTable(
 147                                         $this->getDBConnection( DB_MASTER, LoadBalancer::CONN_TRX_AUTOCOMMIT )
 148                                 );
 149                                 $searchResult = array_search( $name, $table, true );
 150                                 if ( $searchResult === false ) {
 151                                         // Insert failed due to IGNORE flag, but DB_MASTER didn't give us the data
 152                                         $m = "No insert possible but master didn't give us a record for " .
 153                                                 "'{$name}' in '{$this->table}'";
 154                                         $this->logger->error( $m );
 155                                         throw new NameTableAccessException( $m );
 156                                 }
 157                                 $this->purgeWANCache(
 158                                         function () {
 159                                                 $this->cache->reap( $this->getCacheKey(), INF );
 160                                         }
 161                                 );
 162                         } else {
 163                                 $table[$id] = $name;
 164                                 $searchResult = $id;
 165                                 // As store returned an ID we know we inserted so delete from WAN cache
 166                                 $this->purgeWANCache(
 167                                         function () {
 168                                                 $this->cache->delete( $this->getCacheKey() );
 169                                         }
 170                                 );
 171                         }
 172                         $this->tableCache = $table;
 173                 }
 174
 175                 return $searchResult;
 176         }
 177
 178         /**
 179          * Get the id of the given name.
 180          * If the name doesn't exist this will throw.
 181          * This should be used in cases where we believe the name already exists or want to check for
 182          * existence.
 183          *
 184          * @param string $name
 185          * @throws NameTableAccessException The name does not exist
 186          * @return int Id
 187          */
 188         public function getId( $name ) {
 189                 Assert::parameterType( 'string', $name, '$name' );
 190                 $name = $this->normalizeName( $name );
 191
 192                 $table = $this->getTableFromCachesOrReplica();
 193                 $searchResult = array_search( $name, $table, true );
 194
 195                 if ( $searchResult !== false ) {
 196                         return $searchResult;
 197                 }
 198
 199                 throw NameTableAccessException::newFromDetails( $this->table, 'name', $name );
 200         }
 201
 202         /**
 203          * Get the name of the given id.
 204          * If the id doesn't exist this will throw.
 205          * This should be used in cases where we believe the id already exists.
 206          *
 207          * Note: Calls to this method will result in a master select for non existing IDs.
 208          *
 209          * @param int $id
 210          * @throws NameTableAccessException The id does not exist
 211          * @return string name
 212          */
 213         public function getName( $id ) {
 214                 Assert::parameterType( 'integer', $id, '$id' );
 215
 216                 $table = $this->getTableFromCachesOrReplica();
 217                 if ( array_key_exists( $id, $table ) ) {
 218                         return $table[$id];
 219                 }
 220
 221                 $table = $this->cache->getWithSetCallback(
 222                         $this->getCacheKey(),
 223                         $this->cacheTTL,
 224                         function ( $oldValue, &$ttl, &$setOpts ) use ( $id ) {
 225                                 // Check if cached value is up-to-date enough to have $id
 226                                 if ( is_array( $oldValue ) && array_key_exists( $id, $oldValue ) ) {
 227                                         // Completely leave the cache key alone
 228                                         $ttl = WANObjectCache::TTL_UNCACHEABLE;
 229                                         // Use the old value
 230                                         return $oldValue;
 231                                 }
 232                                 // Regenerate from replica DB, and master DB if needed
 233                                 foreach ( [ DB_REPLICA, DB_MASTER ] as $source ) {
 234                                         // Log a fallback to master
 235                                         if ( $source === DB_MASTER ) {
 236                                                 $this->logger->info(
 237                                                         __METHOD__ . 'falling back to master select from ' .
 238                                                         $this->table . ' with id ' . $id
 239                                                 );
 240                                         }
 241                                         $db = $this->getDBConnection( $source );
 242                                         $cacheSetOpts = Database::getCacheSetOptions( $db );
 243                                         $table = $this->loadTable( $db );
 244                                         if ( array_key_exists( $id, $table ) ) {
 245                                                 break; // found it
 246                                         }
 247                                 }
 248                                 // Use the value from last source checked
 249                                 $setOpts += $cacheSetOpts;
 250
 251                                 return $table;
 252                         },
 253                         [ 'minAsOf' => INF ] // force callback run
 254                 );
 255
 256                 $this->tableCache = $table;
 257
 258                 if ( array_key_exists( $id, $table ) ) {
 259                         return $table[$id];
 260                 }
 261
 262                 throw NameTableAccessException::newFromDetails( $this->table, 'id', $id );
 263         }
 264
 265         /**
 266          * Get the whole table, in no particular order as a map of ids to names.
 267          * This method could be subject to DB or cache lag.
 268          *
 269          * @return string[] keys are the name ids, values are the names themselves
 270          *  Example: [ 1 => 'foo', 3 => 'bar' ]
 271          */
 272         public function getMap() {
 273                 return $this->getTableFromCachesOrReplica();
 274         }
 275
 276         /**
 277          * @return string[]
 278          */
 279         private function getTableFromCachesOrReplica() {
 280                 if ( $this->tableCache !== null ) {
 281                         return $this->tableCache;
 282                 }
 283
 284                 $table = $this->cache->getWithSetCallback(
 285                         $this->getCacheKey(),
 286                         $this->cacheTTL,
 287                         function ( $oldValue, &$ttl, &$setOpts ) {
 288                                 $dbr = $this->getDBConnection( DB_REPLICA );
 289                                 $setOpts += Database::getCacheSetOptions( $dbr );
 290                                 return $this->loadTable( $dbr );
 291                         }
 292                 );
 293
 294                 $this->tableCache = $table;
 295
 296                 return $table;
 297         }
 298
 299         /**
 300          * Reap the WANCache entry for this table.
 301          *
 302          * @param callable $purgeCallback callback to 'purge' the WAN cache
 303          */
 304         private function purgeWANCache( $purgeCallback ) {
 305                 // If the LB has no DB changes don't both with onTransactionPreCommitOrIdle
 306                 if ( !$this->loadBalancer->hasOrMadeRecentMasterChanges() ) {
 307                         $purgeCallback();
 308                         return;
 309                 }
 310
 311                 $this->getDBConnection( DB_MASTER )
 312                         ->onTransactionPreCommitOrIdle( $purgeCallback, __METHOD__ );
 313         }
 314
 315         /**
 316          * Gets the table from the db
 317          *
 318          * @param IDatabase $db
 319          *
 320          * @return string[]
 321          */
 322         private function loadTable( IDatabase $db ) {
 323                 $result = $db->select(
 324                         $this->table,
 325                         [
 326                                 'id' => $this->idField,
 327                                 'name' => $this->nameField
 328                         ],
 329                         [],
 330                         __METHOD__,
 331                         [ 'ORDER BY' => 'id' ]
 332                 );
 333
 334                 $assocArray = [];
 335                 foreach ( $result as $row ) {
 336                         $assocArray[$row->id] = $row->name;
 337                 }
 338
 339                 return $assocArray;
 340         }
 341
 342         /**
 343          * Stores the given name in the DB, returning the ID when an insert occurs.
 344          *
 345          * @param string $name
 346          * @return int|null int if we know the ID, null if we don't
 347          */
 348         private function store( $name ) {
 349                 Assert::parameterType( 'string', $name, '$name' );
 350                 Assert::parameter( $name !== '', '$name', 'should not be an empty string' );
 351                 // Note: this is only called internally so normalization of $name has already occurred.
 352
 353                 $dbw = $this->getDBConnection( DB_MASTER );
 354
 355                 $dbw->insert(
 356                         $this->table,
 357                         $this->getFieldsToStore( $name ),
 358                         __METHOD__,
 359                         [ 'IGNORE' ]
 360                 );
 361
 362                 if ( $dbw->affectedRows() === 0 ) {
 363                         $this->logger->info(
 364                                 'Tried to insert name into table ' . $this->table . ', but value already existed.'
 365                         );
 366                         return null;
 367                 }
 368
 369                 return $dbw->insertId();
 370         }
 371
 372         /**
 373          * @param string $name
 374          * @return array
 375          */
 376         private function getFieldsToStore( $name ) {
 377                 $fields = [ $this->nameField => $name ];
 378                 if ( $this->insertCallback !== null ) {
 379                         $fields = call_user_func( $this->insertCallback, $fields );
 380                 }
 381                 return $fields;
 382         }
 383
 384 }