tests/phpunit/includes/Storage/NameTableStoreTest.php

   1 <?php
   2
   3 namespace MediaWiki\Tests\Storage;
   4
   5 use BagOStuff;
   6 use EmptyBagOStuff;
   7 use HashBagOStuff;
   8 use MediaWiki\MediaWikiServices;
   9 use MediaWiki\Storage\NameTableAccessException;
  10 use MediaWiki\Storage\NameTableStore;
  11 use MediaWikiTestCase;
  12 use PHPUnit\Framework\MockObject\MockObject;
  13 use Psr\Log\NullLogger;
  14 use RuntimeException;
  15 use WANObjectCache;
  16 use Wikimedia\Rdbms\IDatabase;
  17 use Wikimedia\Rdbms\LoadBalancer;
  18 use Wikimedia\Rdbms\MaintainableDBConnRef;
  19 use Wikimedia\TestingAccessWrapper;
  20
  21 /**
  22  * @author Addshore
  23  * @group Database
  24  * @covers \MediaWiki\Storage\NameTableStore
  25  */
  26 class NameTableStoreTest extends MediaWikiTestCase {
  27
  28         public function setUp() {
  29                 $this->tablesUsed[] = 'slot_roles';
  30                 parent::setUp();
  31         }
  32
  33         protected function addCoreDBData() {
  34                 // The default implementation causes the slot_roles to already have content. Skip that.
  35         }
  36
  37         private function populateTable( $values ) {
  38                 $insertValues = [];
  39                 foreach ( $values as $name ) {
  40                         $insertValues[] = [ 'role_name' => $name ];
  41                 }
  42                 $this->db->insert( 'slot_roles', $insertValues );
  43         }
  44
  45         private function getHashWANObjectCache( $cacheBag ) {
  46                 return new WANObjectCache( [ 'cache' => $cacheBag ] );
  47         }
  48
  49         /**
  50          * @param $db
  51          * @return \PHPUnit_Framework_MockObject_MockObject|LoadBalancer
  52          */
  53         private function getMockLoadBalancer( $db ) {
  54                 $mock = $this->getMockBuilder( LoadBalancer::class )
  55                         ->disableOriginalConstructor()
  56                         ->getMock();
  57                 $mock->expects( $this->any() )
  58                         ->method( 'getConnectionRef' )
  59                         ->willReturnCallback( function ( $i ) use ( $mock, $db ) {
  60                                 return new MaintainableDBConnRef( $mock, $db, $i );
  61                         } );
  62                 return $mock;
  63         }
  64
  65         /**
  66          * @param null $insertCalls
  67          * @param null $selectCalls
  68          *
  69          * @return MockObject|IDatabase
  70          */
  71         private function getProxyDb( $insertCalls = null, $selectCalls = null ) {
  72                 $proxiedMethods = [
  73                         'select' => $selectCalls,
  74                         'insert' => $insertCalls,
  75                         'affectedRows' => null,
  76                         'insertId' => null,
  77                         'getSessionLagStatus' => null,
  78                         'writesPending' => null,
  79                         'onTransactionPreCommitOrIdle' => null,
  80                         'onAtomicSectionCancel' => null,
  81                         'doAtomicSection' => null,
  82                         'begin' => null,
  83                         'rollback' => null,
  84                         'commit' => null,
  85                 ];
  86                 $mock = $this->getMockBuilder( IDatabase::class )
  87                         ->disableOriginalConstructor()
  88                         ->getMock();
  89                 foreach ( $proxiedMethods as $method => $count ) {
  90                         $mock->expects( is_int( $count ) ? $this->exactly( $count ) : $this->any() )
  91                                 ->method( $method )
  92                                 ->willReturnCallback( function ( ...$args ) use ( $method ) {
  93                                         return call_user_func_array( [ $this->db, $method ], $args );
  94                                 } );
  95                 }
  96                 return $mock;
  97         }
  98
  99         private function getNameTableSqlStore(
 100                 BagOStuff $cacheBag,
 101                 $insertCalls,
 102                 $selectCalls,
 103                 $normalizationCallback = null,
 104                 $insertCallback = null
 105         ) {
 106                 return new NameTableStore(
 107                         $this->getMockLoadBalancer( $this->getProxyDb( $insertCalls, $selectCalls ) ),
 108                         $this->getHashWANObjectCache( $cacheBag ),
 109                         new NullLogger(),
 110                         'slot_roles', 'role_id', 'role_name',
 111                         $normalizationCallback,
 112                         false,
 113                         $insertCallback
 114                 );
 115         }
 116
 117         public function provideGetAndAcquireId() {
 118                 return [
 119                         'no wancache, empty table' =>
 120                                 [ new EmptyBagOStuff(), true, 1, [], 'foo', 1 ],
 121                         'no wancache, one matching value' =>
 122                                 [ new EmptyBagOStuff(), false, 1, [ 'foo' ], 'foo', 1 ],
 123                         'no wancache, one not matching value' =>
 124                                 [ new EmptyBagOStuff(), true, 1, [ 'bar' ], 'foo', 2 ],
 125                         'no wancache, multiple, one matching value' =>
 126                                 [ new EmptyBagOStuff(), false, 1, [ 'foo', 'bar' ], 'bar', 2 ],
 127                         'no wancache, multiple, no matching value' =>
 128                                 [ new EmptyBagOStuff(), true, 1, [ 'foo', 'bar' ], 'baz', 3 ],
 129                         'wancache, empty table' =>
 130                                 [ new HashBagOStuff(), true, 1, [], 'foo', 1 ],
 131                         'wancache, one matching value' =>
 132                                 [ new HashBagOStuff(), false, 1, [ 'foo' ], 'foo', 1 ],
 133                         'wancache, one not matching value' =>
 134                                 [ new HashBagOStuff(), true, 1, [ 'bar' ], 'foo', 2 ],
 135                         'wancache, multiple, one matching value' =>
 136                                 [ new HashBagOStuff(), false, 1, [ 'foo', 'bar' ], 'bar', 2 ],
 137                         'wancache, multiple, no matching value' =>
 138                                 [ new HashBagOStuff(), true, 1, [ 'foo', 'bar' ], 'baz', 3 ],
 139                 ];
 140         }
 141
 142         /**
 143          * @dataProvider provideGetAndAcquireId
 144          * @param BagOStuff $cacheBag to use in the WANObjectCache service
 145          * @param bool $needsInsert Does the value we are testing need to be inserted?
 146          * @param int $selectCalls Number of times the select DB method will be called
 147          * @param string[] $existingValues to be added to the db table
 148          * @param string $name name to acquire
 149          * @param int $expectedId the id we expect the name to have
 150          */
 151         public function testGetAndAcquireId(
 152                 $cacheBag,
 153                 $needsInsert,
 154                 $selectCalls,
 155                 $existingValues,
 156                 $name,
 157                 $expectedId
 158         ) {
 159                 // Make sure the table is empty!
 160                 $this->truncateTable( 'slot_roles' );
 161
 162                 $this->populateTable( $existingValues );
 163                 $store = $this->getNameTableSqlStore( $cacheBag, (int)$needsInsert, $selectCalls );
 164
 165                 // Some names will not initially exist
 166                 try {
 167                         $result = $store->getId( $name );
 168                         $this->assertSame( $expectedId, $result );
 169                 } catch ( NameTableAccessException $e ) {
 170                         if ( $needsInsert ) {
 171                                 $this->assertTrue( true ); // Expected exception
 172                         } else {
 173                                 $this->fail( 'Did not expect an exception, but got one: ' . $e->getMessage() );
 174                         }
 175                 }
 176
 177                 // All names should return their id here
 178                 $this->assertSame( $expectedId, $store->acquireId( $name ) );
 179
 180                 // acquireId inserted these names, so now everything should exist with getId
 181                 $this->assertSame( $expectedId, $store->getId( $name ) );
 182
 183                 // calling getId again will also still work, and not result in more selects
 184                 $this->assertSame( $expectedId, $store->getId( $name ) );
 185         }
 186
 187         public function provideTestGetAndAcquireIdNameNormalization() {
 188                 yield [ 'A', 'a', 'strtolower' ];
 189                 yield [ 'b', 'B', 'strtoupper' ];
 190                 yield [
 191                         'X',
 192                         'X',
 193                         function ( $name ) {
 194                                 return $name;
 195                         }
 196                 ];
 197                 yield [ 'ZZ', 'ZZ-a', __CLASS__ . '::appendDashAToString' ];
 198         }
 199
 200         public static function appendDashAToString( $string ) {
 201                 return $string . '-a';
 202         }
 203
 204         /**
 205          * @dataProvider provideTestGetAndAcquireIdNameNormalization
 206          */
 207         public function testGetAndAcquireIdNameNormalization(
 208                 $nameIn,
 209                 $nameOut,
 210                 $normalizationCallback
 211         ) {
 212                 $store = $this->getNameTableSqlStore(
 213                         new EmptyBagOStuff(),
 214                         1,
 215                         1,
 216                         $normalizationCallback
 217                 );
 218                 $acquiredId = $store->acquireId( $nameIn );
 219                 $this->assertSame( $nameOut, $store->getName( $acquiredId ) );
 220         }
 221
 222         public function provideGetName() {
 223                 return [
 224                         [ new HashBagOStuff(), 3, 2 ],
 225                         [ new EmptyBagOStuff(), 3, 3 ],
 226                 ];
 227         }
 228
 229         /**
 230          * @dataProvider provideGetName
 231          */
 232         public function testGetName( BagOStuff $cacheBag, $insertCalls, $selectCalls ) {
 233                 $now = microtime( true );
 234                 $cacheBag->setMockTime( $now );
 235                 // Check for operations to in-memory cache (IMC) and persistent cache (PC)
 236                 $store = $this->getNameTableSqlStore( $cacheBag, $insertCalls, $selectCalls );
 237
 238                 // Get 1 ID and make sure getName returns correctly
 239                 $fooId = $store->acquireId( 'foo' ); // regen PC, set IMC, update IMC, tombstone PC
 240                 $now += 0.01;
 241                 $this->assertSame( 'foo', $store->getName( $fooId ) ); // use IMC
 242                 $now += 0.01;
 243
 244                 // Get another ID and make sure getName returns correctly
 245                 $barId = $store->acquireId( 'bar' ); // update IMC, tombstone PC
 246                 $now += 0.01;
 247                 $this->assertSame( 'bar', $store->getName( $barId ) ); // use IMC
 248                 $now += 0.01;
 249
 250                 // Blitz the cache and make sure it still returns
 251                 TestingAccessWrapper::newFromObject( $store )->tableCache = null; // clear IMC
 252                 $this->assertSame( 'foo', $store->getName( $fooId ) ); // regen interim PC, set IMC
 253                 $this->assertSame( 'bar', $store->getName( $barId ) ); // use IMC
 254
 255                 // Blitz the cache again and get another ID and make sure getName returns correctly
 256                 TestingAccessWrapper::newFromObject( $store )->tableCache = null; // clear IMC
 257                 $bazId = $store->acquireId( 'baz' ); // set IMC using interim PC, update IMC, tombstone PC
 258                 $now += 0.01;
 259                 $this->assertSame( 'baz', $store->getName( $bazId ) ); // uses IMC
 260                 $this->assertSame( 'baz', $store->getName( $bazId ) ); // uses IMC
 261         }
 262
 263         public function testGetName_masterFallback() {
 264                 $store = $this->getNameTableSqlStore( new EmptyBagOStuff(), 1, 2 );
 265
 266                 // Insert a new name
 267                 $fooId = $store->acquireId( 'foo' );
 268
 269                 // Empty the process cache, getCachedTable() will now return this empty array
 270                 TestingAccessWrapper::newFromObject( $store )->tableCache = [];
 271
 272                 // getName should fallback to master, which is why we assert 2 selectCalls above
 273                 $this->assertSame( 'foo', $store->getName( $fooId ) );
 274         }
 275
 276         public function testGetMap_empty() {
 277                 $this->populateTable( [] );
 278                 $store = $this->getNameTableSqlStore( new HashBagOStuff(), 0, 1 );
 279                 $table = $store->getMap();
 280                 $this->assertSame( [], $table );
 281         }
 282
 283         public function testGetMap_twoValues() {
 284                 $this->populateTable( [ 'foo', 'bar' ] );
 285                 $store = $this->getNameTableSqlStore( new HashBagOStuff(), 0, 1 );
 286
 287                 // We are using a cache, so 2 calls should only result in 1 select on the db
 288                 $store->getMap();
 289                 $table = $store->getMap();
 290
 291                 $expected = [ 1 => 'foo', 2 => 'bar' ];
 292                 $this->assertSame( $expected, $table );
 293                 // Make sure the table returned is the same as the cached table
 294                 $this->assertSame( $expected, TestingAccessWrapper::newFromObject( $store )->tableCache );
 295         }
 296
 297         public function testReloadMap() {
 298                 $this->populateTable( [ 'foo' ] );
 299                 $store = $this->getNameTableSqlStore( new HashBagOStuff(), 0, 2 );
 300
 301                 // force load
 302                 $this->assertCount( 1, $store->getMap() );
 303
 304                 // add more stuff to the table, so the cache gets out of sync
 305                 $this->populateTable( [ 'bar' ] );
 306
 307                 $expected = [ 1 => 'foo', 2 => 'bar' ];
 308                 $this->assertSame( $expected, $store->reloadMap() );
 309                 $this->assertSame( $expected, $store->getMap() );
 310         }
 311
 312         public function testCacheRaceCondition() {
 313                 $wanHashBag = new HashBagOStuff();
 314                 $store1 = $this->getNameTableSqlStore( $wanHashBag, 1, 1 );
 315                 $store2 = $this->getNameTableSqlStore( $wanHashBag, 1, 0 );
 316                 $store3 = $this->getNameTableSqlStore( $wanHashBag, 1, 1 );
 317
 318                 // Cache the current table in the instances we will use
 319                 // This simulates multiple requests running simultaneously
 320                 $store1->getMap();
 321                 $store2->getMap();
 322                 $store3->getMap();
 323
 324                 // Store 2 separate names using different instances
 325                 $fooId = $store1->acquireId( 'foo' );
 326                 $barId = $store2->acquireId( 'bar' );
 327
 328                 // Each of these instances should be aware of what they have inserted
 329                 $this->assertSame( $fooId, $store1->acquireId( 'foo' ) );
 330                 $this->assertSame( $barId, $store2->acquireId( 'bar' ) );
 331
 332                 // A new store should be able to get both of these new Ids
 333                 // Note: before there was a race condition here where acquireId( 'bar' ) would update the
 334                 //       cache with data missing the 'foo' key that it was not aware of
 335                 $store4 = $this->getNameTableSqlStore( $wanHashBag, 0, 1 );
 336                 $this->assertSame( $fooId, $store4->getId( 'foo' ) );
 337                 $this->assertSame( $barId, $store4->getId( 'bar' ) );
 338
 339                 // If a store with old cached data tries to acquire these we will get the same ids.
 340                 $this->assertSame( $fooId, $store3->acquireId( 'foo' ) );
 341                 $this->assertSame( $barId, $store3->acquireId( 'bar' ) );
 342         }
 343
 344         public function testGetAndAcquireIdInsertCallback() {
 345                 // FIXME: fails under postgres
 346                 $this->markTestSkippedIfDbType( 'postgres' );
 347
 348                 $store = $this->getNameTableSqlStore(
 349                         new EmptyBagOStuff(),
 350                         1,
 351                         1,
 352                         null,
 353                         function ( $insertFields ) {
 354                                 $insertFields['role_id'] = 7251;
 355                                 return $insertFields;
 356                         }
 357                 );
 358                 $this->assertSame( 7251, $store->acquireId( 'A' ) );
 359         }
 360
 361         public function testTransactionRollback() {
 362                 $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
 363
 364                 // Two instances hitting the real database using separate caches.
 365                 $store1 = new NameTableStore(
 366                         $lb,
 367                         $this->getHashWANObjectCache( new HashBagOStuff() ),
 368                         new NullLogger(),
 369                         'slot_roles', 'role_id', 'role_name'
 370                 );
 371                 $store2 = new NameTableStore(
 372                         $lb,
 373                         $this->getHashWANObjectCache( new HashBagOStuff() ),
 374                         new NullLogger(),
 375                         'slot_roles', 'role_id', 'role_name'
 376                 );
 377
 378                 $this->db->begin( __METHOD__ );
 379                 $fooId = $store1->acquireId( 'foo' );
 380                 $this->db->rollback( __METHOD__ );
 381
 382                 $this->assertSame( $fooId, $store2->getId( 'foo' ) );
 383                 $this->assertSame( $fooId, $store1->getId( 'foo' ) );
 384         }
 385
 386         public function testTransactionRollbackWithFailedRedo() {
 387                 $insertCalls = 0;
 388
 389                 $db = $this->getProxyDb( 2 );
 390                 $db->method( 'insert' )
 391                         ->willReturnCallback( function () use ( &$insertCalls, $db ) {
 392                                 $insertCalls++;
 393                                 switch ( $insertCalls ) {
 394                                         case 1:
 395                                                 return true;
 396                                         case 2:
 397                                                 throw new RuntimeException( 'Testing' );
 398                                 }
 399
 400                                 return true;
 401                         } );
 402
 403                 $lb = $this->getMockBuilder( LoadBalancer::class )
 404                         ->disableOriginalConstructor()
 405                         ->getMock();
 406                 $lb->method( 'getConnectionRef' )
 407                         ->willReturn( $db );
 408                 $lb->method( 'resolveDomainID' )
 409                         ->willReturnArgument( 0 );
 410
 411                 // Two instances hitting the real database using separate caches.
 412                 $store1 = new NameTableStore(
 413                         $lb,
 414                         $this->getHashWANObjectCache( new HashBagOStuff() ),
 415                         new NullLogger(),
 416                         'slot_roles', 'role_id', 'role_name'
 417                 );
 418
 419                 $this->db->begin( __METHOD__ );
 420                 $store1->acquireId( 'foo' );
 421                 $this->db->rollback( __METHOD__ );
 422
 423                 $this->assertArrayNotHasKey( 'foo', $store1->getMap() );
 424         }
 425
 426         public function testTransactionRollbackWithInterference() {
 427                 $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
 428
 429                 // Two instances hitting the real database using separate caches.
 430                 $store1 = new NameTableStore(
 431                         $lb,
 432                         $this->getHashWANObjectCache( new HashBagOStuff() ),
 433                         new NullLogger(),
 434                         'slot_roles', 'role_id', 'role_name'
 435                 );
 436                 $store2 = new NameTableStore(
 437                         $lb,
 438                         $this->getHashWANObjectCache( new HashBagOStuff() ),
 439                         new NullLogger(),
 440                         'slot_roles', 'role_id', 'role_name'
 441                 );
 442
 443                 $this->db->begin( __METHOD__ );
 444
 445                 $quuxId = null;
 446                 $this->db->onTransactionResolution(
 447                         function () use ( $store1, &$quuxId ) {
 448                                 $quuxId = $store1->acquireId( 'quux' );
 449                         }
 450                 );
 451
 452                 $store1->acquireId( 'foo' );
 453                 $this->db->rollback( __METHOD__ );
 454
 455                 // $store2 should know about the insert by $store1
 456                 $this->assertSame( $quuxId, $store2->getId( 'quux' ) );
 457
 458                 // A "best effort" attempt was made to restore the entry for 'foo'
 459                 // after the transaction failed. This may succeed on some databases like MySQL,
 460                 // while it fails on others. Since we are giving no guarantee about this,
 461                 // the only thing we can test here is that acquireId( 'foo' ) returns an
 462                 // ID that is distinct from the ID of quux (but might be different from the
 463                 // value returned by the original call to acquireId( 'foo' ).
 464                 // Note that $store2 will not know about the ID for 'foo' acquired by $store1,
 465                 // because it's using a separate cache, and getId() does not fall back to
 466                 // checking the database.
 467                 $this->assertNotSame( $quuxId, $store1->acquireId( 'foo' ) );
 468         }
 469
 470         public function testTransactionDoubleRollback() {
 471                 $fname = __METHOD__;
 472
 473                 $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
 474                 $store = new NameTableStore(
 475                         $lb,
 476                         $this->getHashWANObjectCache( new HashBagOStuff() ),
 477                         new NullLogger(),
 478                         'slot_roles', 'role_id', 'role_name'
 479                 );
 480
 481                 // Nested atomic sections
 482                 $atomic1 = $this->db->startAtomic( $fname, $this->db::ATOMIC_CANCELABLE );
 483                 $atomic2 = $this->db->startAtomic( $fname, $this->db::ATOMIC_CANCELABLE );
 484
 485                 // Acquire ID
 486                 $id = $store->acquireId( 'foo' );
 487
 488                 // Oops, rolled back
 489                 $this->db->cancelAtomic( $fname, $atomic2 );
 490
 491                 // Should have been re-inserted
 492                 $store->reloadMap();
 493                 $this->assertSame( $id, $store->getId( 'foo' ) );
 494
 495                 // Oops, re-insert was rolled back too.
 496                 $this->db->cancelAtomic( $fname, $atomic1 );
 497
 498                 // This time, no re-insertion happened.
 499                 try {
 500                         $id2 = $store->getId( 'foo' );
 501                         $this->fail( "Expected NameTableAccessException, got $id2 (originally was $id)" );
 502                 } catch ( NameTableAccessException $ex ) {
 503                         // expected
 504                 }
 505         }
 506
 507 }