tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 use MediaWiki\MediaWikiServices;
   4
   5 /**
   6  * @group JobQueue
   7  * @group medium
   8  * @group Database
   9  */
  10 class JobQueueTest extends MediaWikiTestCase {
  11         protected $key;
  12         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  13
  14         function __construct( $name = null, array $data = [], $dataName = '' ) {
  15                 parent::__construct( $name, $data, $dataName );
  16
  17                 $this->tablesUsed[] = 'job';
  18         }
  19
  20         protected function setUp() {
  21                 global $wgJobTypeConf;
  22                 parent::setUp();
  23
  24                 if ( $this->getCliArg( 'use-jobqueue' ) ) {
  25                         $name = $this->getCliArg( 'use-jobqueue' );
  26                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  27                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  28                         }
  29                         $baseConfig = $wgJobTypeConf[$name];
  30                 } else {
  31                         $baseConfig = [ 'class' => JobQueueDBSingle::class ];
  32                 }
  33                 $baseConfig['type'] = 'null';
  34                 $baseConfig['domain'] = WikiMap::getCurrentWikiDbDomain()->getId();
  35                 $baseConfig['stash'] = new HashBagOStuff();
  36                 $baseConfig['wanCache'] = new WANObjectCache( [ 'cache' => new HashBagOStuff() ] );
  37                 $variants = [
  38                         'queueRand' => [ 'order' => 'random', 'claimTTL' => 0 ],
  39                         'queueRandTTL' => [ 'order' => 'random', 'claimTTL' => 10 ],
  40                         'queueTimestamp' => [ 'order' => 'timestamp', 'claimTTL' => 0 ],
  41                         'queueTimestampTTL' => [ 'order' => 'timestamp', 'claimTTL' => 10 ],
  42                         'queueFifo' => [ 'order' => 'fifo', 'claimTTL' => 0 ],
  43                         'queueFifoTTL' => [ 'order' => 'fifo', 'claimTTL' => 10 ],
  44                 ];
  45                 foreach ( $variants as $q => $settings ) {
  46                         try {
  47                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  48                         } catch ( MWException $e ) {
  49                                 // unsupported?
  50                                 // @todo What if it was another error?
  51                         }
  52                 }
  53         }
  54
  55         protected function tearDown() {
  56                 parent::tearDown();
  57                 foreach (
  58                         [
  59                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  60                                 'queueFifo', 'queueFifoTTL'
  61                         ] as $q
  62                 ) {
  63                         if ( $this->$q ) {
  64                                 $this->$q->delete();
  65                         }
  66                         $this->$q = null;
  67                 }
  68         }
  69
  70         /**
  71          * @dataProvider provider_queueLists
  72          * @covers JobQueue::getWiki
  73          */
  74         public function testGetWiki( $queue, $recycles, $desc ) {
  75                 $queue = $this->$queue;
  76                 if ( !$queue ) {
  77                         $this->markTestSkipped( $desc );
  78                 }
  79                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  80                 $this->assertEquals(
  81                         WikiMap::getCurrentWikiDbDomain()->getId(),
  82                         $queue->getDomain(),
  83                         "Proper wiki ID ($desc)" );
  84         }
  85
  86         /**
  87          * @dataProvider provider_queueLists
  88          * @covers JobQueue::getType
  89          */
  90         public function testGetType( $queue, $recycles, $desc ) {
  91                 $queue = $this->$queue;
  92                 if ( !$queue ) {
  93                         $this->markTestSkipped( $desc );
  94                 }
  95                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  96         }
  97
  98         /**
  99          * @dataProvider provider_queueLists
 100          * @covers JobQueue
 101          */
 102         public function testBasicOperations( $queue, $recycles, $desc ) {
 103                 $queue = $this->$queue;
 104                 if ( !$queue ) {
 105                         $this->markTestSkipped( $desc );
 106                 }
 107
 108                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 109
 110                 $queue->flushCaches();
 111                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 112                 $this->assertSame( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 113
 114                 $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 115                 $this->assertNull( $queue->batchPush( [ $this->newJob() ] ), "Push worked ($desc)" );
 116
 117                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 118
 119                 $queue->flushCaches();
 120                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 121                 $this->assertSame( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 122                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 123                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 124
 125                 $job1 = $queue->pop();
 126                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 127
 128                 $queue->flushCaches();
 129                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 130
 131                 $queue->flushCaches();
 132                 if ( $recycles ) {
 133                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 134                 }
 135
 136                 $job2 = $queue->pop();
 137                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 138                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 139
 140                 $queue->flushCaches();
 141                 if ( $recycles ) {
 142                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 143                 }
 144
 145                 $queue->ack( $job1 );
 146
 147                 $queue->flushCaches();
 148                 if ( $recycles ) {
 149                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 150                 }
 151
 152                 $queue->ack( $job2 );
 153
 154                 $queue->flushCaches();
 155                 $this->assertSame( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 156
 157                 $this->assertNull( $queue->batchPush( [ $this->newJob(), $this->newJob() ] ),
 158                         "Push worked ($desc)" );
 159                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 160
 161                 $queue->delete();
 162                 $queue->flushCaches();
 163                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 164                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 165         }
 166
 167         /**
 168          * @dataProvider provider_queueLists
 169          * @covers JobQueue
 170          */
 171         public function testBasicDeduplication( $queue, $recycles, $desc ) {
 172                 $queue = $this->$queue;
 173                 if ( !$queue ) {
 174                         $this->markTestSkipped( $desc );
 175                 }
 176
 177                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 178
 179                 $queue->flushCaches();
 180                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 181                 $this->assertSame( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 182
 183                 $this->assertNull(
 184                         $queue->batchPush(
 185                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 186                         ),
 187                         "Push worked ($desc)" );
 188
 189                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 190
 191                 $queue->flushCaches();
 192                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 193                 $this->assertSame( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 194
 195                 $this->assertNull(
 196                         $queue->batchPush(
 197                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 198                         ),
 199                         "Push worked ($desc)"
 200                 );
 201
 202                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 203
 204                 $queue->flushCaches();
 205                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 206                 $this->assertSame( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 207
 208                 $job1 = $queue->pop();
 209                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 210
 211                 $queue->flushCaches();
 212                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 213                 if ( $recycles ) {
 214                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 215                 }
 216
 217                 $queue->ack( $job1 );
 218
 219                 $queue->flushCaches();
 220                 $this->assertSame( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 221         }
 222
 223         /**
 224          * @dataProvider provider_queueLists
 225          * @covers JobQueue
 226          */
 227         public function testDeduplicationWhileClaimed( $queue, $recycles, $desc ) {
 228                 $queue = $this->$queue;
 229                 if ( !$queue ) {
 230                         $this->markTestSkipped( $desc );
 231                 }
 232
 233                 $job = $this->newDedupedJob();
 234                 $queue->push( $job );
 235
 236                 // De-duplication does not apply to already-claimed jobs
 237                 $j = $queue->pop();
 238                 $queue->push( $job );
 239                 $queue->ack( $j );
 240
 241                 $j = $queue->pop();
 242                 // Make sure ack() of the twin did not delete the sibling data
 243                 $this->assertType( NullJob::class, $j );
 244         }
 245
 246         /**
 247          * @dataProvider provider_queueLists
 248          * @covers JobQueue
 249          */
 250         public function testRootDeduplication( $queue, $recycles, $desc ) {
 251                 $queue = $this->$queue;
 252                 if ( !$queue ) {
 253                         $this->markTestSkipped( $desc );
 254                 }
 255
 256                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 257
 258                 $queue->flushCaches();
 259                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 260                 $this->assertSame( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 261
 262                 $root1 = Job::newRootJobParams( "nulljobspam:testId" ); // task ID/timestamp
 263                 for ( $i = 0; $i < 5; ++$i ) {
 264                         $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 265                 }
 266                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 267
 268                 $root2 = $root1;
 269                 # Add a second to UNIX epoch and format back to TS_MW
 270                 $root2_ts = strtotime( $root2['rootJobTimestamp'] );
 271                 $root2_ts++;
 272                 $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
 273
 274                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 275                         "Root job signatures have different timestamps." );
 276                 for ( $i = 0; $i < 5; ++$i ) {
 277                         $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 278                 }
 279                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 280
 281                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 282
 283                 $queue->flushCaches();
 284                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 285                 $this->assertSame( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 286
 287                 $dupcount = 0;
 288                 $jobs = [];
 289                 do {
 290                         $job = $queue->pop();
 291                         if ( $job ) {
 292                                 $jobs[] = $job;
 293                                 $queue->ack( $job );
 294                         }
 295                         if ( $job instanceof DuplicateJob ) {
 296                                 ++$dupcount;
 297                         }
 298                 } while ( $job );
 299
 300                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 301                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 302         }
 303
 304         /**
 305          * @dataProvider provider_fifoQueueLists
 306          * @covers JobQueue
 307          */
 308         public function testJobOrder( $queue, $recycles, $desc ) {
 309                 $queue = $this->$queue;
 310                 if ( !$queue ) {
 311                         $this->markTestSkipped( $desc );
 312                 }
 313
 314                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 315
 316                 $queue->flushCaches();
 317                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 318                 $this->assertSame( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 319
 320                 for ( $i = 0; $i < 10; ++$i ) {
 321                         $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 322                 }
 323
 324                 for ( $i = 0; $i < 10; ++$i ) {
 325                         $job = $queue->pop();
 326                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 327                         $params = $job->getParams();
 328                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 329                         $queue->ack( $job );
 330                 }
 331
 332                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 333
 334                 $queue->flushCaches();
 335                 $this->assertSame( 0, $queue->getSize(), "Queue is empty ($desc)" );
 336                 $this->assertSame( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 337         }
 338
 339         /**
 340          * @covers JobQueue
 341          */
 342         public function testQueueAggregateTable() {
 343                 $queue = $this->queueFifo;
 344                 if ( !$queue || !method_exists( $queue, 'getServerQueuesWithJobs' ) ) {
 345                         $this->markTestSkipped();
 346                 }
 347
 348                 $this->assertNotContains(
 349                         [ $queue->getType(), $queue->getWiki() ],
 350                         $queue->getServerQueuesWithJobs(),
 351                         "Null queue not in listing"
 352                 );
 353
 354                 $queue->push( $this->newJob( 0 ) );
 355
 356                 $this->assertContains(
 357                         [ $queue->getType(), $queue->getWiki() ],
 358                         $queue->getServerQueuesWithJobs(),
 359                         "Null queue in listing"
 360                 );
 361         }
 362
 363         public static function provider_queueLists() {
 364                 return [
 365                         [ 'queueRand', false, 'Random queue without ack()' ],
 366                         [ 'queueRandTTL', true, 'Random queue with ack()' ],
 367                         [ 'queueTimestamp', false, 'Time ordered queue without ack()' ],
 368                         [ 'queueTimestampTTL', true, 'Time ordered queue with ack()' ],
 369                         [ 'queueFifo', false, 'FIFO ordered queue without ack()' ],
 370                         [ 'queueFifoTTL', true, 'FIFO ordered queue with ack()' ]
 371                 ];
 372         }
 373
 374         public static function provider_fifoQueueLists() {
 375                 return [
 376                         [ 'queueFifo', false, 'Ordered queue without ack()' ],
 377                         [ 'queueFifoTTL', true, 'Ordered queue with ack()' ]
 378                 ];
 379         }
 380
 381         function newJob( $i = 0, $rootJob = [] ) {
 382                 return Job::factory( 'null', Title::newMainPage(),
 383                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ] + $rootJob );
 384         }
 385
 386         function newDedupedJob( $i = 0, $rootJob = [] ) {
 387                 return Job::factory( 'null', Title::newMainPage(),
 388                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ] + $rootJob );
 389         }
 390 }
 391
 392 class JobQueueDBSingle extends JobQueueDB {
 393         protected function getDB( $index ) {
 394                 $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
 395                 // Override to not use CONN_TRX_AUTOCOMMIT so that we see the same temporary `job` table
 396                 return $lb->getConnection( $index, [], $this->domain );
 397         }
 398 }