tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 use MediaWiki\MediaWikiServices;
   4
   5 /**
   6  * @group JobQueue
   7  * @group medium
   8  * @group Database
   9  */
  10 class JobQueueTest extends MediaWikiTestCase {
  11         protected $key;
  12         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  13
  14         function __construct( $name = null, array $data = [], $dataName = '' ) {
  15                 parent::__construct( $name, $data, $dataName );
  16
  17                 $this->tablesUsed[] = 'job';
  18         }
  19
  20         protected function setUp() {
  21                 global $wgJobTypeConf;
  22                 parent::setUp();
  23
  24                 if ( $this->getCliArg( 'use-jobqueue' ) ) {
  25                         $name = $this->getCliArg( 'use-jobqueue' );
  26                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  27                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  28                         }
  29                         $baseConfig = $wgJobTypeConf[$name];
  30                 } else {
  31                         $baseConfig = [ 'class' => JobQueueDBSingle::class ];
  32                 }
  33                 $baseConfig['type'] = 'null';
  34                 $baseConfig['wiki'] = wfWikiID();
  35                 $variants = [
  36                         'queueRand' => [ 'order' => 'random', 'claimTTL' => 0 ],
  37                         'queueRandTTL' => [ 'order' => 'random', 'claimTTL' => 10 ],
  38                         'queueTimestamp' => [ 'order' => 'timestamp', 'claimTTL' => 0 ],
  39                         'queueTimestampTTL' => [ 'order' => 'timestamp', 'claimTTL' => 10 ],
  40                         'queueFifo' => [ 'order' => 'fifo', 'claimTTL' => 0 ],
  41                         'queueFifoTTL' => [ 'order' => 'fifo', 'claimTTL' => 10 ],
  42                 ];
  43                 foreach ( $variants as $q => $settings ) {
  44                         try {
  45                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  46                         } catch ( MWException $e ) {
  47                                 // unsupported?
  48                                 // @todo What if it was another error?
  49                         };
  50                 }
  51         }
  52
  53         protected function tearDown() {
  54                 parent::tearDown();
  55                 foreach (
  56                         [
  57                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  58                                 'queueFifo', 'queueFifoTTL'
  59                         ] as $q
  60                 ) {
  61                         if ( $this->$q ) {
  62                                 $this->$q->delete();
  63                         }
  64                         $this->$q = null;
  65                 }
  66         }
  67
  68         /**
  69          * @dataProvider provider_queueLists
  70          * @covers JobQueue::getWiki
  71          */
  72         public function testGetWiki( $queue, $recycles, $desc ) {
  73                 $queue = $this->$queue;
  74                 if ( !$queue ) {
  75                         $this->markTestSkipped( $desc );
  76                 }
  77                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  78         }
  79
  80         /**
  81          * @dataProvider provider_queueLists
  82          * @covers JobQueue::getType
  83          */
  84         public function testGetType( $queue, $recycles, $desc ) {
  85                 $queue = $this->$queue;
  86                 if ( !$queue ) {
  87                         $this->markTestSkipped( $desc );
  88                 }
  89                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  90         }
  91
  92         /**
  93          * @dataProvider provider_queueLists
  94          * @covers JobQueue
  95          */
  96         public function testBasicOperations( $queue, $recycles, $desc ) {
  97                 $queue = $this->$queue;
  98                 if ( !$queue ) {
  99                         $this->markTestSkipped( $desc );
 100                 }
 101
 102                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 103
 104                 $queue->flushCaches();
 105                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 106                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 107
 108                 $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 109                 $this->assertNull( $queue->batchPush( [ $this->newJob() ] ), "Push worked ($desc)" );
 110
 111                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 112
 113                 $queue->flushCaches();
 114                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 115                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 116                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 117                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 118
 119                 $job1 = $queue->pop();
 120                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 121
 122                 $queue->flushCaches();
 123                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 124
 125                 $queue->flushCaches();
 126                 if ( $recycles ) {
 127                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 128                 }
 129
 130                 $job2 = $queue->pop();
 131                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 132                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 133
 134                 $queue->flushCaches();
 135                 if ( $recycles ) {
 136                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 137                 }
 138
 139                 $queue->ack( $job1 );
 140
 141                 $queue->flushCaches();
 142                 if ( $recycles ) {
 143                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 144                 }
 145
 146                 $queue->ack( $job2 );
 147
 148                 $queue->flushCaches();
 149                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 150
 151                 $this->assertNull( $queue->batchPush( [ $this->newJob(), $this->newJob() ] ),
 152                         "Push worked ($desc)" );
 153                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 154
 155                 $queue->delete();
 156                 $queue->flushCaches();
 157                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 158                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 159         }
 160
 161         /**
 162          * @dataProvider provider_queueLists
 163          * @covers JobQueue
 164          */
 165         public function testBasicDeduplication( $queue, $recycles, $desc ) {
 166                 $queue = $this->$queue;
 167                 if ( !$queue ) {
 168                         $this->markTestSkipped( $desc );
 169                 }
 170
 171                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 172
 173                 $queue->flushCaches();
 174                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 175                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 176
 177                 $this->assertNull(
 178                         $queue->batchPush(
 179                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 180                         ),
 181                         "Push worked ($desc)" );
 182
 183                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 184
 185                 $queue->flushCaches();
 186                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 187                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 188
 189                 $this->assertNull(
 190                         $queue->batchPush(
 191                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 192                         ),
 193                         "Push worked ($desc)"
 194                 );
 195
 196                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 197
 198                 $queue->flushCaches();
 199                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 200                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 201
 202                 $job1 = $queue->pop();
 203                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 204
 205                 $queue->flushCaches();
 206                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 207                 if ( $recycles ) {
 208                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 209                 }
 210
 211                 $queue->ack( $job1 );
 212
 213                 $queue->flushCaches();
 214                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 215         }
 216
 217         /**
 218          * @dataProvider provider_queueLists
 219          * @covers JobQueue
 220          */
 221         public function testDeduplicationWhileClaimed( $queue, $recycles, $desc ) {
 222                 $queue = $this->$queue;
 223                 if ( !$queue ) {
 224                         $this->markTestSkipped( $desc );
 225                 }
 226
 227                 $job = $this->newDedupedJob();
 228                 $queue->push( $job );
 229
 230                 // De-duplication does not apply to already-claimed jobs
 231                 $j = $queue->pop();
 232                 $queue->push( $job );
 233                 $queue->ack( $j );
 234
 235                 $j = $queue->pop();
 236                 // Make sure ack() of the twin did not delete the sibling data
 237                 $this->assertType( NullJob::class, $j );
 238         }
 239
 240         /**
 241          * @dataProvider provider_queueLists
 242          * @covers JobQueue
 243          */
 244         public function testRootDeduplication( $queue, $recycles, $desc ) {
 245                 $queue = $this->$queue;
 246                 if ( !$queue ) {
 247                         $this->markTestSkipped( $desc );
 248                 }
 249
 250                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 251
 252                 $queue->flushCaches();
 253                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 254                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 255
 256                 $id = wfRandomString( 32 );
 257                 $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 258                 for ( $i = 0; $i < 5; ++$i ) {
 259                         $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 260                 }
 261                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 262
 263                 $root2 = $root1;
 264                 # Add a second to UNIX epoch and format back to TS_MW
 265                 $root2_ts = strtotime( $root2['rootJobTimestamp'] );
 266                 $root2_ts++;
 267                 $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
 268
 269                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 270                         "Root job signatures have different timestamps." );
 271                 for ( $i = 0; $i < 5; ++$i ) {
 272                         $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 273                 }
 274                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 275
 276                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 277
 278                 $queue->flushCaches();
 279                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 280                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 281
 282                 $dupcount = 0;
 283                 $jobs = [];
 284                 do {
 285                         $job = $queue->pop();
 286                         if ( $job ) {
 287                                 $jobs[] = $job;
 288                                 $queue->ack( $job );
 289                         }
 290                         if ( $job instanceof DuplicateJob ) {
 291                                 ++$dupcount;
 292                         }
 293                 } while ( $job );
 294
 295                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 296                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 297         }
 298
 299         /**
 300          * @dataProvider provider_fifoQueueLists
 301          * @covers JobQueue
 302          */
 303         public function testJobOrder( $queue, $recycles, $desc ) {
 304                 $queue = $this->$queue;
 305                 if ( !$queue ) {
 306                         $this->markTestSkipped( $desc );
 307                 }
 308
 309                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 310
 311                 $queue->flushCaches();
 312                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 313                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 314
 315                 for ( $i = 0; $i < 10; ++$i ) {
 316                         $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 317                 }
 318
 319                 for ( $i = 0; $i < 10; ++$i ) {
 320                         $job = $queue->pop();
 321                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 322                         $params = $job->getParams();
 323                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 324                         $queue->ack( $job );
 325                 }
 326
 327                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 328
 329                 $queue->flushCaches();
 330                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 331                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 332         }
 333
 334         /**
 335          * @covers JobQueue
 336          */
 337         public function testQueueAggregateTable() {
 338                 $queue = $this->queueFifo;
 339                 if ( !$queue || !method_exists( $queue, 'getServerQueuesWithJobs' ) ) {
 340                         $this->markTestSkipped();
 341                 }
 342
 343                 $this->assertNotContains(
 344                         [ $queue->getType(), $queue->getWiki() ],
 345                         $queue->getServerQueuesWithJobs(),
 346                         "Null queue not in listing"
 347                 );
 348
 349                 $queue->push( $this->newJob( 0 ) );
 350
 351                 $this->assertContains(
 352                         [ $queue->getType(), $queue->getWiki() ],
 353                         $queue->getServerQueuesWithJobs(),
 354                         "Null queue in listing"
 355                 );
 356         }
 357
 358         public static function provider_queueLists() {
 359                 return [
 360                         [ 'queueRand', false, 'Random queue without ack()' ],
 361                         [ 'queueRandTTL', true, 'Random queue with ack()' ],
 362                         [ 'queueTimestamp', false, 'Time ordered queue without ack()' ],
 363                         [ 'queueTimestampTTL', true, 'Time ordered queue with ack()' ],
 364                         [ 'queueFifo', false, 'FIFO ordered queue without ack()' ],
 365                         [ 'queueFifoTTL', true, 'FIFO ordered queue with ack()' ]
 366                 ];
 367         }
 368
 369         public static function provider_fifoQueueLists() {
 370                 return [
 371                         [ 'queueFifo', false, 'Ordered queue without ack()' ],
 372                         [ 'queueFifoTTL', true, 'Ordered queue with ack()' ]
 373                 ];
 374         }
 375
 376         function newJob( $i = 0, $rootJob = [] ) {
 377                 return new NullJob( Title::newMainPage(),
 378                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ] + $rootJob );
 379         }
 380
 381         function newDedupedJob( $i = 0, $rootJob = [] ) {
 382                 return new NullJob( Title::newMainPage(),
 383                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ] + $rootJob );
 384         }
 385 }
 386
 387 class JobQueueDBSingle extends JobQueueDB {
 388         protected function getDB( $index ) {
 389                 $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
 390                 // Override to not use CONN_TRX_AUTO so that we see the same temporary `job` table
 391                 return $lb->getConnection( $index, [], $this->wiki );
 392         }
 393 }