tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 use MediaWiki\MediaWikiServices;
   4
   5 /**
   6  * @group JobQueue
   7  * @group medium
   8  * @group Database
   9  */
  10 class JobQueueTest extends MediaWikiTestCase {
  11         protected $key;
  12         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  13
  14         function __construct( $name = null, array $data = [], $dataName = '' ) {
  15                 parent::__construct( $name, $data, $dataName );
  16
  17                 $this->tablesUsed[] = 'job';
  18         }
  19
  20         protected function setUp() {
  21                 global $wgJobTypeConf;
  22                 parent::setUp();
  23
  24                 if ( $this->getCliArg( 'use-jobqueue' ) ) {
  25                         $name = $this->getCliArg( 'use-jobqueue' );
  26                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  27                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  28                         }
  29                         $baseConfig = $wgJobTypeConf[$name];
  30                 } else {
  31                         $baseConfig = [ 'class' => JobQueueDBSingle::class ];
  32                 }
  33                 $baseConfig['type'] = 'null';
  34                 $baseConfig['domain'] = WikiMap::getCurrentWikiDbDomain()->getId();
  35                 $baseConfig['stash'] = new HashBagOStuff();
  36                 $baseConfig['wanCache'] = new WANObjectCache( [ 'cache' => new HashBagOStuff() ] );
  37                 $variants = [
  38                         'queueRand' => [ 'order' => 'random', 'claimTTL' => 0 ],
  39                         'queueRandTTL' => [ 'order' => 'random', 'claimTTL' => 10 ],
  40                         'queueTimestamp' => [ 'order' => 'timestamp', 'claimTTL' => 0 ],
  41                         'queueTimestampTTL' => [ 'order' => 'timestamp', 'claimTTL' => 10 ],
  42                         'queueFifo' => [ 'order' => 'fifo', 'claimTTL' => 0 ],
  43                         'queueFifoTTL' => [ 'order' => 'fifo', 'claimTTL' => 10 ],
  44                 ];
  45                 foreach ( $variants as $q => $settings ) {
  46                         try {
  47                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  48                         } catch ( MWException $e ) {
  49                                 // unsupported?
  50                                 // @todo What if it was another error?
  51                         };
  52                 }
  53         }
  54
  55         protected function tearDown() {
  56                 parent::tearDown();
  57                 foreach (
  58                         [
  59                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  60                                 'queueFifo', 'queueFifoTTL'
  61                         ] as $q
  62                 ) {
  63                         if ( $this->$q ) {
  64                                 $this->$q->delete();
  65                         }
  66                         $this->$q = null;
  67                 }
  68         }
  69
  70         /**
  71          * @dataProvider provider_queueLists
  72          * @covers JobQueue::getWiki
  73          */
  74         public function testGetWiki( $queue, $recycles, $desc ) {
  75                 $queue = $this->$queue;
  76                 if ( !$queue ) {
  77                         $this->markTestSkipped( $desc );
  78                 }
  79                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  80                 $this->assertEquals(
  81                         WikiMap::getCurrentWikiDbDomain()->getId(),
  82                         $queue->getDomain(),
  83                         "Proper wiki ID ($desc)" );
  84         }
  85
  86         /**
  87          * @dataProvider provider_queueLists
  88          * @covers JobQueue::getType
  89          */
  90         public function testGetType( $queue, $recycles, $desc ) {
  91                 $queue = $this->$queue;
  92                 if ( !$queue ) {
  93                         $this->markTestSkipped( $desc );
  94                 }
  95                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  96         }
  97
  98         /**
  99          * @dataProvider provider_queueLists
 100          * @covers JobQueue
 101          */
 102         public function testBasicOperations( $queue, $recycles, $desc ) {
 103                 $queue = $this->$queue;
 104                 if ( !$queue ) {
 105                         $this->markTestSkipped( $desc );
 106                 }
 107
 108                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 109
 110                 $queue->flushCaches();
 111                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 112                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 113
 114                 $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 115                 $this->assertNull( $queue->batchPush( [ $this->newJob() ] ), "Push worked ($desc)" );
 116
 117                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 118
 119                 $queue->flushCaches();
 120                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 121                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 122                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 123                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 124
 125                 $job1 = $queue->pop();
 126                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 127
 128                 $queue->flushCaches();
 129                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 130
 131                 $queue->flushCaches();
 132                 if ( $recycles ) {
 133                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 134                 }
 135
 136                 $job2 = $queue->pop();
 137                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 138                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 139
 140                 $queue->flushCaches();
 141                 if ( $recycles ) {
 142                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 143                 }
 144
 145                 $queue->ack( $job1 );
 146
 147                 $queue->flushCaches();
 148                 if ( $recycles ) {
 149                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 150                 }
 151
 152                 $queue->ack( $job2 );
 153
 154                 $queue->flushCaches();
 155                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 156
 157                 $this->assertNull( $queue->batchPush( [ $this->newJob(), $this->newJob() ] ),
 158                         "Push worked ($desc)" );
 159                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 160
 161                 $queue->delete();
 162                 $queue->flushCaches();
 163                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 164                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 165         }
 166
 167         /**
 168          * @dataProvider provider_queueLists
 169          * @covers JobQueue
 170          */
 171         public function testBasicDeduplication( $queue, $recycles, $desc ) {
 172                 $queue = $this->$queue;
 173                 if ( !$queue ) {
 174                         $this->markTestSkipped( $desc );
 175                 }
 176
 177                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 178
 179                 $queue->flushCaches();
 180                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 181                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 182
 183                 $this->assertNull(
 184                         $queue->batchPush(
 185                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 186                         ),
 187                         "Push worked ($desc)" );
 188
 189                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 190
 191                 $queue->flushCaches();
 192                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 193                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 194
 195                 $this->assertNull(
 196                         $queue->batchPush(
 197                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 198                         ),
 199                         "Push worked ($desc)"
 200                 );
 201
 202                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 203
 204                 $queue->flushCaches();
 205                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 206                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 207
 208                 $job1 = $queue->pop();
 209                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 210
 211                 $queue->flushCaches();
 212                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 213                 if ( $recycles ) {
 214                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 215                 }
 216
 217                 $queue->ack( $job1 );
 218
 219                 $queue->flushCaches();
 220                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 221         }
 222
 223         /**
 224          * @dataProvider provider_queueLists
 225          * @covers JobQueue
 226          */
 227         public function testDeduplicationWhileClaimed( $queue, $recycles, $desc ) {
 228                 $queue = $this->$queue;
 229                 if ( !$queue ) {
 230                         $this->markTestSkipped( $desc );
 231                 }
 232
 233                 $job = $this->newDedupedJob();
 234                 $queue->push( $job );
 235
 236                 // De-duplication does not apply to already-claimed jobs
 237                 $j = $queue->pop();
 238                 $queue->push( $job );
 239                 $queue->ack( $j );
 240
 241                 $j = $queue->pop();
 242                 // Make sure ack() of the twin did not delete the sibling data
 243                 $this->assertType( NullJob::class, $j );
 244         }
 245
 246         /**
 247          * @dataProvider provider_queueLists
 248          * @covers JobQueue
 249          */
 250         public function testRootDeduplication( $queue, $recycles, $desc ) {
 251                 $queue = $this->$queue;
 252                 if ( !$queue ) {
 253                         $this->markTestSkipped( $desc );
 254                 }
 255
 256                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 257
 258                 $queue->flushCaches();
 259                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 260                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 261
 262                 $id = wfRandomString( 32 );
 263                 $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 264                 for ( $i = 0; $i < 5; ++$i ) {
 265                         $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 266                 }
 267                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 268
 269                 $root2 = $root1;
 270                 # Add a second to UNIX epoch and format back to TS_MW
 271                 $root2_ts = strtotime( $root2['rootJobTimestamp'] );
 272                 $root2_ts++;
 273                 $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
 274
 275                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 276                         "Root job signatures have different timestamps." );
 277                 for ( $i = 0; $i < 5; ++$i ) {
 278                         $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 279                 }
 280                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 281
 282                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 283
 284                 $queue->flushCaches();
 285                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 286                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 287
 288                 $dupcount = 0;
 289                 $jobs = [];
 290                 do {
 291                         $job = $queue->pop();
 292                         if ( $job ) {
 293                                 $jobs[] = $job;
 294                                 $queue->ack( $job );
 295                         }
 296                         if ( $job instanceof DuplicateJob ) {
 297                                 ++$dupcount;
 298                         }
 299                 } while ( $job );
 300
 301                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 302                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 303         }
 304
 305         /**
 306          * @dataProvider provider_fifoQueueLists
 307          * @covers JobQueue
 308          */
 309         public function testJobOrder( $queue, $recycles, $desc ) {
 310                 $queue = $this->$queue;
 311                 if ( !$queue ) {
 312                         $this->markTestSkipped( $desc );
 313                 }
 314
 315                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 316
 317                 $queue->flushCaches();
 318                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 319                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 320
 321                 for ( $i = 0; $i < 10; ++$i ) {
 322                         $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 323                 }
 324
 325                 for ( $i = 0; $i < 10; ++$i ) {
 326                         $job = $queue->pop();
 327                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 328                         $params = $job->getParams();
 329                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 330                         $queue->ack( $job );
 331                 }
 332
 333                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 334
 335                 $queue->flushCaches();
 336                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 337                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 338         }
 339
 340         /**
 341          * @covers JobQueue
 342          */
 343         public function testQueueAggregateTable() {
 344                 $queue = $this->queueFifo;
 345                 if ( !$queue || !method_exists( $queue, 'getServerQueuesWithJobs' ) ) {
 346                         $this->markTestSkipped();
 347                 }
 348
 349                 $this->assertNotContains(
 350                         [ $queue->getType(), $queue->getWiki() ],
 351                         $queue->getServerQueuesWithJobs(),
 352                         "Null queue not in listing"
 353                 );
 354
 355                 $queue->push( $this->newJob( 0 ) );
 356
 357                 $this->assertContains(
 358                         [ $queue->getType(), $queue->getWiki() ],
 359                         $queue->getServerQueuesWithJobs(),
 360                         "Null queue in listing"
 361                 );
 362         }
 363
 364         public static function provider_queueLists() {
 365                 return [
 366                         [ 'queueRand', false, 'Random queue without ack()' ],
 367                         [ 'queueRandTTL', true, 'Random queue with ack()' ],
 368                         [ 'queueTimestamp', false, 'Time ordered queue without ack()' ],
 369                         [ 'queueTimestampTTL', true, 'Time ordered queue with ack()' ],
 370                         [ 'queueFifo', false, 'FIFO ordered queue without ack()' ],
 371                         [ 'queueFifoTTL', true, 'FIFO ordered queue with ack()' ]
 372                 ];
 373         }
 374
 375         public static function provider_fifoQueueLists() {
 376                 return [
 377                         [ 'queueFifo', false, 'Ordered queue without ack()' ],
 378                         [ 'queueFifoTTL', true, 'Ordered queue with ack()' ]
 379                 ];
 380         }
 381
 382         function newJob( $i = 0, $rootJob = [] ) {
 383                 return Job::factory( 'null', Title::newMainPage(),
 384                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ] + $rootJob );
 385         }
 386
 387         function newDedupedJob( $i = 0, $rootJob = [] ) {
 388                 return Job::factory( 'null', Title::newMainPage(),
 389                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ] + $rootJob );
 390         }
 391 }
 392
 393 class JobQueueDBSingle extends JobQueueDB {
 394         protected function getDB( $index ) {
 395                 $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
 396                 // Override to not use CONN_TRX_AUTOCOMMIT so that we see the same temporary `job` table
 397                 return $lb->getConnection( $index, [], $this->domain );
 398         }
 399 }