tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 use MediaWiki\MediaWikiServices;
   4
   5 /**
   6  * @group JobQueue
   7  * @group medium
   8  * @group Database
   9  */
  10 class JobQueueTest extends MediaWikiTestCase {
  11         protected $key;
  12         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  13
  14         function __construct( $name = null, array $data = [], $dataName = '' ) {
  15                 parent::__construct( $name, $data, $dataName );
  16
  17                 $this->tablesUsed[] = 'job';
  18         }
  19
  20         protected function setUp() {
  21                 global $wgJobTypeConf;
  22                 parent::setUp();
  23
  24                 if ( $this->getCliArg( 'use-jobqueue' ) ) {
  25                         $name = $this->getCliArg( 'use-jobqueue' );
  26                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  27                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  28                         }
  29                         $baseConfig = $wgJobTypeConf[$name];
  30                 } else {
  31                         $baseConfig = [ 'class' => JobQueueDBSingle::class ];
  32                 }
  33                 $baseConfig['type'] = 'null';
  34                 $baseConfig['wiki'] = wfWikiID();
  35                 $variants = [
  36                         'queueRand' => [ 'order' => 'random', 'claimTTL' => 0 ],
  37                         'queueRandTTL' => [ 'order' => 'random', 'claimTTL' => 10 ],
  38                         'queueTimestamp' => [ 'order' => 'timestamp', 'claimTTL' => 0 ],
  39                         'queueTimestampTTL' => [ 'order' => 'timestamp', 'claimTTL' => 10 ],
  40                         'queueFifo' => [ 'order' => 'fifo', 'claimTTL' => 0 ],
  41                         'queueFifoTTL' => [ 'order' => 'fifo', 'claimTTL' => 10 ],
  42                 ];
  43                 foreach ( $variants as $q => $settings ) {
  44                         try {
  45                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  46                         } catch ( MWException $e ) {
  47                                 // unsupported?
  48                                 // @todo What if it was another error?
  49                         };
  50                 }
  51         }
  52
  53         protected function tearDown() {
  54                 parent::tearDown();
  55                 foreach (
  56                         [
  57                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  58                                 'queueFifo', 'queueFifoTTL'
  59                         ] as $q
  60                 ) {
  61                         if ( $this->$q ) {
  62                                 $this->$q->delete();
  63                         }
  64                         $this->$q = null;
  65                 }
  66         }
  67
  68         /**
  69          * @dataProvider provider_queueLists
  70          * @covers JobQueue::getWiki
  71          */
  72         public function testGetWiki( $queue, $recycles, $desc ) {
  73                 $queue = $this->$queue;
  74                 if ( !$queue ) {
  75                         $this->markTestSkipped( $desc );
  76                 }
  77                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  78                 $this->assertEquals( wfWikiID(), $queue->getDomain(), "Proper wiki ID ($desc)" );
  79         }
  80
  81         /**
  82          * @dataProvider provider_queueLists
  83          * @covers JobQueue::getType
  84          */
  85         public function testGetType( $queue, $recycles, $desc ) {
  86                 $queue = $this->$queue;
  87                 if ( !$queue ) {
  88                         $this->markTestSkipped( $desc );
  89                 }
  90                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  91         }
  92
  93         /**
  94          * @dataProvider provider_queueLists
  95          * @covers JobQueue
  96          */
  97         public function testBasicOperations( $queue, $recycles, $desc ) {
  98                 $queue = $this->$queue;
  99                 if ( !$queue ) {
 100                         $this->markTestSkipped( $desc );
 101                 }
 102
 103                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 104
 105                 $queue->flushCaches();
 106                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 107                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 108
 109                 $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 110                 $this->assertNull( $queue->batchPush( [ $this->newJob() ] ), "Push worked ($desc)" );
 111
 112                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 113
 114                 $queue->flushCaches();
 115                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 116                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 117                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 118                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 119
 120                 $job1 = $queue->pop();
 121                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 122
 123                 $queue->flushCaches();
 124                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 125
 126                 $queue->flushCaches();
 127                 if ( $recycles ) {
 128                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 129                 }
 130
 131                 $job2 = $queue->pop();
 132                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 133                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 134
 135                 $queue->flushCaches();
 136                 if ( $recycles ) {
 137                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 138                 }
 139
 140                 $queue->ack( $job1 );
 141
 142                 $queue->flushCaches();
 143                 if ( $recycles ) {
 144                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 145                 }
 146
 147                 $queue->ack( $job2 );
 148
 149                 $queue->flushCaches();
 150                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 151
 152                 $this->assertNull( $queue->batchPush( [ $this->newJob(), $this->newJob() ] ),
 153                         "Push worked ($desc)" );
 154                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 155
 156                 $queue->delete();
 157                 $queue->flushCaches();
 158                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 159                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 160         }
 161
 162         /**
 163          * @dataProvider provider_queueLists
 164          * @covers JobQueue
 165          */
 166         public function testBasicDeduplication( $queue, $recycles, $desc ) {
 167                 $queue = $this->$queue;
 168                 if ( !$queue ) {
 169                         $this->markTestSkipped( $desc );
 170                 }
 171
 172                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 173
 174                 $queue->flushCaches();
 175                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 176                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 177
 178                 $this->assertNull(
 179                         $queue->batchPush(
 180                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 181                         ),
 182                         "Push worked ($desc)" );
 183
 184                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 185
 186                 $queue->flushCaches();
 187                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 188                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 189
 190                 $this->assertNull(
 191                         $queue->batchPush(
 192                                 [ $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ]
 193                         ),
 194                         "Push worked ($desc)"
 195                 );
 196
 197                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 198
 199                 $queue->flushCaches();
 200                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 201                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 202
 203                 $job1 = $queue->pop();
 204                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 205
 206                 $queue->flushCaches();
 207                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 208                 if ( $recycles ) {
 209                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 210                 }
 211
 212                 $queue->ack( $job1 );
 213
 214                 $queue->flushCaches();
 215                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 216         }
 217
 218         /**
 219          * @dataProvider provider_queueLists
 220          * @covers JobQueue
 221          */
 222         public function testDeduplicationWhileClaimed( $queue, $recycles, $desc ) {
 223                 $queue = $this->$queue;
 224                 if ( !$queue ) {
 225                         $this->markTestSkipped( $desc );
 226                 }
 227
 228                 $job = $this->newDedupedJob();
 229                 $queue->push( $job );
 230
 231                 // De-duplication does not apply to already-claimed jobs
 232                 $j = $queue->pop();
 233                 $queue->push( $job );
 234                 $queue->ack( $j );
 235
 236                 $j = $queue->pop();
 237                 // Make sure ack() of the twin did not delete the sibling data
 238                 $this->assertType( NullJob::class, $j );
 239         }
 240
 241         /**
 242          * @dataProvider provider_queueLists
 243          * @covers JobQueue
 244          */
 245         public function testRootDeduplication( $queue, $recycles, $desc ) {
 246                 $queue = $this->$queue;
 247                 if ( !$queue ) {
 248                         $this->markTestSkipped( $desc );
 249                 }
 250
 251                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 252
 253                 $queue->flushCaches();
 254                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 255                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 256
 257                 $id = wfRandomString( 32 );
 258                 $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 259                 for ( $i = 0; $i < 5; ++$i ) {
 260                         $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 261                 }
 262                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 263
 264                 $root2 = $root1;
 265                 # Add a second to UNIX epoch and format back to TS_MW
 266                 $root2_ts = strtotime( $root2['rootJobTimestamp'] );
 267                 $root2_ts++;
 268                 $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
 269
 270                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 271                         "Root job signatures have different timestamps." );
 272                 for ( $i = 0; $i < 5; ++$i ) {
 273                         $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 274                 }
 275                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 276
 277                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 278
 279                 $queue->flushCaches();
 280                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 281                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 282
 283                 $dupcount = 0;
 284                 $jobs = [];
 285                 do {
 286                         $job = $queue->pop();
 287                         if ( $job ) {
 288                                 $jobs[] = $job;
 289                                 $queue->ack( $job );
 290                         }
 291                         if ( $job instanceof DuplicateJob ) {
 292                                 ++$dupcount;
 293                         }
 294                 } while ( $job );
 295
 296                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 297                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 298         }
 299
 300         /**
 301          * @dataProvider provider_fifoQueueLists
 302          * @covers JobQueue
 303          */
 304         public function testJobOrder( $queue, $recycles, $desc ) {
 305                 $queue = $this->$queue;
 306                 if ( !$queue ) {
 307                         $this->markTestSkipped( $desc );
 308                 }
 309
 310                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 311
 312                 $queue->flushCaches();
 313                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 314                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 315
 316                 for ( $i = 0; $i < 10; ++$i ) {
 317                         $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 318                 }
 319
 320                 for ( $i = 0; $i < 10; ++$i ) {
 321                         $job = $queue->pop();
 322                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 323                         $params = $job->getParams();
 324                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 325                         $queue->ack( $job );
 326                 }
 327
 328                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 329
 330                 $queue->flushCaches();
 331                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 332                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 333         }
 334
 335         /**
 336          * @covers JobQueue
 337          */
 338         public function testQueueAggregateTable() {
 339                 $queue = $this->queueFifo;
 340                 if ( !$queue || !method_exists( $queue, 'getServerQueuesWithJobs' ) ) {
 341                         $this->markTestSkipped();
 342                 }
 343
 344                 $this->assertNotContains(
 345                         [ $queue->getType(), $queue->getWiki() ],
 346                         $queue->getServerQueuesWithJobs(),
 347                         "Null queue not in listing"
 348                 );
 349
 350                 $queue->push( $this->newJob( 0 ) );
 351
 352                 $this->assertContains(
 353                         [ $queue->getType(), $queue->getWiki() ],
 354                         $queue->getServerQueuesWithJobs(),
 355                         "Null queue in listing"
 356                 );
 357         }
 358
 359         public static function provider_queueLists() {
 360                 return [
 361                         [ 'queueRand', false, 'Random queue without ack()' ],
 362                         [ 'queueRandTTL', true, 'Random queue with ack()' ],
 363                         [ 'queueTimestamp', false, 'Time ordered queue without ack()' ],
 364                         [ 'queueTimestampTTL', true, 'Time ordered queue with ack()' ],
 365                         [ 'queueFifo', false, 'FIFO ordered queue without ack()' ],
 366                         [ 'queueFifoTTL', true, 'FIFO ordered queue with ack()' ]
 367                 ];
 368         }
 369
 370         public static function provider_fifoQueueLists() {
 371                 return [
 372                         [ 'queueFifo', false, 'Ordered queue without ack()' ],
 373                         [ 'queueFifoTTL', true, 'Ordered queue with ack()' ]
 374                 ];
 375         }
 376
 377         function newJob( $i = 0, $rootJob = [] ) {
 378                 return new NullJob( Title::newMainPage(),
 379                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ] + $rootJob );
 380         }
 381
 382         function newDedupedJob( $i = 0, $rootJob = [] ) {
 383                 return new NullJob( Title::newMainPage(),
 384                         [ 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ] + $rootJob );
 385         }
 386 }
 387
 388 class JobQueueDBSingle extends JobQueueDB {
 389         protected function getDB( $index ) {
 390                 $lb = MediaWikiServices::getInstance()->getDBLoadBalancer();
 391                 // Override to not use CONN_TRX_AUTOCOMMIT so that we see the same temporary `job` table
 392                 return $lb->getConnection( $index, [], $this->domain );
 393         }
 394 }