tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 /**
   4  * @group JobQueue
   5  * @group medium
   6  * @group Database
   7  */
   8 class JobQueueTest extends MediaWikiTestCase {
   9         protected $key;
  10         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  11
  12         function __construct( $name = null, array $data = array(), $dataName = '' ) {
  13                 parent::__construct( $name, $data, $dataName );
  14
  15                 $this->tablesUsed[] = 'job';
  16         }
  17
  18         protected function setUp() {
  19                 global $wgJobTypeConf;
  20                 parent::setUp();
  21
  22                 $this->setMwGlobals( 'wgMemc', new HashBagOStuff() );
  23
  24                 if ( $this->getCliArg( 'use-jobqueue' ) ) {
  25                         $name = $this->getCliArg( 'use-jobqueue' );
  26                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  27                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  28                         }
  29                         $baseConfig = $wgJobTypeConf[$name];
  30                 } else {
  31                         $baseConfig = array( 'class' => 'JobQueueDB' );
  32                 }
  33                 $baseConfig['type'] = 'null';
  34                 $baseConfig['wiki'] = wfWikiID();
  35                 $variants = array(
  36                         'queueRand' => array( 'order' => 'random', 'claimTTL' => 0 ),
  37                         'queueRandTTL' => array( 'order' => 'random', 'claimTTL' => 10 ),
  38                         'queueTimestamp' => array( 'order' => 'timestamp', 'claimTTL' => 0 ),
  39                         'queueTimestampTTL' => array( 'order' => 'timestamp', 'claimTTL' => 10 ),
  40                         'queueFifo' => array( 'order' => 'fifo', 'claimTTL' => 0 ),
  41                         'queueFifoTTL' => array( 'order' => 'fifo', 'claimTTL' => 10 ),
  42                 );
  43                 foreach ( $variants as $q => $settings ) {
  44                         try {
  45                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  46                                 if ( !( $this->$q instanceof JobQueueDB ) ) {
  47                                         $this->$q->setTestingPrefix( 'unittests-' . wfRandomString( 32 ) );
  48                                 }
  49                         } catch ( MWException $e ) {
  50                                 // unsupported?
  51                                 // @todo What if it was another error?
  52                         };
  53                 }
  54         }
  55
  56         protected function tearDown() {
  57                 parent::tearDown();
  58                 foreach (
  59                         array(
  60                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  61                                 'queueFifo', 'queueFifoTTL'
  62                         ) as $q
  63                 ) {
  64                         if ( $this->$q ) {
  65                                 $this->$q->delete();
  66                         }
  67                         $this->$q = null;
  68                 }
  69         }
  70
  71         /**
  72          * @dataProvider provider_queueLists
  73          * @covers JobQueue::getWiki
  74          */
  75         public function testGetWiki( $queue, $recycles, $desc ) {
  76                 $queue = $this->$queue;
  77                 if ( !$queue ) {
  78                         $this->markTestSkipped( $desc );
  79                 }
  80                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  81         }
  82
  83         /**
  84          * @dataProvider provider_queueLists
  85          * @covers JobQueue::getType
  86          */
  87         public function testGetType( $queue, $recycles, $desc ) {
  88                 $queue = $this->$queue;
  89                 if ( !$queue ) {
  90                         $this->markTestSkipped( $desc );
  91                 }
  92                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  93         }
  94
  95         /**
  96          * @dataProvider provider_queueLists
  97          * @covers JobQueue
  98          */
  99         public function testBasicOperations( $queue, $recycles, $desc ) {
 100                 $queue = $this->$queue;
 101                 if ( !$queue ) {
 102                         $this->markTestSkipped( $desc );
 103                 }
 104
 105                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 106
 107                 $queue->flushCaches();
 108                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 109                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 110
 111                 $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 112                 $this->assertNull( $queue->batchPush( array( $this->newJob() ) ), "Push worked ($desc)" );
 113
 114                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 115
 116                 $queue->flushCaches();
 117                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 118                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 119                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 120                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 121
 122                 $job1 = $queue->pop();
 123                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 124
 125                 $queue->flushCaches();
 126                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 127
 128                 $queue->flushCaches();
 129                 if ( $recycles ) {
 130                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 131                 }
 132
 133                 $job2 = $queue->pop();
 134                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 135                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 136
 137                 $queue->flushCaches();
 138                 if ( $recycles ) {
 139                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 140                 }
 141
 142                 $queue->ack( $job1 );
 143
 144                 $queue->flushCaches();
 145                 if ( $recycles ) {
 146                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 147                 }
 148
 149                 $queue->ack( $job2 );
 150
 151                 $queue->flushCaches();
 152                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 153
 154                 $this->assertNull( $queue->batchPush( array( $this->newJob(), $this->newJob() ) ),
 155                         "Push worked ($desc)" );
 156                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 157
 158                 $queue->delete();
 159                 $queue->flushCaches();
 160                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 161                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 162         }
 163
 164         /**
 165          * @dataProvider provider_queueLists
 166          * @covers JobQueue
 167          */
 168         public function testBasicDeduplication( $queue, $recycles, $desc ) {
 169                 $queue = $this->$queue;
 170                 if ( !$queue ) {
 171                         $this->markTestSkipped( $desc );
 172                 }
 173
 174                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 175
 176                 $queue->flushCaches();
 177                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 178                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 179
 180                 $this->assertNull(
 181                         $queue->batchPush(
 182                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 183                         ),
 184                         "Push worked ($desc)" );
 185
 186                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 187
 188                 $queue->flushCaches();
 189                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 190                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 191
 192                 $this->assertNull(
 193                         $queue->batchPush(
 194                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 195                         ),
 196                         "Push worked ($desc)"
 197                 );
 198
 199                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 200
 201                 $queue->flushCaches();
 202                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 203                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 204
 205                 $job1 = $queue->pop();
 206                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 207
 208                 $queue->flushCaches();
 209                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 210                 if ( $recycles ) {
 211                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 212                 }
 213
 214                 $queue->ack( $job1 );
 215
 216                 $queue->flushCaches();
 217                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 218         }
 219
 220         /**
 221          * @dataProvider provider_queueLists
 222          * @covers JobQueue
 223          */
 224         public function testDeduplicationWhileClaimed( $queue, $recycles, $desc ) {
 225                 $queue = $this->$queue;
 226                 if ( !$queue ) {
 227                         $this->markTestSkipped( $desc );
 228                 }
 229
 230                 $job = $this->newDedupedJob();
 231                 $queue->push( $job );
 232
 233                 // De-duplication does not apply to already-claimed jobs
 234                 $j = $queue->pop();
 235                 $queue->push( $job );
 236                 $queue->ack( $j );
 237
 238                 $j = $queue->pop();
 239                 // Make sure ack() of the twin did not delete the sibling data
 240                 $this->assertType( 'NullJob', $j );
 241         }
 242
 243         /**
 244          * @dataProvider provider_queueLists
 245          * @covers JobQueue
 246          */
 247         public function testRootDeduplication( $queue, $recycles, $desc ) {
 248                 $queue = $this->$queue;
 249                 if ( !$queue ) {
 250                         $this->markTestSkipped( $desc );
 251                 }
 252
 253                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 254
 255                 $queue->flushCaches();
 256                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 257                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 258
 259                 $id = wfRandomString( 32 );
 260                 $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 261                 for ( $i = 0; $i < 5; ++$i ) {
 262                         $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 263                 }
 264                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 265
 266                 $root2 = $root1;
 267                 # Add a second to UNIX epoch and format back to TS_MW
 268                 $root2_ts = strtotime( $root2['rootJobTimestamp'] );
 269                 $root2_ts++;
 270                 $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
 271
 272                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 273                         "Root job signatures have different timestamps." );
 274                 for ( $i = 0; $i < 5; ++$i ) {
 275                         $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 276                 }
 277                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 278
 279                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 280
 281                 $queue->flushCaches();
 282                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 283                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 284
 285                 $dupcount = 0;
 286                 $jobs = array();
 287                 do {
 288                         $job = $queue->pop();
 289                         if ( $job ) {
 290                                 $jobs[] = $job;
 291                                 $queue->ack( $job );
 292                         }
 293                         if ( $job instanceof DuplicateJob ) {
 294                                 ++$dupcount;
 295                         }
 296                 } while ( $job );
 297
 298                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 299                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 300         }
 301
 302         /**
 303          * @dataProvider provider_fifoQueueLists
 304          * @covers JobQueue
 305          */
 306         public function testJobOrder( $queue, $recycles, $desc ) {
 307                 $queue = $this->$queue;
 308                 if ( !$queue ) {
 309                         $this->markTestSkipped( $desc );
 310                 }
 311
 312                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 313
 314                 $queue->flushCaches();
 315                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 316                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 317
 318                 for ( $i = 0; $i < 10; ++$i ) {
 319                         $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 320                 }
 321
 322                 for ( $i = 0; $i < 10; ++$i ) {
 323                         $job = $queue->pop();
 324                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 325                         $params = $job->getParams();
 326                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 327                         $queue->ack( $job );
 328                 }
 329
 330                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 331
 332                 $queue->flushCaches();
 333                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 334                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 335         }
 336
 337         public static function provider_queueLists() {
 338                 return array(
 339                         array( 'queueRand', false, 'Random queue without ack()' ),
 340                         array( 'queueRandTTL', true, 'Random queue with ack()' ),
 341                         array( 'queueTimestamp', false, 'Time ordered queue without ack()' ),
 342                         array( 'queueTimestampTTL', true, 'Time ordered queue with ack()' ),
 343                         array( 'queueFifo', false, 'FIFO ordered queue without ack()' ),
 344                         array( 'queueFifoTTL', true, 'FIFO ordered queue with ack()' )
 345                 );
 346         }
 347
 348         public static function provider_fifoQueueLists() {
 349                 return array(
 350                         array( 'queueFifo', false, 'Ordered queue without ack()' ),
 351                         array( 'queueFifoTTL', true, 'Ordered queue with ack()' )
 352                 );
 353         }
 354
 355         function newJob( $i = 0, $rootJob = array() ) {
 356                 return new NullJob( Title::newMainPage(),
 357                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ) + $rootJob );
 358         }
 359
 360         function newDedupedJob( $i = 0, $rootJob = array() ) {
 361                 return new NullJob( Title::newMainPage(),
 362                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ) + $rootJob );
 363         }
 364 }