tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 /**
   4  * @group JobQueue
   5  * @group medium
   6  * @group Database
   7  */
   8 class JobQueueTest extends MediaWikiTestCase {
   9         protected $key;
  10         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  11
  12         function __construct( $name = null, array $data = array(), $dataName = '' ) {
  13                 parent::__construct( $name, $data, $dataName );
  14
  15                 $this->tablesUsed[] = 'job';
  16         }
  17
  18         protected function setUp() {
  19                 global $wgJobTypeConf;
  20                 parent::setUp();
  21
  22                 $this->setMwGlobals( 'wgMemc', new HashBagOStuff() );
  23
  24                 if ( $this->getCliArg( 'use-jobqueue=' ) ) {
  25                         $name = $this->getCliArg( 'use-jobqueue=' );
  26                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  27                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  28                         }
  29                         $baseConfig = $wgJobTypeConf[$name];
  30                 } else {
  31                         $baseConfig = array( 'class' => 'JobQueueDB' );
  32                 }
  33                 $baseConfig['type'] = 'null';
  34                 $baseConfig['wiki'] = wfWikiID();
  35                 $variants = array(
  36                         'queueRand' => array( 'order' => 'random', 'claimTTL' => 0 ),
  37                         'queueRandTTL' => array( 'order' => 'random', 'claimTTL' => 10 ),
  38                         'queueTimestamp' => array( 'order' => 'timestamp', 'claimTTL' => 0 ),
  39                         'queueTimestampTTL' => array( 'order' => 'timestamp', 'claimTTL' => 10 ),
  40                         'queueFifo' => array( 'order' => 'fifo', 'claimTTL' => 0 ),
  41                         'queueFifoTTL' => array( 'order' => 'fifo', 'claimTTL' => 10 ),
  42                 );
  43                 foreach ( $variants as $q => $settings ) {
  44                         try {
  45                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  46                                 if ( !( $this->$q instanceof JobQueueDB ) ) {
  47                                         $this->$q->setTestingPrefix( 'unittests-' . wfRandomString( 32 ) );
  48                                 }
  49                         } catch ( MWException $e ) {
  50                         }; // unsupported? (@TODO: what if it was another error?)
  51                 }
  52         }
  53
  54         protected function tearDown() {
  55                 parent::tearDown();
  56                 foreach (
  57                         array(
  58                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  59                                 'queueFifo', 'queueFifoTTL'
  60                         ) as $q
  61                 ) {
  62                         if ( $this->$q ) {
  63                                 do {
  64                                         $job = $this->$q->pop();
  65                                         if ( $job ) {
  66                                                 $this->$q->ack( $job );
  67                                         }
  68                                 } while ( $job );
  69                         }
  70                         $this->$q = null;
  71                 }
  72         }
  73
  74         /**
  75          * @dataProvider provider_queueLists
  76          */
  77         function testProperties( $queue, $recycles, $desc ) {
  78                 $queue = $this->$queue;
  79                 if ( !$queue ) {
  80                         $this->markTestSkipped( $desc );
  81                 }
  82
  83                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  84                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  85         }
  86
  87         /**
  88          * @dataProvider provider_queueLists
  89          */
  90         function testBasicOperations( $queue, $recycles, $desc ) {
  91                 $queue = $this->$queue;
  92                 if ( !$queue ) {
  93                         $this->markTestSkipped( $desc );
  94                 }
  95
  96                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
  97
  98                 $queue->flushCaches();
  99                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 100                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 101
 102                 $this->assertTrue( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 103                 $this->assertTrue( $queue->batchPush( array( $this->newJob() ) ), "Push worked ($desc)" );
 104
 105                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 106
 107                 $queue->flushCaches();
 108                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 109                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 110                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 111                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 112
 113                 $job1 = $queue->pop();
 114                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 115
 116                 $queue->flushCaches();
 117                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 118
 119                 $queue->flushCaches();
 120                 if ( $recycles ) {
 121                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 122                 } else {
 123                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 124                 }
 125
 126                 $job2 = $queue->pop();
 127                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 128                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 129
 130                 $queue->flushCaches();
 131                 if ( $recycles ) {
 132                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 133                 } else {
 134                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 135                 }
 136
 137                 $queue->ack( $job1 );
 138
 139                 $queue->flushCaches();
 140                 if ( $recycles ) {
 141                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 142                 } else {
 143                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 144                 }
 145
 146                 $queue->ack( $job2 );
 147
 148                 $queue->flushCaches();
 149                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 150         }
 151
 152         /**
 153          * @dataProvider provider_queueLists
 154          */
 155         function testBasicDeduplication( $queue, $recycles, $desc ) {
 156                 $queue = $this->$queue;
 157                 if ( !$queue ) {
 158                         $this->markTestSkipped( $desc );
 159                 }
 160
 161                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 162
 163                 $queue->flushCaches();
 164                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 165                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 166
 167                 $this->assertTrue(
 168                         $queue->batchPush(
 169                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 170                         ),
 171                         "Push worked ($desc)" );
 172
 173                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 174
 175                 $queue->flushCaches();
 176                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 177                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 178
 179                 $this->assertTrue(
 180                         $queue->batchPush(
 181                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 182                         ),
 183                         "Push worked ($desc)"
 184                 );
 185
 186                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 187
 188                 $queue->flushCaches();
 189                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 190                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 191
 192                 $job1 = $queue->pop();
 193                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 194
 195                 $queue->flushCaches();
 196                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 197                 if ( $recycles ) {
 198                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 199                 } else {
 200                         $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 201                 }
 202
 203                 $queue->ack( $job1 );
 204
 205                 $queue->flushCaches();
 206                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 207         }
 208
 209         /**
 210          * @dataProvider provider_queueLists
 211          */
 212         function testRootDeduplication( $queue, $recycles, $desc ) {
 213                 $queue = $this->$queue;
 214                 if ( !$queue ) {
 215                         $this->markTestSkipped( $desc );
 216                 }
 217
 218                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 219
 220                 $queue->flushCaches();
 221                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 222                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 223
 224                 $id = wfRandomString( 32 );
 225                 $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 226                 for ( $i = 0; $i < 5; ++$i ) {
 227                         $this->assertTrue( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 228                 }
 229                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 230                 sleep( 1 ); // roo job timestamp will increase
 231                 $root2 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 232                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 233                         "Root job signatures have different timestamps." );
 234                 for ( $i = 0; $i < 5; ++$i ) {
 235                         $this->assertTrue( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 236                 }
 237                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 238
 239                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 240
 241                 $queue->flushCaches();
 242                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 243                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 244
 245                 $dupcount = 0;
 246                 $jobs = array();
 247                 do {
 248                         $job = $queue->pop();
 249                         if ( $job ) {
 250                                 $jobs[] = $job;
 251                                 $queue->ack( $job );
 252                         }
 253                         if ( $job instanceof DuplicateJob ) {
 254                                 ++$dupcount;
 255                         }
 256                 } while ( $job );
 257
 258                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 259                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 260         }
 261
 262         /**
 263          * @dataProvider provider_fifoQueueLists
 264          */
 265         function testJobOrder( $queue, $recycles, $desc ) {
 266                 $queue = $this->$queue;
 267                 if ( !$queue ) {
 268                         $this->markTestSkipped( $desc );
 269                 }
 270
 271                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 272
 273                 $queue->flushCaches();
 274                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 275                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 276
 277                 for ( $i = 0; $i < 10; ++$i ) {
 278                         $this->assertTrue( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 279                 }
 280
 281                 for ( $i = 0; $i < 10; ++$i ) {
 282                         $job = $queue->pop();
 283                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 284                         $params = $job->getParams();
 285                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 286                         $queue->ack( $job );
 287                 }
 288
 289                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 290
 291                 $queue->flushCaches();
 292                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 293                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 294         }
 295
 296         public static function provider_queueLists() {
 297                 return array(
 298                         array( 'queueRand', false, 'Random queue without ack()' ),
 299                         array( 'queueRandTTL', true, 'Random queue with ack()' ),
 300                         array( 'queueTimestamp', false, 'Time ordered queue without ack()' ),
 301                         array( 'queueTimestampTTL', true, 'Time ordered queue with ack()' ),
 302                         array( 'queueFifo', false, 'FIFO ordered queue without ack()' ),
 303                         array( 'queueFifoTTL', true, 'FIFO ordered queue with ack()' )
 304                 );
 305         }
 306
 307         public static function provider_fifoQueueLists() {
 308                 return array(
 309                         array( 'queueFifo', false, 'Ordered queue without ack()' ),
 310                         array( 'queueFifoTTL', true, 'Ordered queue with ack()' )
 311                 );
 312         }
 313
 314         function newJob( $i = 0, $rootJob = array() ) {
 315                 return new NullJob( Title::newMainPage(),
 316                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ) + $rootJob );
 317         }
 318
 319         function newDedupedJob( $i = 0, $rootJob = array() ) {
 320                 return new NullJob( Title::newMainPage(),
 321                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ) + $rootJob );
 322         }
 323 }