tests/phpunit/includes/jobqueue/JobQueueTest.php

   1 <?php
   2
   3 /**
   4  * @group JobQueue
   5  * @group medium
   6  * @group Database
   7  */
   8 class JobQueueTest extends MediaWikiTestCase {
   9         protected $key;
  10         protected $queueRand, $queueRandTTL, $queueFifo, $queueFifoTTL;
  11
  12         function __construct( $name = null, array $data = array(), $dataName = '' ) {
  13                 parent::__construct( $name, $data, $dataName );
  14
  15                 $this->tablesUsed[] = 'job';
  16         }
  17
  18         protected function setUp() {
  19                 global $wgJobTypeConf;
  20                 parent::setUp();
  21
  22                 if ( $this->getCliArg( 'use-jobqueue' ) ) {
  23                         $name = $this->getCliArg( 'use-jobqueue' );
  24                         if ( !isset( $wgJobTypeConf[$name] ) ) {
  25                                 throw new MWException( "No \$wgJobTypeConf entry for '$name'." );
  26                         }
  27                         $baseConfig = $wgJobTypeConf[$name];
  28                 } else {
  29                         $baseConfig = array( 'class' => 'JobQueueDB' );
  30                 }
  31                 $baseConfig['type'] = 'null';
  32                 $baseConfig['wiki'] = wfWikiID();
  33                 $variants = array(
  34                         'queueRand' => array( 'order' => 'random', 'claimTTL' => 0 ),
  35                         'queueRandTTL' => array( 'order' => 'random', 'claimTTL' => 10 ),
  36                         'queueTimestamp' => array( 'order' => 'timestamp', 'claimTTL' => 0 ),
  37                         'queueTimestampTTL' => array( 'order' => 'timestamp', 'claimTTL' => 10 ),
  38                         'queueFifo' => array( 'order' => 'fifo', 'claimTTL' => 0 ),
  39                         'queueFifoTTL' => array( 'order' => 'fifo', 'claimTTL' => 10 ),
  40                 );
  41                 foreach ( $variants as $q => $settings ) {
  42                         try {
  43                                 $this->$q = JobQueue::factory( $settings + $baseConfig );
  44                                 if ( !( $this->$q instanceof JobQueueDB ) ) {
  45                                         $this->$q->setTestingPrefix( 'unittests-' . wfRandomString( 32 ) );
  46                                 }
  47                         } catch ( MWException $e ) {
  48                                 // unsupported?
  49                                 // @todo What if it was another error?
  50                         };
  51                 }
  52         }
  53
  54         protected function tearDown() {
  55                 parent::tearDown();
  56                 foreach (
  57                         array(
  58                                 'queueRand', 'queueRandTTL', 'queueTimestamp', 'queueTimestampTTL',
  59                                 'queueFifo', 'queueFifoTTL'
  60                         ) as $q
  61                 ) {
  62                         if ( $this->$q ) {
  63                                 $this->$q->delete();
  64                         }
  65                         $this->$q = null;
  66                 }
  67         }
  68
  69         /**
  70          * @dataProvider provider_queueLists
  71          * @covers JobQueue::getWiki
  72          */
  73         public function testGetWiki( $queue, $recycles, $desc ) {
  74                 $queue = $this->$queue;
  75                 if ( !$queue ) {
  76                         $this->markTestSkipped( $desc );
  77                 }
  78                 $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" );
  79         }
  80
  81         /**
  82          * @dataProvider provider_queueLists
  83          * @covers JobQueue::getType
  84          */
  85         public function testGetType( $queue, $recycles, $desc ) {
  86                 $queue = $this->$queue;
  87                 if ( !$queue ) {
  88                         $this->markTestSkipped( $desc );
  89                 }
  90                 $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" );
  91         }
  92
  93         /**
  94          * @dataProvider provider_queueLists
  95          * @covers JobQueue
  96          */
  97         public function testBasicOperations( $queue, $recycles, $desc ) {
  98                 $queue = $this->$queue;
  99                 if ( !$queue ) {
 100                         $this->markTestSkipped( $desc );
 101                 }
 102
 103                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 104
 105                 $queue->flushCaches();
 106                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 107                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 108
 109                 $this->assertNull( $queue->push( $this->newJob() ), "Push worked ($desc)" );
 110                 $this->assertNull( $queue->batchPush( array( $this->newJob() ) ), "Push worked ($desc)" );
 111
 112                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 113
 114                 $queue->flushCaches();
 115                 $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" );
 116                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 117                 $jobs = iterator_to_array( $queue->getAllQueuedJobs() );
 118                 $this->assertEquals( 2, count( $jobs ), "Queue iterator size is correct ($desc)" );
 119
 120                 $job1 = $queue->pop();
 121                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 122
 123                 $queue->flushCaches();
 124                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 125
 126                 $queue->flushCaches();
 127                 if ( $recycles ) {
 128                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 129                 }
 130
 131                 $job2 = $queue->pop();
 132                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 133                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 134
 135                 $queue->flushCaches();
 136                 if ( $recycles ) {
 137                         $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" );
 138                 }
 139
 140                 $queue->ack( $job1 );
 141
 142                 $queue->flushCaches();
 143                 if ( $recycles ) {
 144                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 145                 }
 146
 147                 $queue->ack( $job2 );
 148
 149                 $queue->flushCaches();
 150                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 151
 152                 $this->assertNull( $queue->batchPush( array( $this->newJob(), $this->newJob() ) ),
 153                         "Push worked ($desc)" );
 154                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 155
 156                 $queue->delete();
 157                 $queue->flushCaches();
 158                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 159                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 160         }
 161
 162         /**
 163          * @dataProvider provider_queueLists
 164          * @covers JobQueue
 165          */
 166         public function testBasicDeduplication( $queue, $recycles, $desc ) {
 167                 $queue = $this->$queue;
 168                 if ( !$queue ) {
 169                         $this->markTestSkipped( $desc );
 170                 }
 171
 172                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 173
 174                 $queue->flushCaches();
 175                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 176                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 177
 178                 $this->assertNull(
 179                         $queue->batchPush(
 180                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 181                         ),
 182                         "Push worked ($desc)" );
 183
 184                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 185
 186                 $queue->flushCaches();
 187                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 188                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 189
 190                 $this->assertNull(
 191                         $queue->batchPush(
 192                                 array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() )
 193                         ),
 194                         "Push worked ($desc)"
 195                 );
 196
 197                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 198
 199                 $queue->flushCaches();
 200                 $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" );
 201                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 202
 203                 $job1 = $queue->pop();
 204                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 205
 206                 $queue->flushCaches();
 207                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 208                 if ( $recycles ) {
 209                         $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" );
 210                 }
 211
 212                 $queue->ack( $job1 );
 213
 214                 $queue->flushCaches();
 215                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" );
 216         }
 217
 218         /**
 219          * @dataProvider provider_queueLists
 220          * @covers JobQueue
 221          */
 222         public function testDeduplicationWhileClaimed( $queue, $recycles, $desc ) {
 223                 $queue = $this->$queue;
 224                 if ( !$queue ) {
 225                         $this->markTestSkipped( $desc );
 226                 }
 227
 228                 $job = $this->newDedupedJob();
 229                 $queue->push( $job );
 230
 231                 // De-duplication does not apply to already-claimed jobs
 232                 $j = $queue->pop();
 233                 $queue->push( $job );
 234                 $queue->ack( $j );
 235
 236                 $j = $queue->pop();
 237                 // Make sure ack() of the twin did not delete the sibling data
 238                 $this->assertType( 'NullJob', $j );
 239         }
 240
 241         /**
 242          * @dataProvider provider_queueLists
 243          * @covers JobQueue
 244          */
 245         public function testRootDeduplication( $queue, $recycles, $desc ) {
 246                 $queue = $this->$queue;
 247                 if ( !$queue ) {
 248                         $this->markTestSkipped( $desc );
 249                 }
 250
 251                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 252
 253                 $queue->flushCaches();
 254                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 255                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 256
 257                 $id = wfRandomString( 32 );
 258                 $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp
 259                 for ( $i = 0; $i < 5; ++$i ) {
 260                         $this->assertNull( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" );
 261                 }
 262                 $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) );
 263
 264                 $root2 = $root1;
 265                 # Add a second to UNIX epoch and format back to TS_MW
 266                 $root2_ts = strtotime( $root2['rootJobTimestamp'] );
 267                 $root2_ts++;
 268                 $root2['rootJobTimestamp'] = wfTimestamp( TS_MW, $root2_ts );
 269
 270                 $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'],
 271                         "Root job signatures have different timestamps." );
 272                 for ( $i = 0; $i < 5; ++$i ) {
 273                         $this->assertNull( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" );
 274                 }
 275                 $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) );
 276
 277                 $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" );
 278
 279                 $queue->flushCaches();
 280                 $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" );
 281                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 282
 283                 $dupcount = 0;
 284                 $jobs = array();
 285                 do {
 286                         $job = $queue->pop();
 287                         if ( $job ) {
 288                                 $jobs[] = $job;
 289                                 $queue->ack( $job );
 290                         }
 291                         if ( $job instanceof DuplicateJob ) {
 292                                 ++$dupcount;
 293                         }
 294                 } while ( $job );
 295
 296                 $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" );
 297                 $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" );
 298         }
 299
 300         /**
 301          * @dataProvider provider_fifoQueueLists
 302          * @covers JobQueue
 303          */
 304         public function testJobOrder( $queue, $recycles, $desc ) {
 305                 $queue = $this->$queue;
 306                 if ( !$queue ) {
 307                         $this->markTestSkipped( $desc );
 308                 }
 309
 310                 $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" );
 311
 312                 $queue->flushCaches();
 313                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 314                 $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" );
 315
 316                 for ( $i = 0; $i < 10; ++$i ) {
 317                         $this->assertNull( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" );
 318                 }
 319
 320                 for ( $i = 0; $i < 10; ++$i ) {
 321                         $job = $queue->pop();
 322                         $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" );
 323                         $params = $job->getParams();
 324                         $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" );
 325                         $queue->ack( $job );
 326                 }
 327
 328                 $this->assertFalse( $queue->pop(), "Queue is not empty ($desc)" );
 329
 330                 $queue->flushCaches();
 331                 $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" );
 332                 $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" );
 333         }
 334
 335         public static function provider_queueLists() {
 336                 return array(
 337                         array( 'queueRand', false, 'Random queue without ack()' ),
 338                         array( 'queueRandTTL', true, 'Random queue with ack()' ),
 339                         array( 'queueTimestamp', false, 'Time ordered queue without ack()' ),
 340                         array( 'queueTimestampTTL', true, 'Time ordered queue with ack()' ),
 341                         array( 'queueFifo', false, 'FIFO ordered queue without ack()' ),
 342                         array( 'queueFifoTTL', true, 'FIFO ordered queue with ack()' )
 343                 );
 344         }
 345
 346         public static function provider_fifoQueueLists() {
 347                 return array(
 348                         array( 'queueFifo', false, 'Ordered queue without ack()' ),
 349                         array( 'queueFifoTTL', true, 'Ordered queue with ack()' )
 350                 );
 351         }
 352
 353         function newJob( $i = 0, $rootJob = array() ) {
 354                 return new NullJob( Title::newMainPage(),
 355                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ) + $rootJob );
 356         }
 357
 358         function newDedupedJob( $i = 0, $rootJob = array() ) {
 359                 return new NullJob( Title::newMainPage(),
 360                         array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ) + $rootJob );
 361         }
 362 }