tests/testHelpers.inc

   1 <?php
   2 /**
   3  * Recording for passing/failing tests.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Testing
  22  */
  23
  24 class TestRecorder {
  25         var $parent;
  26         var $term;
  27
  28         function __construct( $parent ) {
  29                 $this->parent = $parent;
  30                 $this->term = $parent->term;
  31         }
  32
  33         function start() {
  34                 $this->total = 0;
  35                 $this->success = 0;
  36         }
  37
  38         function record( $test, $result ) {
  39                 $this->total++;
  40                 $this->success += ( $result ? 1 : 0 );
  41         }
  42
  43         function end() {
  44                 // dummy
  45         }
  46
  47         function report() {
  48                 if ( $this->total > 0 ) {
  49                         $this->reportPercentage( $this->success, $this->total );
  50                 } else {
  51                         throw new MWException( "No tests found.\n" );
  52                 }
  53         }
  54
  55         function reportPercentage( $success, $total ) {
  56                 $ratio = wfPercent( 100 * $success / $total );
  57                 print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... ";
  58
  59                 if ( $success == $total ) {
  60                         print $this->term->color( 32 ) . "ALL TESTS PASSED!";
  61                 } else {
  62                         $failed = $total - $success;
  63                         print $this->term->color( 31 ) . "$failed tests failed!";
  64                 }
  65
  66                 print $this->term->reset() . "\n";
  67
  68                 return ( $success == $total );
  69         }
  70 }
  71
  72 class DbTestPreviewer extends TestRecorder {
  73         protected $lb; // /< Database load balancer
  74         protected $db; // /< Database connection to the main DB
  75         protected $curRun; // /< run ID number for the current run
  76         protected $prevRun; // /< run ID number for the previous run, if any
  77         protected $results; // /< Result array
  78
  79         /**
  80          * This should be called before the table prefix is changed
  81          */
  82         function __construct( $parent ) {
  83                 parent::__construct( $parent );
  84
  85                 $this->lb = wfGetLBFactory()->newMainLB();
  86                 // This connection will have the wiki's table prefix, not parsertest_
  87                 $this->db = $this->lb->getConnection( DB_MASTER );
  88         }
  89
  90         /**
  91          * Set up result recording; insert a record for the run with the date
  92          * and all that fun stuff
  93          */
  94         function start() {
  95                 parent::start();
  96
  97                 if ( !$this->db->tableExists( 'testrun', __METHOD__ )
  98                         || !$this->db->tableExists( 'testitem', __METHOD__ )
  99                 ) {
 100                         print "WARNING> `testrun` table not found in database.\n";
 101                         $this->prevRun = false;
 102                 } else {
 103                         // We'll make comparisons against the previous run later...
 104                         $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' );
 105                 }
 106
 107                 $this->results = array();
 108         }
 109
 110         function record( $test, $result ) {
 111                 parent::record( $test, $result );
 112                 $this->results[$test] = $result;
 113         }
 114
 115         function report() {
 116                 if ( $this->prevRun ) {
 117                         // f = fail, p = pass, n = nonexistent
 118                         // codes show before then after
 119                         $table = array(
 120                                 'fp' => 'previously failing test(s) now PASSING! :)',
 121                                 'pn' => 'previously PASSING test(s) removed o_O',
 122                                 'np' => 'new PASSING test(s) :)',
 123
 124                                 'pf' => 'previously passing test(s) now FAILING! :(',
 125                                 'fn' => 'previously FAILING test(s) removed O_o',
 126                                 'nf' => 'new FAILING test(s) :(',
 127                                 'ff' => 'still FAILING test(s) :(',
 128                         );
 129
 130                         $prevResults = array();
 131
 132                         $res = $this->db->select( 'testitem', array( 'ti_name', 'ti_success' ),
 133                                 array( 'ti_run' => $this->prevRun ), __METHOD__ );
 134
 135                         foreach ( $res as $row ) {
 136                                 if ( !$this->parent->regex
 137                                         || preg_match( "/{$this->parent->regex}/i", $row->ti_name )
 138                                 ) {
 139                                         $prevResults[$row->ti_name] = $row->ti_success;
 140                                 }
 141                         }
 142
 143                         $combined = array_keys( $this->results + $prevResults );
 144
 145                         # Determine breakdown by change type
 146                         $breakdown = array();
 147                         foreach ( $combined as $test ) {
 148                                 if ( !isset( $prevResults[$test] ) ) {
 149                                         $before = 'n';
 150                                 } elseif ( $prevResults[$test] == 1 ) {
 151                                         $before = 'p';
 152                                 } else /* if ( $prevResults[$test] == 0 )*/ {
 153                                         $before = 'f';
 154                                 }
 155
 156                                 if ( !isset( $this->results[$test] ) ) {
 157                                         $after = 'n';
 158                                 } elseif ( $this->results[$test] == 1 ) {
 159                                         $after = 'p';
 160                                 } else /*if ( $this->results[$test] == 0 ) */ {
 161                                         $after = 'f';
 162                                 }
 163
 164                                 $code = $before . $after;
 165
 166                                 if ( isset( $table[$code] ) ) {
 167                                         $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
 168                                 }
 169                         }
 170
 171                         # Write out results
 172                         foreach ( $table as $code => $label ) {
 173                                 if ( !empty( $breakdown[$code] ) ) {
 174                                         $count = count( $breakdown[$code] );
 175                                         printf( "\n%4d %s\n", $count, $label );
 176
 177                                         foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
 178                                                 print "      * $differing_test_name  [$statusInfo]\n";
 179                                         }
 180                                 }
 181                         }
 182                 } else {
 183                         print "No previous test runs to compare against.\n";
 184                 }
 185
 186                 print "\n";
 187                 parent::report();
 188         }
 189
 190         /**
 191          * Returns a string giving information about when a test last had a status change.
 192          * Could help to track down when regressions were introduced, as distinct from tests
 193          * which have never passed (which are more change requests than regressions).
 194          */
 195         private function getTestStatusInfo( $testname, $after ) {
 196                 // If we're looking at a test that has just been removed, then say when it first appeared.
 197                 if ( $after == 'n' ) {
 198                         $changedRun = $this->db->selectField( 'testitem',
 199                                 'MIN(ti_run)',
 200                                 array( 'ti_name' => $testname ),
 201                                 __METHOD__ );
 202                         $appear = $this->db->selectRow( 'testrun',
 203                                 array( 'tr_date', 'tr_mw_version' ),
 204                                 array( 'tr_id' => $changedRun ),
 205                                 __METHOD__ );
 206
 207                         return "First recorded appearance: "
 208                                 . date( "d-M-Y H:i:s", strtotime( $appear->tr_date ) )
 209                                 . ", " . $appear->tr_mw_version;
 210                 }
 211
 212                 // Otherwise, this test has previous recorded results.
 213                 // See when this test last had a different result to what we're seeing now.
 214                 $conds = array(
 215                         'ti_name' => $testname,
 216                         'ti_success' => ( $after == 'f' ? "1" : "0" ) );
 217
 218                 if ( $this->curRun ) {
 219                         $conds[] = "ti_run != " . $this->db->addQuotes( $this->curRun );
 220                 }
 221
 222                 $changedRun = $this->db->selectField( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ );
 223
 224                 // If no record of ever having had a different result.
 225                 if ( is_null( $changedRun ) ) {
 226                         if ( $after == "f" ) {
 227                                 return "Has never passed";
 228                         } else {
 229                                 return "Has never failed";
 230                         }
 231                 }
 232
 233                 // Otherwise, we're looking at a test whose status has changed.
 234                 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
 235                 // In this situation, give as much info as we can as to when it changed status.
 236                 $pre = $this->db->selectRow( 'testrun',
 237                         array( 'tr_date', 'tr_mw_version' ),
 238                         array( 'tr_id' => $changedRun ),
 239                         __METHOD__ );
 240                 $post = $this->db->selectRow( 'testrun',
 241                         array( 'tr_date', 'tr_mw_version' ),
 242                         array( "tr_id > " . $this->db->addQuotes( $changedRun ) ),
 243                         __METHOD__,
 244                         array( "LIMIT" => 1, "ORDER BY" => 'tr_id' )
 245                 );
 246
 247                 if ( $post ) {
 248                         $postDate = date( "d-M-Y H:i:s", strtotime( $post->tr_date ) ) . ", {$post->tr_mw_version}";
 249                 } else {
 250                         $postDate = 'now';
 251                 }
 252
 253                 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
 254                         . date( "d-M-Y H:i:s", strtotime( $pre->tr_date ) ) . ", " . $pre->tr_mw_version
 255                         . " and $postDate";
 256         }
 257
 258         /**
 259          * Commit transaction and clean up for result recording
 260          */
 261         function end() {
 262                 $this->lb->commitMasterChanges();
 263                 $this->lb->closeAll();
 264                 parent::end();
 265         }
 266 }
 267
 268 class DbTestRecorder extends DbTestPreviewer {
 269         var $version;
 270
 271         /**
 272          * Set up result recording; insert a record for the run with the date
 273          * and all that fun stuff
 274          */
 275         function start() {
 276                 $this->db->begin( __METHOD__ );
 277
 278                 if ( !$this->db->tableExists( 'testrun' )
 279                         || !$this->db->tableExists( 'testitem' )
 280                 ) {
 281                         print "WARNING> `testrun` table not found in database. Trying to create table.\n";
 282                         $this->db->sourceFile( $this->db->patchPath( 'patch-testrun.sql' ) );
 283                         echo "OK, resuming.\n";
 284                 }
 285
 286                 parent::start();
 287
 288                 $this->db->insert( 'testrun',
 289                         array(
 290                                 'tr_date' => $this->db->timestamp(),
 291                                 'tr_mw_version' => $this->version,
 292                                 'tr_php_version' => phpversion(),
 293                                 'tr_db_version' => $this->db->getServerVersion(),
 294                                 'tr_uname' => php_uname()
 295                         ),
 296                         __METHOD__ );
 297                 if ( $this->db->getType() === 'postgres' ) {
 298                         $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' );
 299                 } else {
 300                         $this->curRun = $this->db->insertId();
 301                 }
 302         }
 303
 304         /**
 305          * Record an individual test item's success or failure to the db
 306          *
 307          * @param $test String
 308          * @param $result Boolean
 309          */
 310         function record( $test, $result ) {
 311                 parent::record( $test, $result );
 312
 313                 $this->db->insert( 'testitem',
 314                         array(
 315                                 'ti_run' => $this->curRun,
 316                                 'ti_name' => $test,
 317                                 'ti_success' => $result ? 1 : 0,
 318                         ),
 319                         __METHOD__ );
 320         }
 321 }
 322
 323 class TestFileIterator implements Iterator {
 324         private $file;
 325         private $fh;
 326         private $parserTest; /* An instance of ParserTest (parserTests.php) or MediaWikiParserTest (phpunit) */
 327         private $index = 0;
 328         private $test;
 329         private $section = null;
 330         /** String|null: current test section being analyzed */
 331         private $sectionData = array();
 332         private $lineNum;
 333         private $eof;
 334
 335         function __construct( $file, $parserTest ) {
 336                 $this->file = $file;
 337                 $this->fh = fopen( $this->file, "rt" );
 338
 339                 if ( !$this->fh ) {
 340                         throw new MWException( "Couldn't open file '$file'\n" );
 341                 }
 342
 343                 $this->parserTest = $parserTest;
 344
 345                 $this->lineNum = $this->index = 0;
 346         }
 347
 348         function rewind() {
 349                 if ( fseek( $this->fh, 0 ) ) {
 350                         throw new MWException( "Couldn't fseek to the start of '$this->file'\n" );
 351                 }
 352
 353                 $this->index = -1;
 354                 $this->lineNum = 0;
 355                 $this->eof = false;
 356                 $this->next();
 357
 358                 return true;
 359         }
 360
 361         function current() {
 362                 return $this->test;
 363         }
 364
 365         function key() {
 366                 return $this->index;
 367         }
 368
 369         function next() {
 370                 if ( $this->readNextTest() ) {
 371                         $this->index++;
 372                         return true;
 373                 } else {
 374                         $this->eof = true;
 375                 }
 376         }
 377
 378         function valid() {
 379                 return $this->eof != true;
 380         }
 381
 382         function readNextTest() {
 383                 $this->clearSection();
 384
 385                 # Create a fake parser tests which never run anything unless
 386                 # asked to do so. This will avoid running hooks for a disabled test
 387                 $delayedParserTest = new DelayedParserTest();
 388
 389                 while ( false !== ( $line = fgets( $this->fh ) ) ) {
 390                         $this->lineNum++;
 391                         $matches = array();
 392
 393                         if ( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) {
 394                                 $this->section = strtolower( $matches[1] );
 395
 396                                 if ( $this->section == 'endarticle' ) {
 397                                         $this->checkSection( 'text' );
 398                                         $this->checkSection( 'article' );
 399
 400                                         $this->parserTest->addArticle( ParserTest::chomp( $this->sectionData['article'] ), $this->sectionData['text'], $this->lineNum );
 401
 402                                         $this->clearSection();
 403
 404                                         continue;
 405                                 }
 406
 407                                 if ( $this->section == 'endhooks' ) {
 408                                         $this->checkSection( 'hooks' );
 409
 410                                         foreach ( explode( "\n", $this->sectionData['hooks'] ) as $line ) {
 411                                                 $line = trim( $line );
 412
 413                                                 if ( $line ) {
 414                                                         $delayedParserTest->requireHook( $line );
 415                                                 }
 416                                         }
 417
 418                                         $this->clearSection();
 419
 420                                         continue;
 421                                 }
 422
 423                                 if ( $this->section == 'endfunctionhooks' ) {
 424                                         $this->checkSection( 'functionhooks' );
 425
 426                                         foreach ( explode( "\n", $this->sectionData['functionhooks'] ) as $line ) {
 427                                                 $line = trim( $line );
 428
 429                                                 if ( $line ) {
 430                                                         $delayedParserTest->requireFunctionHook( $line );
 431                                                 }
 432                                         }
 433
 434                                         $this->clearSection();
 435
 436                                         continue;
 437                                 }
 438
 439                                 if ( $this->section == 'end' ) {
 440                                         $this->checkSection( 'test' );
 441                                         $this->checkSection( 'input' );
 442                                         $this->checkSection( 'result' );
 443
 444                                         if ( !isset( $this->sectionData['options'] ) ) {
 445                                                 $this->sectionData['options'] = '';
 446                                         }
 447
 448                                         if ( !isset( $this->sectionData['config'] ) ) {
 449                                                 $this->sectionData['config'] = '';
 450                                         }
 451
 452                                         if ( ( ( preg_match( '/\\bdisabled\\b/i', $this->sectionData['options'] ) && !$this->parserTest->runDisabled )
 453                                                 || ( preg_match( '/\\bparsoid\\b/i', $this->sectionData['options'] ) && !$this->parserTest->runParsoid )
 454                                                 || !preg_match( "/" . $this->parserTest->regex . "/i", $this->sectionData['test'] ) )
 455                                         ) {
 456                                                 # disabled test
 457                                                 $this->clearSection();
 458
 459                                                 # Forget any pending hooks call since test is disabled
 460                                                 $delayedParserTest->reset();
 461
 462                                                 continue;
 463                                         }
 464
 465                                         # We are really going to run the test, run pending hooks and hooks function
 466                                         wfDebug( __METHOD__ . " unleashing delayed test for: {$this->sectionData['test']}" );
 467                                         $hooksResult = $delayedParserTest->unleash( $this->parserTest );
 468                                         if ( !$hooksResult ) {
 469                                                 # Some hook reported an issue. Abort.
 470                                                 return false;
 471                                         }
 472
 473                                         $this->test = array(
 474                                                 'test' => ParserTest::chomp( $this->sectionData['test'] ),
 475                                                 'input' => ParserTest::chomp( $this->sectionData['input'] ),
 476                                                 'result' => ParserTest::chomp( $this->sectionData['result'] ),
 477                                                 'options' => ParserTest::chomp( $this->sectionData['options'] ),
 478                                                 'config' => ParserTest::chomp( $this->sectionData['config'] ),
 479                                         );
 480
 481                                         return true;
 482                                 }
 483
 484                                 if ( isset ( $this->sectionData[$this->section] ) ) {
 485                                         throw new MWException( "duplicate section '$this->section' at line {$this->lineNum} of $this->file\n" );
 486                                 }
 487
 488                                 $this->sectionData[$this->section] = '';
 489
 490                                 continue;
 491                         }
 492
 493                         if ( $this->section ) {
 494                                 $this->sectionData[$this->section] .= $line;
 495                         }
 496                 }
 497
 498                 return false;
 499         }
 500
 501
 502         /**
 503          * Clear section name and its data
 504          */
 505         private function clearSection() {
 506                 $this->sectionData = array();
 507                 $this->section = null;
 508
 509         }
 510
 511         /**
 512          * Verify the current section data has some value for the given token
 513          * name (first parameter).
 514          * Throw an exception if it is not set, referencing current section
 515          * and adding the current file name and line number
 516          *
 517          * @param $token String: expected token that should have been mentionned before closing this section
 518          */
 519         private function checkSection( $token ) {
 520                 if ( is_null( $this->section ) ) {
 521                         throw new MWException( __METHOD__ . " can not verify a null section!\n" );
 522                 }
 523
 524                 if ( !isset( $this->sectionData[$token] ) ) {
 525                         throw new MWException( sprintf(
 526                                 "'%s' without '%s' at line %s of %s\n",
 527                                 $this->section,
 528                                 $token,
 529                                 $this->lineNum,
 530                                 $this->file
 531                         ) );
 532                 }
 533                 return true;
 534         }
 535 }
 536
 537 /**
 538  * A class to delay execution of a parser test hooks.
 539  */
 540 class DelayedParserTest {
 541
 542         /** Initialized on construction */
 543         private $hooks;
 544         private $fnHooks;
 545
 546         public function __construct() {
 547                 $this->reset();
 548         }
 549
 550         /**
 551          * Init/reset or forgot about the current delayed test.
 552          * Call to this will erase any hooks function that were pending.
 553          */
 554         public function reset() {
 555                 $this->hooks = array();
 556                 $this->fnHooks = array();
 557         }
 558
 559         /**
 560          * Called whenever we actually want to run the hook.
 561          * Should be the case if we found the parserTest is not disabled
 562          */
 563         public function unleash( &$parserTest ) {
 564                 if ( !( $parserTest instanceof ParserTest || $parserTest instanceof NewParserTest )     ) {
 565                         throw new MWException( __METHOD__ . " must be passed an instance of ParserTest or NewParserTest classes\n" );
 566                 }
 567
 568                 # Trigger delayed hooks. Any failure will make us abort
 569                 foreach ( $this->hooks as $hook ) {
 570                         $ret = $parserTest->requireHook( $hook );
 571                         if ( !$ret ) {
 572                                 return false;
 573                         }
 574                 }
 575
 576                 # Trigger delayed function hooks. Any failure will make us abort
 577                 foreach ( $this->fnHooks as $fnHook ) {
 578                         $ret = $parserTest->requireFunctionHook( $fnHook );
 579                         if ( !$ret ) {
 580                                 return false;
 581                         }
 582                 }
 583
 584                 # Delayed execution was successful.
 585                 return true;
 586         }
 587
 588         /**
 589          * Similar to ParserTest object but does not run anything
 590          * Use unleash() to really execute the hook
 591          */
 592         public function requireHook( $hook ) {
 593                 $this->hooks[] = $hook;
 594         }
 595
 596         /**
 597          * Similar to ParserTest object but does not run anything
 598          * Use unleash() to really execute the hook function
 599          */
 600         public function requireFunctionHook( $fnHook ) {
 601                 $this->fnHooks[] = $fnHook;
 602         }
 603
 604 }