tests/parser/editTests.php

   1 <?php
   2
   3 require __DIR__.'/../../maintenance/Maintenance.php';
   4
   5 define( 'MW_PARSER_TEST', true );
   6
   7 /**
   8  * Interactive parser test runner and test file editor
   9  */
  10 class ParserEditTests extends Maintenance {
  11         private $termWidth;
  12         private $testFiles;
  13         private $testCount;
  14         private $recorder;
  15         private $runner;
  16         private $numExecuted;
  17         private $numSkipped;
  18         private $numFailed;
  19
  20         function __construct() {
  21                 parent::__construct();
  22                 $this->addOption( 'session-data', 'internal option, do not use', false, true );
  23                 $this->addOption( 'use-tidy-config',
  24                         'Use the wiki\'s Tidy configuration instead of known-good' .
  25                         'defaults.' );
  26         }
  27
  28         public function finalSetup() {
  29                 parent::finalSetup();
  30                 self::requireTestsAutoloader();
  31                 TestSetup::applyInitialConfig();
  32         }
  33
  34         public function execute() {
  35                 $this->termWidth = $this->getTermSize()[0] - 1;
  36
  37                 $this->recorder = new TestRecorder();
  38                 $this->setupFileData();
  39
  40                 if ( $this->hasOption( 'session-data' ) ) {
  41                         $this->session = json_decode( $this->getOption( 'session-data' ), true );
  42                 } else {
  43                         $this->session = [ 'options' => [] ];
  44                 }
  45                 if ( $this->hasOption( 'use-tidy-config' ) ) {
  46                         $this->session['options']['use-tidy-config'] = true;
  47                 }
  48                 $this->runner = new ParserTestRunner( $this->recorder, $this->session['options'] );
  49
  50                 $this->runTests();
  51
  52                 if ( $this->numFailed === 0 ) {
  53                         if ( $this->numSkipped === 0 ) {
  54                                 print "All tests passed!\n";
  55                         } else {
  56                                 print "All tests passed (but skipped {$this->numSkipped})\n";
  57                         }
  58                         return;
  59                 }
  60                 print "{$this->numFailed} test(s) failed.\n";
  61                 $this->showResults();
  62         }
  63
  64         protected function setupFileData() {
  65                 global $wgParserTestFiles;
  66                 $this->testFiles = [];
  67                 $this->testCount = 0;
  68                 foreach ( $wgParserTestFiles as $file ) {
  69                         $fileInfo = TestFileReader::read( $file );
  70                         $this->testFiles[$file] = $fileInfo;
  71                         $this->testCount += count( $fileInfo['tests'] );
  72                 }
  73         }
  74
  75         protected function runTests() {
  76                 $teardown = $this->runner->staticSetup();
  77                 $teardown = $this->runner->setupDatabase( $teardown );
  78                 $teardown = $this->runner->setupUploads( $teardown );
  79
  80                 print "Running tests...\n";
  81                 $this->results = [];
  82                 $this->numExecuted = 0;
  83                 $this->numSkipped = 0;
  84                 $this->numFailed = 0;
  85                 foreach ( $this->testFiles as $fileName => $fileInfo ) {
  86                         $this->runner->addArticles( $fileInfo['articles'] );
  87                         foreach ( $fileInfo['tests'] as $testInfo ) {
  88                                 $result = $this->runner->runTest( $testInfo );
  89                                 if ( $result === false ) {
  90                                         $this->numSkipped++;
  91                                 } elseif ( !$result->isSuccess() ) {
  92                                         $this->results[$fileName][$testInfo['desc']] = $result;
  93                                         $this->numFailed++;
  94                                 }
  95                                 $this->numExecuted++;
  96                                 $this->showProgress();
  97                         }
  98                 }
  99                 print "\n";
 100         }
 101
 102         protected function showProgress() {
 103                 $done = $this->numExecuted;
 104                 $total = $this->testCount;
 105                 $width = $this->termWidth - 9;
 106                 $pos = round( $width * $done / $total );
 107                 printf( '│' . str_repeat( '█', $pos ) . str_repeat( '-', $width - $pos ) .
 108                         "│ %5.1f%%\r", $done / $total * 100 );
 109         }
 110
 111         protected function showResults() {
 112                 if ( isset( $this->session['startFile'] ) ) {
 113                         $startFile = $this->session['startFile'];
 114                         $startTest = $this->session['startTest'];
 115                         $foundStart = false;
 116                 } else {
 117                         $startFile = false;
 118                         $startTest = false;
 119                         $foundStart = true;
 120                 }
 121
 122                 $testIndex = 0;
 123                 foreach ( $this->testFiles as $fileName => $fileInfo ) {
 124                         if ( !isset( $this->results[$fileName] ) ) {
 125                                 continue;
 126                         }
 127                         if ( !$foundStart && $startFile !== false && $fileName !== $startFile ) {
 128                                 $testIndex += count( $this->results[$fileName] );
 129                                 continue;
 130                         }
 131                         foreach ( $fileInfo['tests'] as $testInfo ) {
 132                                 if ( !isset( $this->results[$fileName][$testInfo['desc']] ) ) {
 133                                         continue;
 134                                 }
 135                                 $result = $this->results[$fileName][$testInfo['desc']];
 136                                 $testIndex++;
 137                                 if ( !$foundStart && $startTest !== false ) {
 138                                         if ( $testInfo['desc'] !== $startTest ) {
 139                                                 continue;
 140                                         }
 141                                         $foundStart = true;
 142                                 }
 143
 144                                 $this->handleFailure( $testIndex, $testInfo, $result );
 145                         }
 146                 }
 147
 148                 if ( !$foundStart ) {
 149                         print "Could not find the test after a restart, did you rename it?";
 150                         unset( $this->session['startFile'] );
 151                         unset( $this->session['startTest'] );
 152                         $this->showResults();
 153                 }
 154                 print "All done\n";
 155         }
 156
 157         protected function heading( $text ) {
 158                 $term = new AnsiTermColorer;
 159                 $heading = "─── $text ";
 160                 $heading .= str_repeat( '─', $this->termWidth - mb_strlen( $heading ) );
 161                 $heading = $term->color( 34 ) . $heading . $term->reset() . "\n";
 162                 return $heading;
 163         }
 164
 165         protected function unifiedDiff( $left, $right ) {
 166                 $fromLines = explode( "\n", $left );
 167                 $toLines = explode( "\n", $right );
 168                 $formatter = new UnifiedDiffFormatter;
 169                 return $formatter->format( new Diff( $fromLines, $toLines ) );
 170         }
 171
 172         protected function handleFailure( $index, $testInfo, $result ) {
 173                 $term = new AnsiTermColorer;
 174                 $div1 = $term->color( 34 ) . str_repeat( '━', $this->termWidth ) .
 175                         $term->reset() . "\n";
 176                 $div2 = $term->color( 34 ) . str_repeat( '─', $this->termWidth ) .
 177                         $term->reset() . "\n";
 178
 179                 print $div1;
 180                 print "Failure $index/{$this->numFailed}: {$testInfo['file']} line {$testInfo['line']}\n" .
 181                         "{$testInfo['desc']}\n";
 182
 183                 print $this->heading( 'Input' );
 184                 print "{$testInfo['input']}\n";
 185
 186                 print $this->heading( 'Alternating expected/actual output' );
 187                 print $this->alternatingAligned( $result->expected, $result->actual );
 188
 189                 print $this->heading( 'Diff' );
 190
 191                 $dwdiff = $this->dwdiff( $result->expected, $result->actual );
 192                 if ( $dwdiff !== false ) {
 193                         $diff = $dwdiff;
 194                 } else {
 195                         $diff = $this->unifiedDiff( $result->expected, $result->actual );
 196                 }
 197                 print $diff;
 198
 199                 if ( $testInfo['options'] || $testInfo['config'] ) {
 200                         print $this->heading( 'Options / Config' );
 201                         if ( $testInfo['options'] ) {
 202                                 print $testInfo['options'] . "\n";
 203                         }
 204                         if ( $testInfo['config'] ) {
 205                                 print $testInfo['config'] . "\n";
 206                         }
 207                 }
 208
 209                 print $div2;
 210                 print "What do you want to do?\n";
 211                 $specs = [
 212                         '[R]eload code and run again',
 213                         '[U]pdate source file, copy actual to expected',
 214                         '[I]gnore' ];
 215
 216                 if ( strpos( $testInfo['options'], ' tidy' ) === false ) {
 217                         if ( empty( $testInfo['isSubtest'] ) ) {
 218                                 $specs[] = "Enable [T]idy";
 219                         }
 220                 } else {
 221                         $specs[] = 'Disable [T]idy';
 222                 }
 223
 224                 if ( !empty( $testInfo['isSubtest'] ) ) {
 225                         $specs[] = 'Delete [s]ubtest';
 226                 }
 227                 $specs[] = '[D]elete test';
 228                 $specs[] = '[Q]uit';
 229
 230                 $options = [];
 231                 foreach ( $specs as $spec ) {
 232                         if ( !preg_match( '/^(.*\[)(.)(\].*)$/', $spec, $m ) ) {
 233                                 throw new MWException( 'Invalid option spec: ' . $spec );
 234                         }
 235                         print '* ' . $m[1] . $term->color( 35 ) . $m[2] . $term->color( 0 ) . $m[3] . "\n";
 236                         $options[strtoupper( $m[2] )] = true;
 237                 }
 238
 239                 do {
 240                         $response = $this->readconsole();
 241                         $cmdResult = false;
 242                         if ( $response === false ) {
 243                                 exit( 0 );
 244                         }
 245
 246                         $response = strtoupper( trim( $response ) );
 247                         if ( !isset( $options[$response] ) ) {
 248                                 print "Invalid response, please enter a single letter from the list above\n";
 249                                 continue;
 250                         }
 251
 252                         switch ( strtoupper( trim( $response ) ) ) {
 253                                 case 'R':
 254                                         $cmdResult = $this->reload( $testInfo );
 255                                         break;
 256                                 case 'U':
 257                                         $cmdResult = $this->update( $testInfo, $result );
 258                                         break;
 259                                 case 'I':
 260                                         return;
 261                                 case 'T':
 262                                         $cmdResult = $this->switchTidy( $testInfo );
 263                                         break;
 264                                 case 'S':
 265                                         $cmdResult = $this->deleteSubtest( $testInfo );
 266                                         break;
 267                                 case 'D':
 268                                         $cmdResult = $this->deleteTest( $testInfo );
 269                                         break;
 270                                 case 'Q':
 271                                         exit( 0 );
 272                         }
 273                 } while ( !$cmdResult );
 274         }
 275
 276         protected function dwdiff( $expected, $actual ) {
 277                 if ( !is_executable( '/usr/bin/dwdiff' ) ) {
 278                         return false;
 279                 }
 280
 281                 $markers = [
 282                         "\n" => '¶',
 283                         ' ' => '·',
 284                         "\t" => '→'
 285                 ];
 286                 $markedExpected = strtr( $expected, $markers );
 287                 $markedActual = strtr( $actual, $markers );
 288                 $diff = $this->unifiedDiff( $markedExpected, $markedActual );
 289
 290                 $tempFile = tmpfile();
 291                 fwrite( $tempFile, $diff );
 292                 fseek( $tempFile, 0 );
 293                 $pipes = [];
 294                 $proc = proc_open( '/usr/bin/dwdiff -Pc --diff-input',
 295                         [ 0 => $tempFile, 1 => [ 'pipe', 'w' ], 2 => STDERR ],
 296                         $pipes );
 297
 298                 if ( !$proc ) {
 299                         return false;
 300                 }
 301
 302                 $result = stream_get_contents( $pipes[1] );
 303                 proc_close( $proc );
 304                 fclose( $tempFile );
 305                 return $result;
 306         }
 307
 308         protected function alternatingAligned( $expectedStr, $actualStr ) {
 309                 $expectedLines = explode( "\n", $expectedStr );
 310                 $actualLines = explode( "\n", $actualStr );
 311                 $maxLines = max( count( $expectedLines ), count( $actualLines ) );
 312                 $result = '';
 313                 for ( $i = 0; $i < $maxLines; $i++ ) {
 314                         if ( $i < count( $expectedLines ) ) {
 315                                 $expectedLine = $expectedLines[$i];
 316                                 $expectedChunks = str_split( $expectedLine, $this->termWidth - 3 );
 317                         } else {
 318                                 $expectedChunks = [];
 319                         }
 320
 321                         if ( $i < count( $actualLines ) ) {
 322                                 $actualLine = $actualLines[$i];
 323                                 $actualChunks = str_split( $actualLine, $this->termWidth - 3 );
 324                         } else {
 325                                 $actualChunks = [];
 326                         }
 327
 328                         $maxChunks = max( count( $expectedChunks ), count( $actualChunks ) );
 329
 330                         for ( $j = 0; $j < $maxChunks; $j++ ) {
 331                                 if ( isset( $expectedChunks[$j] ) ) {
 332                                         $result .= "E: " . $expectedChunks[$j];
 333                                         if ( $j === count( $expectedChunks ) - 1 ) {
 334                                                 $result .= "¶";
 335                                         }
 336                                         $result .= "\n";
 337                                 } else {
 338                                         $result .= "E:\n";
 339                                 }
 340                                 $result .= "\33[4m" . // underline
 341                                         "A: ";
 342                                 if ( isset( $actualChunks[$j] ) ) {
 343                                         $result .= $actualChunks[$j];
 344                                         if ( $j === count( $actualChunks ) - 1 ) {
 345                                                 $result .= "¶";
 346                                         }
 347                                 }
 348                                 $result .= "\33[0m\n"; // reset
 349                         }
 350                 }
 351                 return $result;
 352         }
 353
 354         protected function reload( $testInfo ) {
 355                 global $argv;
 356                 pcntl_exec( PHP_BINARY, [
 357                         $argv[0],
 358                         '--session-data',
 359                         json_encode( [
 360                                 'startFile' => $testInfo['file'],
 361                                 'startTest' => $testInfo['desc']
 362                         ] + $this->session ) ] );
 363
 364                 print "pcntl_exec() failed\n";
 365                 return false;
 366         }
 367
 368         protected function findTest( $file, $testInfo ) {
 369                 $initialPart = '';
 370                 for ( $i = 1; $i < $testInfo['line']; $i++ ) {
 371                         $line = fgets( $file );
 372                         if ( $line === false ) {
 373                                 print "Error reading from file\n";
 374                                 return false;
 375                         }
 376                         $initialPart .= $line;
 377                 }
 378
 379                 $line = fgets( $file );
 380                 if ( !preg_match( '/^!!\s*test/', $line ) ) {
 381                         print "Test has moved, cannot edit\n";
 382                         return false;
 383                 }
 384
 385                 $testPart = $line;
 386
 387                 $desc = fgets( $file );
 388                 if ( trim( $desc ) !== $testInfo['desc'] ) {
 389                         print "Description does not match, cannot edit\n";
 390                         return false;
 391                 }
 392                 $testPart .= $desc;
 393                 return [ $initialPart, $testPart ];
 394         }
 395
 396         protected function getOutputFileName( $inputFileName ) {
 397                 if ( is_writable( $inputFileName ) ) {
 398                         $outputFileName = $inputFileName;
 399                 } else {
 400                         $outputFileName = wfTempDir() . '/' . basename( $inputFileName );
 401                         print "Cannot write to input file, writing to $outputFileName instead\n";
 402                 }
 403                 return $outputFileName;
 404         }
 405
 406         protected function editTest( $fileName, $deletions, $changes ) {
 407                 $text = file_get_contents( $fileName );
 408                 if ( $text === false ) {
 409                         print "Unable to open test file!";
 410                         return false;
 411                 }
 412                 $result = TestFileEditor::edit( $text, $deletions, $changes,
 413                         function ( $msg ) {
 414                                 print "$msg\n";
 415                         }
 416                 );
 417                 if ( is_writable( $fileName ) ) {
 418                         file_put_contents( $fileName, $result );
 419                         print "Wrote updated file\n";
 420                 } else {
 421                         print "Cannot write updated file, here is a patch you can paste:\n\n";
 422                         print
 423                                 "--- {$fileName}\n" .
 424                                 "+++ {$fileName}~\n" .
 425                                 $this->unifiedDiff( $text, $result ) .
 426                                 "\n";
 427                 }
 428         }
 429
 430         protected function update( $testInfo, $result ) {
 431                 $this->editTest( $testInfo['file'],
 432                         [], // deletions
 433                         [ // changes
 434                                 $testInfo['test'] => [
 435                                         $testInfo['resultSection'] => [
 436                                                 'op' => 'update',
 437                                                 'value' => $result->actual . "\n"
 438                                         ]
 439                                 ]
 440                         ]
 441                 );
 442         }
 443
 444         protected function deleteTest( $testInfo ) {
 445                 $this->editTest( $testInfo['file'],
 446                         [ $testInfo['test'] ], // deletions
 447                         [] // changes
 448                 );
 449         }
 450
 451         protected function switchTidy( $testInfo ) {
 452                 $resultSection = $testInfo['resultSection'];
 453                 if ( in_array( $resultSection, [ 'html/php', 'html/*', 'html', 'result' ] ) ) {
 454                         $newSection = 'html+tidy';
 455                 } elseif ( in_array( $resultSection, [ 'html/php+tidy', 'html+tidy' ] ) ) {
 456                         $newSection = 'html';
 457                 } else {
 458                         print "Unrecognised result section name \"$resultSection\"";
 459                         return;
 460                 }
 461
 462                 $this->editTest( $testInfo['file'],
 463                         [], // deletions
 464                         [ // changes
 465                                 $testInfo['test'] => [
 466                                         $resultSection => [
 467                                                 'op' => 'rename',
 468                                                 'value' => $newSection
 469                                         ]
 470                                 ]
 471                         ]
 472                 );
 473         }
 474
 475         protected function deleteSubtest( $testInfo ) {
 476                 $this->editTest( $testInfo['file'],
 477                         [], // deletions
 478                         [ // changes
 479                                 $testInfo['test'] => [
 480                                         $testInfo['resultSection'] => [
 481                                                 'op' => 'delete'
 482                                         ]
 483                                 ]
 484                         ]
 485                 );
 486         }
 487 }
 488
 489 $maintClass = 'ParserEditTests';
 490 require RUN_MAINTENANCE_IF_MAIN;