tests/parser/ParserTestRunner.php

   1 <?php
   2 /**
   3  * Generic backend for the MediaWiki parser test suite, used by both the
   4  * standalone parserTests.php and the PHPUnit "parsertests" suite.
   5  *
   6  * Copyright © 2004, 2010 Brion Vibber <brion@pobox.com>
   7  * https://www.mediawiki.org/
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write to the Free Software Foundation, Inc.,
  21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  22  * http://www.gnu.org/copyleft/gpl.html
  23  *
  24  * @todo Make this more independent of the configuration (and if possible the database)
  25  * @file
  26  * @ingroup Testing
  27  */
  28 use Wikimedia\Rdbms\IDatabase;
  29 use MediaWiki\MediaWikiServices;
  30 use Wikimedia\ScopedCallback;
  31 use Wikimedia\TestingAccessWrapper;
  32
  33 /**
  34  * @ingroup Testing
  35  */
  36 class ParserTestRunner {
  37
  38         /**
  39          * MediaWiki core parser test files, paths
  40          * will be prefixed with __DIR__ . '/'
  41          *
  42          * @var array
  43          */
  44         private static $coreTestFiles = [
  45                 'parserTests.txt',
  46                 'extraParserTests.txt',
  47         ];
  48
  49         /**
  50          * @var bool $useTemporaryTables Use temporary tables for the temporary database
  51          */
  52         private $useTemporaryTables = true;
  53
  54         /**
  55          * @var array $setupDone The status of each setup function
  56          */
  57         private $setupDone = [
  58                 'staticSetup' => false,
  59                 'perTestSetup' => false,
  60                 'setupDatabase' => false,
  61                 'setDatabase' => false,
  62                 'setupUploads' => false,
  63         ];
  64
  65         /**
  66          * Our connection to the database
  67          * @var Database
  68          */
  69         private $db;
  70
  71         /**
  72          * Database clone helper
  73          * @var CloneDatabase
  74          */
  75         private $dbClone;
  76
  77         /**
  78          * @var TidySupport
  79          */
  80         private $tidySupport;
  81
  82         /**
  83          * @var TidyDriverBase
  84          */
  85         private $tidyDriver = null;
  86
  87         /**
  88          * @var TestRecorder
  89          */
  90         private $recorder;
  91
  92         /**
  93          * The upload directory, or null to not set up an upload directory
  94          *
  95          * @var string|null
  96          */
  97         private $uploadDir = null;
  98
  99         /**
 100          * The name of the file backend to use, or null to use MockFileBackend.
 101          * @var string|null
 102          */
 103         private $fileBackendName;
 104
 105         /**
 106          * A complete regex for filtering tests.
 107          * @var string
 108          */
 109         private $regex;
 110
 111         /**
 112          * A list of normalization functions to apply to the expected and actual
 113          * output.
 114          * @var array
 115          */
 116         private $normalizationFunctions = [];
 117
 118         /**
 119          * @param TestRecorder $recorder
 120          * @param array $options
 121          */
 122         public function __construct( TestRecorder $recorder, $options = [] ) {
 123                 $this->recorder = $recorder;
 124
 125                 if ( isset( $options['norm'] ) ) {
 126                         foreach ( $options['norm'] as $func ) {
 127                                 if ( in_array( $func, [ 'removeTbody', 'trimWhitespace' ] ) ) {
 128                                         $this->normalizationFunctions[] = $func;
 129                                 } else {
 130                                         $this->recorder->warning(
 131                                                 "Warning: unknown normalization option \"$func\"\n" );
 132                                 }
 133                         }
 134                 }
 135
 136                 if ( isset( $options['regex'] ) && $options['regex'] !== false ) {
 137                         $this->regex = $options['regex'];
 138                 } else {
 139                         # Matches anything
 140                         $this->regex = '//';
 141                 }
 142
 143                 $this->keepUploads = !empty( $options['keep-uploads'] );
 144
 145                 $this->fileBackendName = isset( $options['file-backend'] ) ?
 146                         $options['file-backend'] : false;
 147
 148                 $this->runDisabled = !empty( $options['run-disabled'] );
 149                 $this->runParsoid = !empty( $options['run-parsoid'] );
 150
 151                 $this->tidySupport = new TidySupport( !empty( $options['use-tidy-config'] ) );
 152                 if ( !$this->tidySupport->isEnabled() ) {
 153                         $this->recorder->warning(
 154                                 "Warning: tidy is not installed, skipping some tests\n" );
 155                 }
 156
 157                 if ( isset( $options['upload-dir'] ) ) {
 158                         $this->uploadDir = $options['upload-dir'];
 159                 }
 160         }
 161
 162         /**
 163          * Get list of filenames to extension and core parser tests
 164          *
 165          * @return array
 166          */
 167         public static function getParserTestFiles() {
 168                 global $wgParserTestFiles;
 169
 170                 // Add core test files
 171                 $files = array_map( function ( $item ) {
 172                         return __DIR__ . "/$item";
 173                 }, self::$coreTestFiles );
 174
 175                 // Plus legacy global files
 176                 $files = array_merge( $files, $wgParserTestFiles );
 177
 178                 // Auto-discover extension parser tests
 179                 $registry = ExtensionRegistry::getInstance();
 180                 foreach ( $registry->getAllThings() as $info ) {
 181                         $dir = dirname( $info['path'] ) . '/tests/parser';
 182                         if ( !file_exists( $dir ) ) {
 183                                 continue;
 184                         }
 185                         $dirIterator = new RecursiveIteratorIterator(
 186                                 new RecursiveDirectoryIterator( $dir )
 187                         );
 188                         foreach ( $dirIterator as $fileInfo ) {
 189                                 /** @var SplFileInfo $fileInfo */
 190                                 if ( substr( $fileInfo->getFilename(), -4 ) === '.txt' ) {
 191                                         $files[] = $fileInfo->getPathname();
 192                                 }
 193                         }
 194                 }
 195
 196                 return array_unique( $files );
 197         }
 198
 199         public function getRecorder() {
 200                 return $this->recorder;
 201         }
 202
 203         /**
 204          * Do any setup which can be done once for all tests, independent of test
 205          * options, except for database setup.
 206          *
 207          * Public setup functions in this class return a ScopedCallback object. When
 208          * this object is destroyed by going out of scope, teardown of the
 209          * corresponding test setup is performed.
 210          *
 211          * Teardown objects may be chained by passing a ScopedCallback from a
 212          * previous setup stage as the $nextTeardown parameter. This enforces the
 213          * convention that teardown actions are taken in reverse order to the
 214          * corresponding setup actions. When $nextTeardown is specified, a
 215          * ScopedCallback will be returned which first tears down the current
 216          * setup stage, and then tears down the previous setup stage which was
 217          * specified by $nextTeardown.
 218          *
 219          * @param ScopedCallback|null $nextTeardown
 220          * @return ScopedCallback
 221          */
 222         public function staticSetup( $nextTeardown = null ) {
 223                 // A note on coding style:
 224
 225                 // The general idea here is to keep setup code together with
 226                 // corresponding teardown code, in a fine-grained manner. We have two
 227                 // arrays: $setup and $teardown. The code snippets in the $setup array
 228                 // are executed at the end of the method, before it returns, and the
 229                 // code snippets in the $teardown array are executed in reverse order
 230                 // when the Wikimedia\ScopedCallback object is consumed.
 231
 232                 // Because it is a common operation to save, set and restore global
 233                 // variables, we have an additional convention: when the array key of
 234                 // $setup is a string, the string is taken to be the name of the global
 235                 // variable, and the element value is taken to be the desired new value.
 236
 237                 // It's acceptable to just do the setup immediately, instead of adding
 238                 // a closure to $setup, except when the setup action depends on global
 239                 // variable initialisation being done first. In this case, you have to
 240                 // append a closure to $setup after the global variable is appended.
 241
 242                 // When you add to setup functions in this class, please keep associated
 243                 // setup and teardown actions together in the source code, and please
 244                 // add comments explaining why the setup action is necessary.
 245
 246                 $setup = [];
 247                 $teardown = [];
 248
 249                 $teardown[] = $this->markSetupDone( 'staticSetup' );
 250
 251                 // Some settings which influence HTML output
 252                 $setup['wgSitename'] = 'MediaWiki';
 253                 $setup['wgServer'] = 'http://example.org';
 254                 $setup['wgServerName'] = 'example.org';
 255                 $setup['wgScriptPath'] = '';
 256                 $setup['wgScript'] = '/index.php';
 257                 $setup['wgResourceBasePath'] = '';
 258                 $setup['wgStylePath'] = '/skins';
 259                 $setup['wgExtensionAssetsPath'] = '/extensions';
 260                 $setup['wgArticlePath'] = '/wiki/$1';
 261                 $setup['wgActionPaths'] = [];
 262                 $setup['wgVariantArticlePath'] = false;
 263                 $setup['wgUploadNavigationUrl'] = false;
 264                 $setup['wgCapitalLinks'] = true;
 265                 $setup['wgNoFollowLinks'] = true;
 266                 $setup['wgNoFollowDomainExceptions'] = [ 'no-nofollow.org' ];
 267                 $setup['wgExternalLinkTarget'] = false;
 268                 $setup['wgExperimentalHtmlIds'] = false;
 269                 $setup['wgLocaltimezone'] = 'UTC';
 270                 $setup['wgHtml5'] = true;
 271                 $setup['wgDisableLangConversion'] = false;
 272                 $setup['wgDisableTitleConversion'] = false;
 273
 274                 // "extra language links"
 275                 // see https://gerrit.wikimedia.org/r/111390
 276                 $setup['wgExtraInterlanguageLinkPrefixes'] = [ 'mul' ];
 277
 278                 // All FileRepo changes should be done here by injecting services,
 279                 // there should be no need to change global variables.
 280                 RepoGroup::setSingleton( $this->createRepoGroup() );
 281                 $teardown[] = function () {
 282                         RepoGroup::destroySingleton();
 283                 };
 284
 285                 // Set up null lock managers
 286                 $setup['wgLockManagers'] = [ [
 287                         'name' => 'fsLockManager',
 288                         'class' => 'NullLockManager',
 289                 ], [
 290                         'name' => 'nullLockManager',
 291                         'class' => 'NullLockManager',
 292                 ] ];
 293                 $reset = function () {
 294                         LockManagerGroup::destroySingletons();
 295                 };
 296                 $setup[] = $reset;
 297                 $teardown[] = $reset;
 298
 299                 // This allows article insertion into the prefixed DB
 300                 $setup['wgDefaultExternalStore'] = false;
 301
 302                 // This might slightly reduce memory usage
 303                 $setup['wgAdaptiveMessageCache'] = true;
 304
 305                 // This is essential and overrides disabling of database messages in TestSetup
 306                 $setup['wgUseDatabaseMessages'] = true;
 307                 $reset = function () {
 308                         MessageCache::destroyInstance();
 309                 };
 310                 $setup[] = $reset;
 311                 $teardown[] = $reset;
 312
 313                 // It's not necessary to actually convert any files
 314                 $setup['wgSVGConverter'] = 'null';
 315                 $setup['wgSVGConverters'] = [ 'null' => 'echo "1">$output' ];
 316
 317                 // Fake constant timestamp
 318                 Hooks::register( 'ParserGetVariableValueTs', function ( &$parser, &$ts ) {
 319                         $ts = $this->getFakeTimestamp();
 320                         return true;
 321                 } );
 322                 $teardown[] = function () {
 323                         Hooks::clear( 'ParserGetVariableValueTs' );
 324                 };
 325
 326                 $this->appendNamespaceSetup( $setup, $teardown );
 327
 328                 // Set up interwikis and append teardown function
 329                 $teardown[] = $this->setupInterwikis();
 330
 331                 // This affects title normalization in links. It invalidates
 332                 // MediaWikiTitleCodec objects.
 333                 $setup['wgLocalInterwikis'] = [ 'local', 'mi' ];
 334                 $reset = function () {
 335                         $this->resetTitleServices();
 336                 };
 337                 $setup[] = $reset;
 338                 $teardown[] = $reset;
 339
 340                 // Set up a mock MediaHandlerFactory
 341                 MediaWikiServices::getInstance()->disableService( 'MediaHandlerFactory' );
 342                 MediaWikiServices::getInstance()->redefineService(
 343                         'MediaHandlerFactory',
 344                         function ( MediaWikiServices $services ) {
 345                                 $handlers = $services->getMainConfig()->get( 'ParserTestMediaHandlers' );
 346                                 return new MediaHandlerFactory( $handlers );
 347                         }
 348                 );
 349                 $teardown[] = function () {
 350                         MediaWikiServices::getInstance()->resetServiceForTesting( 'MediaHandlerFactory' );
 351                 };
 352
 353                 // SqlBagOStuff broke when using temporary tables on r40209 (T17892).
 354                 // It seems to have been fixed since (r55079?), but regressed at some point before r85701.
 355                 // This works around it for now...
 356                 global $wgObjectCaches;
 357                 $setup['wgObjectCaches'] = [ CACHE_DB => $wgObjectCaches['hash'] ] + $wgObjectCaches;
 358                 if ( isset( ObjectCache::$instances[CACHE_DB] ) ) {
 359                         $savedCache = ObjectCache::$instances[CACHE_DB];
 360                         ObjectCache::$instances[CACHE_DB] = new HashBagOStuff;
 361                         $teardown[] = function () use ( $savedCache ) {
 362                                 ObjectCache::$instances[CACHE_DB] = $savedCache;
 363                         };
 364                 }
 365
 366                 $teardown[] = $this->executeSetupSnippets( $setup );
 367
 368                 // Schedule teardown snippets in reverse order
 369                 return $this->createTeardownObject( $teardown, $nextTeardown );
 370         }
 371
 372         private function appendNamespaceSetup( &$setup, &$teardown ) {
 373                 // Add a namespace shadowing a interwiki link, to test
 374                 // proper precedence when resolving links. (T53680)
 375                 $setup['wgExtraNamespaces'] = [
 376                         100 => 'MemoryAlpha',
 377                         101 => 'MemoryAlpha_talk'
 378                 ];
 379                 // Changing wgExtraNamespaces invalidates caches in MWNamespace and
 380                 // any live Language object, both on setup and teardown
 381                 $reset = function () {
 382                         MWNamespace::getCanonicalNamespaces( true );
 383                         $GLOBALS['wgContLang']->resetNamespaces();
 384                 };
 385                 $setup[] = $reset;
 386                 $teardown[] = $reset;
 387         }
 388
 389         /**
 390          * Create a RepoGroup object appropriate for the current configuration
 391          * @return RepoGroup
 392          */
 393         protected function createRepoGroup() {
 394                 if ( $this->uploadDir ) {
 395                         if ( $this->fileBackendName ) {
 396                                 throw new MWException( 'You cannot specify both use-filebackend and upload-dir' );
 397                         }
 398                         $backend = new FSFileBackend( [
 399                                 'name' => 'local-backend',
 400                                 'wikiId' => wfWikiID(),
 401                                 'basePath' => $this->uploadDir,
 402                                 'tmpDirectory' => wfTempDir()
 403                         ] );
 404                 } elseif ( $this->fileBackendName ) {
 405                         global $wgFileBackends;
 406                         $name = $this->fileBackendName;
 407                         $useConfig = false;
 408                         foreach ( $wgFileBackends as $conf ) {
 409                                 if ( $conf['name'] === $name ) {
 410                                         $useConfig = $conf;
 411                                 }
 412                         }
 413                         if ( $useConfig === false ) {
 414                                 throw new MWException( "Unable to find file backend \"$name\"" );
 415                         }
 416                         $useConfig['name'] = 'local-backend'; // swap name
 417                         unset( $useConfig['lockManager'] );
 418                         unset( $useConfig['fileJournal'] );
 419                         $class = $useConfig['class'];
 420                         $backend = new $class( $useConfig );
 421                 } else {
 422                         # Replace with a mock. We do not care about generating real
 423                         # files on the filesystem, just need to expose the file
 424                         # informations.
 425                         $backend = new MockFileBackend( [
 426                                 'name' => 'local-backend',
 427                                 'wikiId' => wfWikiID()
 428                         ] );
 429                 }
 430
 431                 return new RepoGroup(
 432                         [
 433                                 'class' => 'MockLocalRepo',
 434                                 'name' => 'local',
 435                                 'url' => 'http://example.com/images',
 436                                 'hashLevels' => 2,
 437                                 'transformVia404' => false,
 438                                 'backend' => $backend
 439                         ],
 440                         []
 441                 );
 442         }
 443
 444         /**
 445          * Execute an array in which elements with integer keys are taken to be
 446          * callable objects, and other elements are taken to be global variable
 447          * set operations, with the key giving the variable name and the value
 448          * giving the new global variable value. A closure is returned which, when
 449          * executed, sets the global variables back to the values they had before
 450          * this function was called.
 451          *
 452          * @see staticSetup
 453          *
 454          * @param array $setup
 455          * @return closure
 456          */
 457         protected function executeSetupSnippets( $setup ) {
 458                 $saved = [];
 459                 foreach ( $setup as $name => $value ) {
 460                         if ( is_int( $name ) ) {
 461                                 $value();
 462                         } else {
 463                                 $saved[$name] = isset( $GLOBALS[$name] ) ? $GLOBALS[$name] : null;
 464                                 $GLOBALS[$name] = $value;
 465                         }
 466                 }
 467                 return function () use ( $saved ) {
 468                         $this->executeSetupSnippets( $saved );
 469                 };
 470         }
 471
 472         /**
 473          * Take a setup array in the same format as the one given to
 474          * executeSetupSnippets(), and return a ScopedCallback which, when consumed,
 475          * executes the snippets in the setup array in reverse order. This is used
 476          * to create "teardown objects" for the public API.
 477          *
 478          * @see staticSetup
 479          *
 480          * @param array $teardown The snippet array
 481          * @param ScopedCallback|null $nextTeardown A ScopedCallback to consume
 482          * @return ScopedCallback
 483          */
 484         protected function createTeardownObject( $teardown, $nextTeardown = null ) {
 485                 return new ScopedCallback( function () use ( $teardown, $nextTeardown ) {
 486                         // Schedule teardown snippets in reverse order
 487                         $teardown = array_reverse( $teardown );
 488
 489                         $this->executeSetupSnippets( $teardown );
 490                         if ( $nextTeardown ) {
 491                                 ScopedCallback::consume( $nextTeardown );
 492                         }
 493                 } );
 494         }
 495
 496         /**
 497          * Set a setupDone flag to indicate that setup has been done, and return
 498          * the teardown closure. If the flag was already set, throw an exception.
 499          *
 500          * @param string $funcName The setup function name
 501          * @return closure
 502          */
 503         protected function markSetupDone( $funcName ) {
 504                 if ( $this->setupDone[$funcName] ) {
 505                         throw new MWException( "$funcName is already done" );
 506                 }
 507                 $this->setupDone[$funcName] = true;
 508                 return function () use ( $funcName ) {
 509                         $this->setupDone[$funcName] = false;
 510                 };
 511         }
 512
 513         /**
 514          * Ensure a given setup stage has been done, throw an exception if it has
 515          * not.
 516          */
 517         protected function checkSetupDone( $funcName, $funcName2 = null ) {
 518                 if ( !$this->setupDone[$funcName]
 519                         && ( $funcName === null || !$this->setupDone[$funcName2] )
 520                 ) {
 521                         throw new MWException( "$funcName must be called before calling " .
 522                                 wfGetCaller() );
 523                 }
 524         }
 525
 526         /**
 527          * Determine whether a particular setup function has been run
 528          *
 529          * @param string $funcName
 530          * @return bool
 531          */
 532         public function isSetupDone( $funcName ) {
 533                 return isset( $this->setupDone[$funcName] ) ? $this->setupDone[$funcName] : false;
 534         }
 535
 536         /**
 537          * Insert hardcoded interwiki in the lookup table.
 538          *
 539          * This function insert a set of well known interwikis that are used in
 540          * the parser tests. They can be considered has fixtures are injected in
 541          * the interwiki cache by using the 'InterwikiLoadPrefix' hook.
 542          * Since we are not interested in looking up interwikis in the database,
 543          * the hook completely replace the existing mechanism (hook returns false).
 544          *
 545          * @return closure for teardown
 546          */
 547         private function setupInterwikis() {
 548                 # Hack: insert a few Wikipedia in-project interwiki prefixes,
 549                 # for testing inter-language links
 550                 Hooks::register( 'InterwikiLoadPrefix', function ( $prefix, &$iwData ) {
 551                         static $testInterwikis = [
 552                                 'local' => [
 553                                         'iw_url' => 'http://doesnt.matter.org/$1',
 554                                         'iw_api' => '',
 555                                         'iw_wikiid' => '',
 556                                         'iw_local' => 0 ],
 557                                 'wikipedia' => [
 558                                         'iw_url' => 'http://en.wikipedia.org/wiki/$1',
 559                                         'iw_api' => '',
 560                                         'iw_wikiid' => '',
 561                                         'iw_local' => 0 ],
 562                                 'meatball' => [
 563                                         'iw_url' => 'http://www.usemod.com/cgi-bin/mb.pl?$1',
 564                                         'iw_api' => '',
 565                                         'iw_wikiid' => '',
 566                                         'iw_local' => 0 ],
 567                                 'memoryalpha' => [
 568                                         'iw_url' => 'http://www.memory-alpha.org/en/index.php/$1',
 569                                         'iw_api' => '',
 570                                         'iw_wikiid' => '',
 571                                         'iw_local' => 0 ],
 572                                 'zh' => [
 573                                         'iw_url' => 'http://zh.wikipedia.org/wiki/$1',
 574                                         'iw_api' => '',
 575                                         'iw_wikiid' => '',
 576                                         'iw_local' => 1 ],
 577                                 'es' => [
 578                                         'iw_url' => 'http://es.wikipedia.org/wiki/$1',
 579                                         'iw_api' => '',
 580                                         'iw_wikiid' => '',
 581                                         'iw_local' => 1 ],
 582                                 'fr' => [
 583                                         'iw_url' => 'http://fr.wikipedia.org/wiki/$1',
 584                                         'iw_api' => '',
 585                                         'iw_wikiid' => '',
 586                                         'iw_local' => 1 ],
 587                                 'ru' => [
 588                                         'iw_url' => 'http://ru.wikipedia.org/wiki/$1',
 589                                         'iw_api' => '',
 590                                         'iw_wikiid' => '',
 591                                         'iw_local' => 1 ],
 592                                 'mi' => [
 593                                         'iw_url' => 'http://mi.wikipedia.org/wiki/$1',
 594                                         'iw_api' => '',
 595                                         'iw_wikiid' => '',
 596                                         'iw_local' => 1 ],
 597                                 'mul' => [
 598                                         'iw_url' => 'http://wikisource.org/wiki/$1',
 599                                         'iw_api' => '',
 600                                         'iw_wikiid' => '',
 601                                         'iw_local' => 1 ],
 602                         ];
 603                         if ( array_key_exists( $prefix, $testInterwikis ) ) {
 604                                 $iwData = $testInterwikis[$prefix];
 605                         }
 606
 607                         // We only want to rely on the above fixtures
 608                         return false;
 609                 } );// hooks::register
 610
 611                 return function () {
 612                         // Tear down
 613                         Hooks::clear( 'InterwikiLoadPrefix' );
 614                 };
 615         }
 616
 617         /**
 618          * Reset the Title-related services that need resetting
 619          * for each test
 620          */
 621         private function resetTitleServices() {
 622                 $services = MediaWikiServices::getInstance();
 623                 $services->resetServiceForTesting( 'TitleFormatter' );
 624                 $services->resetServiceForTesting( 'TitleParser' );
 625                 $services->resetServiceForTesting( '_MediaWikiTitleCodec' );
 626                 $services->resetServiceForTesting( 'LinkRenderer' );
 627                 $services->resetServiceForTesting( 'LinkRendererFactory' );
 628         }
 629
 630         /**
 631          * Remove last character if it is a newline
 632          * @group utility
 633          * @param string $s
 634          * @return string
 635          */
 636         public static function chomp( $s ) {
 637                 if ( substr( $s, -1 ) === "\n" ) {
 638                         return substr( $s, 0, -1 );
 639                 } else {
 640                         return $s;
 641                 }
 642         }
 643
 644         /**
 645          * Run a series of tests listed in the given text files.
 646          * Each test consists of a brief description, wikitext input,
 647          * and the expected HTML output.
 648          *
 649          * Prints status updates on stdout and counts up the total
 650          * number and percentage of passed tests.
 651          *
 652          * Handles all setup and teardown.
 653          *
 654          * @param array $filenames Array of strings
 655          * @return bool True if passed all tests, false if any tests failed.
 656          */
 657         public function runTestsFromFiles( $filenames ) {
 658                 $ok = false;
 659
 660                 $teardownGuard = $this->staticSetup();
 661                 $teardownGuard = $this->setupDatabase( $teardownGuard );
 662                 $teardownGuard = $this->setupUploads( $teardownGuard );
 663
 664                 $this->recorder->start();
 665                 try {
 666                         $ok = true;
 667
 668                         foreach ( $filenames as $filename ) {
 669                                 $testFileInfo = TestFileReader::read( $filename, [
 670                                         'runDisabled' => $this->runDisabled,
 671                                         'runParsoid' => $this->runParsoid,
 672                                         'regex' => $this->regex ] );
 673
 674                                 // Don't start the suite if there are no enabled tests in the file
 675                                 if ( !$testFileInfo['tests'] ) {
 676                                         continue;
 677                                 }
 678
 679                                 $this->recorder->startSuite( $filename );
 680                                 $ok = $this->runTests( $testFileInfo ) && $ok;
 681                                 $this->recorder->endSuite( $filename );
 682                         }
 683
 684                         $this->recorder->report();
 685                 } catch ( DBError $e ) {
 686                         $this->recorder->warning( $e->getMessage() );
 687                 }
 688                 $this->recorder->end();
 689
 690                 ScopedCallback::consume( $teardownGuard );
 691
 692                 return $ok;
 693         }
 694
 695         /**
 696          * Determine whether the current parser has the hooks registered in it
 697          * that are required by a file read by TestFileReader.
 698          */
 699         public function meetsRequirements( $requirements ) {
 700                 foreach ( $requirements as $requirement ) {
 701                         switch ( $requirement['type'] ) {
 702                         case 'hook':
 703                                 $ok = $this->requireHook( $requirement['name'] );
 704                                 break;
 705                         case 'functionHook':
 706                                 $ok = $this->requireFunctionHook( $requirement['name'] );
 707                                 break;
 708                         case 'transparentHook':
 709                                 $ok = $this->requireTransparentHook( $requirement['name'] );
 710                                 break;
 711                         }
 712                         if ( !$ok ) {
 713                                 return false;
 714                         }
 715                 }
 716                 return true;
 717         }
 718
 719         /**
 720          * Run the tests from a single file. staticSetup() and setupDatabase()
 721          * must have been called already.
 722          *
 723          * @param array $testFileInfo Parsed file info returned by TestFileReader
 724          * @return bool True if passed all tests, false if any tests failed.
 725          */
 726         public function runTests( $testFileInfo ) {
 727                 $ok = true;
 728
 729                 $this->checkSetupDone( 'staticSetup' );
 730
 731                 // Don't add articles from the file if there are no enabled tests from the file
 732                 if ( !$testFileInfo['tests'] ) {
 733                         return true;
 734                 }
 735
 736                 // If any requirements are not met, mark all tests from the file as skipped
 737                 if ( !$this->meetsRequirements( $testFileInfo['requirements'] ) ) {
 738                         foreach ( $testFileInfo['tests'] as $test ) {
 739                                 $this->recorder->startTest( $test );
 740                                 $this->recorder->skipped( $test, 'required extension not enabled' );
 741                         }
 742                         return true;
 743                 }
 744
 745                 // Add articles
 746                 $this->addArticles( $testFileInfo['articles'] );
 747
 748                 // Run tests
 749                 foreach ( $testFileInfo['tests'] as $test ) {
 750                         $this->recorder->startTest( $test );
 751                         $result =
 752                                 $this->runTest( $test );
 753                         if ( $result !== false ) {
 754                                 $ok = $ok && $result->isSuccess();
 755                                 $this->recorder->record( $test, $result );
 756                         }
 757                 }
 758
 759                 return $ok;
 760         }
 761
 762         /**
 763          * Get a Parser object
 764          *
 765          * @param string $preprocessor
 766          * @return Parser
 767          */
 768         function getParser( $preprocessor = null ) {
 769                 global $wgParserConf;
 770
 771                 $class = $wgParserConf['class'];
 772                 $parser = new $class( [ 'preprocessorClass' => $preprocessor ] + $wgParserConf );
 773                 ParserTestParserHook::setup( $parser );
 774
 775                 return $parser;
 776         }
 777
 778         /**
 779          * Run a given wikitext input through a freshly-constructed wiki parser,
 780          * and compare the output against the expected results.
 781          * Prints status and explanatory messages to stdout.
 782          *
 783          * staticSetup() and setupWikiData() must be called before this function
 784          * is entered.
 785          *
 786          * @param array $test The test parameters:
 787          *  - test: The test name
 788          *  - desc: The subtest description
 789          *  - input: Wikitext to try rendering
 790          *  - options: Array of test options
 791          *  - config: Overrides for global variables, one per line
 792          *
 793          * @return ParserTestResult or false if skipped
 794          */
 795         public function runTest( $test ) {
 796                 wfDebug( __METHOD__.": running {$test['desc']}" );
 797                 $opts = $this->parseOptions( $test['options'] );
 798                 $teardownGuard = $this->perTestSetup( $test );
 799
 800                 $context = RequestContext::getMain();
 801                 $user = $context->getUser();
 802                 $options = ParserOptions::newFromContext( $context );
 803                 $options->setTimestamp( $this->getFakeTimestamp() );
 804
 805                 if ( !isset( $opts['wrap'] ) ) {
 806                         $options->setWrapOutputClass( false );
 807                 }
 808
 809                 if ( isset( $opts['tidy'] ) ) {
 810                         if ( !$this->tidySupport->isEnabled() ) {
 811                                 $this->recorder->skipped( $test, 'tidy extension is not installed' );
 812                                 return false;
 813                         } else {
 814                                 $options->setTidy( true );
 815                         }
 816                 }
 817
 818                 if ( isset( $opts['title'] ) ) {
 819                         $titleText = $opts['title'];
 820                 } else {
 821                         $titleText = 'Parser test';
 822                 }
 823
 824                 $local = isset( $opts['local'] );
 825                 $preprocessor = isset( $opts['preprocessor'] ) ? $opts['preprocessor'] : null;
 826                 $parser = $this->getParser( $preprocessor );
 827                 $title = Title::newFromText( $titleText );
 828
 829                 if ( isset( $opts['pst'] ) ) {
 830                         $out = $parser->preSaveTransform( $test['input'], $title, $user, $options );
 831                         $output = $parser->getOutput();
 832                 } elseif ( isset( $opts['msg'] ) ) {
 833                         $out = $parser->transformMsg( $test['input'], $options, $title );
 834                 } elseif ( isset( $opts['section'] ) ) {
 835                         $section = $opts['section'];
 836                         $out = $parser->getSection( $test['input'], $section );
 837                 } elseif ( isset( $opts['replace'] ) ) {
 838                         $section = $opts['replace'][0];
 839                         $replace = $opts['replace'][1];
 840                         $out = $parser->replaceSection( $test['input'], $section, $replace );
 841                 } elseif ( isset( $opts['comment'] ) ) {
 842                         $out = Linker::formatComment( $test['input'], $title, $local );
 843                 } elseif ( isset( $opts['preload'] ) ) {
 844                         $out = $parser->getPreloadText( $test['input'], $title, $options );
 845                 } else {
 846                         $output = $parser->parse( $test['input'], $title, $options, true, true, 1337 );
 847                         $output->setTOCEnabled( !isset( $opts['notoc'] ) );
 848                         $out = $output->getText();
 849                         if ( isset( $opts['tidy'] ) ) {
 850                                 $out = preg_replace( '/\s+$/', '', $out );
 851                         }
 852
 853                         if ( isset( $opts['showtitle'] ) ) {
 854                                 if ( $output->getTitleText() ) {
 855                                         $title = $output->getTitleText();
 856                                 }
 857
 858                                 $out = "$title\n$out";
 859                         }
 860
 861                         if ( isset( $opts['showindicators'] ) ) {
 862                                 $indicators = '';
 863                                 foreach ( $output->getIndicators() as $id => $content ) {
 864                                         $indicators .= "$id=$content\n";
 865                                 }
 866                                 $out = $indicators . $out;
 867                         }
 868
 869                         if ( isset( $opts['ill'] ) ) {
 870                                 $out = implode( ' ', $output->getLanguageLinks() );
 871                         } elseif ( isset( $opts['cat'] ) ) {
 872                                 $out = '';
 873                                 foreach ( $output->getCategories() as $name => $sortkey ) {
 874                                         if ( $out !== '' ) {
 875                                                 $out .= "\n";
 876                                         }
 877                                         $out .= "cat=$name sort=$sortkey";
 878                                 }
 879                         }
 880                 }
 881
 882                 if ( isset( $output ) && isset( $opts['showflags'] ) ) {
 883                         $actualFlags = array_keys( TestingAccessWrapper::newFromObject( $output )->mFlags );
 884                         sort( $actualFlags );
 885                         $out .= "\nflags=" . join( ', ', $actualFlags );
 886                 }
 887
 888                 ScopedCallback::consume( $teardownGuard );
 889
 890                 $expected = $test['result'];
 891                 if ( count( $this->normalizationFunctions ) ) {
 892                         $expected = ParserTestResultNormalizer::normalize(
 893                                 $test['expected'], $this->normalizationFunctions );
 894                         $out = ParserTestResultNormalizer::normalize( $out, $this->normalizationFunctions );
 895                 }
 896
 897                 $testResult = new ParserTestResult( $test, $expected, $out );
 898                 return $testResult;
 899         }
 900
 901         /**
 902          * Use a regex to find out the value of an option
 903          * @param string $key Name of option val to retrieve
 904          * @param array $opts Options array to look in
 905          * @param mixed $default Default value returned if not found
 906          * @return mixed
 907          */
 908         private static function getOptionValue( $key, $opts, $default ) {
 909                 $key = strtolower( $key );
 910
 911                 if ( isset( $opts[$key] ) ) {
 912                         return $opts[$key];
 913                 } else {
 914                         return $default;
 915                 }
 916         }
 917
 918         /**
 919          * Given the options string, return an associative array of options.
 920          * @todo Move this to TestFileReader
 921          *
 922          * @param string $instring
 923          * @return array
 924          */
 925         private function parseOptions( $instring ) {
 926                 $opts = [];
 927                 // foo
 928                 // foo=bar
 929                 // foo="bar baz"
 930                 // foo=[[bar baz]]
 931                 // foo=bar,"baz quux"
 932                 // foo={...json...}
 933                 $defs = '(?(DEFINE)
 934                         (?<qstr>                                        # Quoted string
 935                                 "
 936                                 (?:[^\\\\"] | \\\\.)*
 937                                 "
 938                         )
 939                         (?<json>
 940                                 \{              # Open bracket
 941                                 (?:
 942                                         [^"{}] |                                # Not a quoted string or object, or
 943                                         (?&qstr) |                              # A quoted string, or
 944                                         (?&json)                                # A json object (recursively)
 945                                 )*
 946                                 \}              # Close bracket
 947                         )
 948                         (?<value>
 949                                 (?:
 950                                         (?&qstr)                        # Quoted val
 951                                 |
 952                                         \[\[
 953                                                 [^]]*                   # Link target
 954                                         \]\]
 955                                 |
 956                                         [\w-]+                          # Plain word
 957                                 |
 958                                         (?&json)                        # JSON object
 959                                 )
 960                         )
 961                 )';
 962                 $regex = '/' . $defs . '\b
 963                         (?<k>[\w-]+)                            # Key
 964                         \b
 965                         (?:\s*
 966                                 =                                               # First sub-value
 967                                 \s*
 968                                 (?<v>
 969                                         (?&value)
 970                                         (?:\s*
 971                                                 ,                               # Sub-vals 1..N
 972                                                 \s*
 973                                                 (?&value)
 974                                         )*
 975                                 )
 976                         )?
 977                         /x';
 978                 $valueregex = '/' . $defs . '(?&value)/x';
 979
 980                 if ( preg_match_all( $regex, $instring, $matches, PREG_SET_ORDER ) ) {
 981                         foreach ( $matches as $bits ) {
 982                                 $key = strtolower( $bits['k'] );
 983                                 if ( !isset( $bits['v'] ) ) {
 984                                         $opts[$key] = true;
 985                                 } else {
 986                                         preg_match_all( $valueregex, $bits['v'], $vmatches );
 987                                         $opts[$key] = array_map( [ $this, 'cleanupOption' ], $vmatches[0] );
 988                                         if ( count( $opts[$key] ) == 1 ) {
 989                                                 $opts[$key] = $opts[$key][0];
 990                                         }
 991                                 }
 992                         }
 993                 }
 994                 return $opts;
 995         }
 996
 997         private function cleanupOption( $opt ) {
 998                 if ( substr( $opt, 0, 1 ) == '"' ) {
 999                         return stripcslashes( substr( $opt, 1, -1 ) );
1000                 }
1001
1002                 if ( substr( $opt, 0, 2 ) == '[[' ) {
1003                         return substr( $opt, 2, -2 );
1004                 }
1005
1006                 if ( substr( $opt, 0, 1 ) == '{' ) {
1007                         return FormatJson::decode( $opt, true );
1008                 }
1009                 return $opt;
1010         }
1011
1012         /**
1013          * Do any required setup which is dependent on test options.
1014          *
1015          * @see staticSetup() for more information about setup/teardown
1016          *
1017          * @param array $test Test info supplied by TestFileReader
1018          * @param callable|null $nextTeardown
1019          * @return ScopedCallback
1020          */
1021         public function perTestSetup( $test, $nextTeardown = null ) {
1022                 $teardown = [];
1023
1024                 $this->checkSetupDone( 'setupDatabase', 'setDatabase' );
1025                 $teardown[] = $this->markSetupDone( 'perTestSetup' );
1026
1027                 $opts = $this->parseOptions( $test['options'] );
1028                 $config = $test['config'];
1029
1030                 // Find out values for some special options.
1031                 $langCode =
1032                         self::getOptionValue( 'language', $opts, 'en' );
1033                 $variant =
1034                         self::getOptionValue( 'variant', $opts, false );
1035                 $maxtoclevel =
1036                         self::getOptionValue( 'wgMaxTocLevel', $opts, 999 );
1037                 $linkHolderBatchSize =
1038                         self::getOptionValue( 'wgLinkHolderBatchSize', $opts, 1000 );
1039
1040                 // Default to fallback skin, but allow it to be overridden
1041                 $skin = self::getOptionValue( 'skin', $opts, 'fallback' );
1042
1043                 $setup = [
1044                         'wgEnableUploads' => self::getOptionValue( 'wgEnableUploads', $opts, true ),
1045                         'wgLanguageCode' => $langCode,
1046                         'wgRawHtml' => self::getOptionValue( 'wgRawHtml', $opts, false ),
1047                         'wgNamespacesWithSubpages' => array_fill_keys(
1048                                 MWNamespace::getValidNamespaces(), isset( $opts['subpage'] )
1049                         ),
1050                         'wgMaxTocLevel' => $maxtoclevel,
1051                         'wgAllowExternalImages' => self::getOptionValue( 'wgAllowExternalImages', $opts, true ),
1052                         'wgThumbLimits' => [ self::getOptionValue( 'thumbsize', $opts, 180 ) ],
1053                         'wgDefaultLanguageVariant' => $variant,
1054                         'wgLinkHolderBatchSize' => $linkHolderBatchSize,
1055                         // Set as a JSON object like:
1056                         // wgEnableMagicLinks={"ISBN":false, "PMID":false, "RFC":false}
1057                         'wgEnableMagicLinks' => self::getOptionValue( 'wgEnableMagicLinks', $opts, [] )
1058                                 + [ 'ISBN' => true, 'PMID' => true, 'RFC' => true ],
1059                         // Test with legacy encoding by default until HTML5 is very stable and default
1060                         'wgFragmentMode' => [ 'legacy' ],
1061                 ];
1062
1063                 if ( $config ) {
1064                         $configLines = explode( "\n", $config );
1065
1066                         foreach ( $configLines as $line ) {
1067                                 list( $var, $value )  = explode( '=', $line, 2 );
1068                                 $setup[$var] = eval( "return $value;" );
1069                         }
1070                 }
1071
1072                 /** @since 1.20 */
1073                 Hooks::run( 'ParserTestGlobals', [ &$setup ] );
1074
1075                 // Create tidy driver
1076                 if ( isset( $opts['tidy'] ) ) {
1077                         // Cache a driver instance
1078                         if ( $this->tidyDriver === null ) {
1079                                 $this->tidyDriver = MWTidy::factory( $this->tidySupport->getConfig() );
1080                         }
1081                         $tidy = $this->tidyDriver;
1082                 } else {
1083                         $tidy = false;
1084                 }
1085                 MWTidy::setInstance( $tidy );
1086                 $teardown[] = function () {
1087                         MWTidy::destroySingleton();
1088                 };
1089
1090                 // Set content language. This invalidates the magic word cache and title services
1091                 $lang = Language::factory( $langCode );
1092                 $setup['wgContLang'] = $lang;
1093                 $reset = function () {
1094                         MagicWord::clearCache();
1095                         $this->resetTitleServices();
1096                 };
1097                 $setup[] = $reset;
1098                 $teardown[] = $reset;
1099
1100                 // Make a user object with the same language
1101                 $user = new User;
1102                 $user->setOption( 'language', $langCode );
1103                 $setup['wgLang'] = $lang;
1104
1105                 // We (re)set $wgThumbLimits to a single-element array above.
1106                 $user->setOption( 'thumbsize', 0 );
1107
1108                 $setup['wgUser'] = $user;
1109
1110                 // And put both user and language into the context
1111                 $context = RequestContext::getMain();
1112                 $context->setUser( $user );
1113                 $context->setLanguage( $lang );
1114                 // And the skin!
1115                 $oldSkin = $context->getSkin();
1116                 $skinFactory = MediaWikiServices::getInstance()->getSkinFactory();
1117                 $context->setSkin( $skinFactory->makeSkin( $skin ) );
1118                 $context->setOutput( new OutputPage( $context ) );
1119                 $setup['wgOut'] = $context->getOutput();
1120                 $teardown[] = function () use ( $context, $oldSkin ) {
1121                         // Clear language conversion tables
1122                         $wrapper = TestingAccessWrapper::newFromObject(
1123                                 $context->getLanguage()->getConverter()
1124                         );
1125                         $wrapper->reloadTables();
1126                         // Reset context to the restored globals
1127                         $context->setUser( $GLOBALS['wgUser'] );
1128                         $context->setLanguage( $GLOBALS['wgContLang'] );
1129                         $context->setSkin( $oldSkin );
1130                         $context->setOutput( $GLOBALS['wgOut'] );
1131                 };
1132
1133                 $teardown[] = $this->executeSetupSnippets( $setup );
1134
1135                 return $this->createTeardownObject( $teardown, $nextTeardown );
1136         }
1137
1138         /**
1139          * List of temporary tables to create, without prefix.
1140          * Some of these probably aren't necessary.
1141          * @return array
1142          */
1143         private function listTables() {
1144                 $tables = [ 'user', 'user_properties', 'user_former_groups', 'page', 'page_restrictions',
1145                         'protected_titles', 'revision', 'text', 'pagelinks', 'imagelinks',
1146                         'categorylinks', 'templatelinks', 'externallinks', 'langlinks', 'iwlinks',
1147                         'site_stats', 'ipblocks', 'image', 'oldimage',
1148                         'recentchanges', 'watchlist', 'interwiki', 'logging', 'log_search',
1149                         'querycache', 'objectcache', 'job', 'l10n_cache', 'redirect', 'querycachetwo',
1150                         'archive', 'user_groups', 'page_props', 'category'
1151                 ];
1152
1153                 if ( in_array( $this->db->getType(), [ 'mysql', 'sqlite', 'oracle' ] ) ) {
1154                         array_push( $tables, 'searchindex' );
1155                 }
1156
1157                 // Allow extensions to add to the list of tables to duplicate;
1158                 // may be necessary if they hook into page save or other code
1159                 // which will require them while running tests.
1160                 Hooks::run( 'ParserTestTables', [ &$tables ] );
1161
1162                 return $tables;
1163         }
1164
1165         public function setDatabase( IDatabase $db ) {
1166                 $this->db = $db;
1167                 $this->setupDone['setDatabase'] = true;
1168         }
1169
1170         /**
1171          * Set up temporary DB tables.
1172          *
1173          * For best performance, call this once only for all tests. However, it can
1174          * be called at the start of each test if more isolation is desired.
1175          *
1176          * @todo: This is basically an unrefactored copy of
1177          * MediaWikiTestCase::setupAllTestDBs. They should be factored out somehow.
1178          *
1179          * Do not call this function from a MediaWikiTestCase subclass, since
1180          * MediaWikiTestCase does its own DB setup. Instead use setDatabase().
1181          *
1182          * @see staticSetup() for more information about setup/teardown
1183          *
1184          * @param ScopedCallback|null $nextTeardown The next teardown object
1185          * @return ScopedCallback The teardown object
1186          */
1187         public function setupDatabase( $nextTeardown = null ) {
1188                 global $wgDBprefix;
1189
1190                 $this->db = wfGetDB( DB_MASTER );
1191                 $dbType = $this->db->getType();
1192
1193                 if ( $dbType == 'oracle' ) {
1194                         $suspiciousPrefixes = [ 'pt_', MediaWikiTestCase::ORA_DB_PREFIX ];
1195                 } else {
1196                         $suspiciousPrefixes = [ 'parsertest_', MediaWikiTestCase::DB_PREFIX ];
1197                 }
1198                 if ( in_array( $wgDBprefix, $suspiciousPrefixes ) ) {
1199                         throw new MWException( "\$wgDBprefix=$wgDBprefix suggests DB setup is already done" );
1200                 }
1201
1202                 $teardown = [];
1203
1204                 $teardown[] = $this->markSetupDone( 'setupDatabase' );
1205
1206                 # CREATE TEMPORARY TABLE breaks if there is more than one server
1207                 if ( wfGetLB()->getServerCount() != 1 ) {
1208                         $this->useTemporaryTables = false;
1209                 }
1210
1211                 $temporary = $this->useTemporaryTables || $dbType == 'postgres';
1212                 $prefix = $dbType != 'oracle' ? 'parsertest_' : 'pt_';
1213
1214                 $this->dbClone = new CloneDatabase( $this->db, $this->listTables(), $prefix );
1215                 $this->dbClone->useTemporaryTables( $temporary );
1216                 $this->dbClone->cloneTableStructure();
1217
1218                 if ( $dbType == 'oracle' ) {
1219                         $this->db->query( 'BEGIN FILL_WIKI_INFO; END;' );
1220                         # Insert 0 user to prevent FK violations
1221
1222                         # Anonymous user
1223                         $this->db->insert( 'user', [
1224                                 'user_id' => 0,
1225                                 'user_name' => 'Anonymous' ] );
1226                 }
1227
1228                 $teardown[] = function () {
1229                         $this->teardownDatabase();
1230                 };
1231
1232                 // Wipe some DB query result caches on setup and teardown
1233                 $reset = function () {
1234                         LinkCache::singleton()->clear();
1235
1236                         // Clear the message cache
1237                         MessageCache::singleton()->clear();
1238                 };
1239                 $reset();
1240                 $teardown[] = $reset;
1241                 return $this->createTeardownObject( $teardown, $nextTeardown );
1242         }
1243
1244         /**
1245          * Add data about uploads to the new test DB, and set up the upload
1246          * directory. This should be called after either setDatabase() or
1247          * setupDatabase().
1248          *
1249          * @param ScopedCallback|null $nextTeardown The next teardown object
1250          * @return ScopedCallback The teardown object
1251          */
1252         public function setupUploads( $nextTeardown = null ) {
1253                 $teardown = [];
1254
1255                 $this->checkSetupDone( 'setupDatabase', 'setDatabase' );
1256                 $teardown[] = $this->markSetupDone( 'setupUploads' );
1257
1258                 // Create the files in the upload directory (or pretend to create them
1259                 // in a MockFileBackend). Append teardown callback.
1260                 $teardown[] = $this->setupUploadBackend();
1261
1262                 // Create a user
1263                 $user = User::createNew( 'WikiSysop' );
1264
1265                 // Register the uploads in the database
1266
1267                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Foobar.jpg' ) );
1268                 # note that the size/width/height/bits/etc of the file
1269                 # are actually set by inspecting the file itself; the arguments
1270                 # to recordUpload2 have no effect.  That said, we try to make things
1271                 # match up so it is less confusing to readers of the code & tests.
1272                 $image->recordUpload2( '', 'Upload of some lame file', 'Some lame file', [
1273                         'size' => 7881,
1274                         'width' => 1941,
1275                         'height' => 220,
1276                         'bits' => 8,
1277                         'media_type' => MEDIATYPE_BITMAP,
1278                         'mime' => 'image/jpeg',
1279                         'metadata' => serialize( [] ),
1280                         'sha1' => Wikimedia\base_convert( '1', 16, 36, 31 ),
1281                         'fileExists' => true
1282                 ], $this->db->timestamp( '20010115123500' ), $user );
1283
1284                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Thumb.png' ) );
1285                 # again, note that size/width/height below are ignored; see above.
1286                 $image->recordUpload2( '', 'Upload of some lame thumbnail', 'Some lame thumbnail', [
1287                         'size' => 22589,
1288                         'width' => 135,
1289                         'height' => 135,
1290                         'bits' => 8,
1291                         'media_type' => MEDIATYPE_BITMAP,
1292                         'mime' => 'image/png',
1293                         'metadata' => serialize( [] ),
1294                         'sha1' => Wikimedia\base_convert( '2', 16, 36, 31 ),
1295                         'fileExists' => true
1296                 ], $this->db->timestamp( '20130225203040' ), $user );
1297
1298                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Foobar.svg' ) );
1299                 $image->recordUpload2( '', 'Upload of some lame SVG', 'Some lame SVG', [
1300                                 'size'        => 12345,
1301                                 'width'       => 240,
1302                                 'height'      => 180,
1303                                 'bits'        => 0,
1304                                 'media_type'  => MEDIATYPE_DRAWING,
1305                                 'mime'        => 'image/svg+xml',
1306                                 'metadata'    => serialize( [] ),
1307                                 'sha1'        => Wikimedia\base_convert( '', 16, 36, 31 ),
1308                                 'fileExists'  => true
1309                 ], $this->db->timestamp( '20010115123500' ), $user );
1310
1311                 # This image will be blacklisted in [[MediaWiki:Bad image list]]
1312                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Bad.jpg' ) );
1313                 $image->recordUpload2( '', 'zomgnotcensored', 'Borderline image', [
1314                         'size' => 12345,
1315                         'width' => 320,
1316                         'height' => 240,
1317                         'bits' => 24,
1318                         'media_type' => MEDIATYPE_BITMAP,
1319                         'mime' => 'image/jpeg',
1320                         'metadata' => serialize( [] ),
1321                         'sha1' => Wikimedia\base_convert( '3', 16, 36, 31 ),
1322                         'fileExists' => true
1323                 ], $this->db->timestamp( '20010115123500' ), $user );
1324
1325                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Video.ogv' ) );
1326                 $image->recordUpload2( '', 'A pretty movie', 'Will it play', [
1327                         'size' => 12345,
1328                         'width' => 320,
1329                         'height' => 240,
1330                         'bits' => 0,
1331                         'media_type' => MEDIATYPE_VIDEO,
1332                         'mime' => 'application/ogg',
1333                         'metadata' => serialize( [] ),
1334                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1335                         'fileExists' => true
1336                 ], $this->db->timestamp( '20010115123500' ), $user );
1337
1338                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Audio.oga' ) );
1339                 $image->recordUpload2( '', 'An awesome hitsong', 'Will it play', [
1340                         'size' => 12345,
1341                         'width' => 0,
1342                         'height' => 0,
1343                         'bits' => 0,
1344                         'media_type' => MEDIATYPE_AUDIO,
1345                         'mime' => 'application/ogg',
1346                         'metadata' => serialize( [] ),
1347                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1348                         'fileExists' => true
1349                 ], $this->db->timestamp( '20010115123500' ), $user );
1350
1351                 # A DjVu file
1352                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'LoremIpsum.djvu' ) );
1353                 $image->recordUpload2( '', 'Upload a DjVu', 'A DjVu', [
1354                         'size' => 3249,
1355                         'width' => 2480,
1356                         'height' => 3508,
1357                         'bits' => 0,
1358                         'media_type' => MEDIATYPE_BITMAP,
1359                         'mime' => 'image/vnd.djvu',
1360                         'metadata' => '<?xml version="1.0" ?>
1361 <!DOCTYPE DjVuXML PUBLIC "-//W3C//DTD DjVuXML 1.1//EN" "pubtext/DjVuXML-s.dtd">
1362 <DjVuXML>
1363 <HEAD></HEAD>
1364 <BODY><OBJECT height="3508" width="2480">
1365 <PARAM name="DPI" value="300" />
1366 <PARAM name="GAMMA" value="2.2" />
1367 </OBJECT>
1368 <OBJECT height="3508" width="2480">
1369 <PARAM name="DPI" value="300" />
1370 <PARAM name="GAMMA" value="2.2" />
1371 </OBJECT>
1372 <OBJECT height="3508" width="2480">
1373 <PARAM name="DPI" value="300" />
1374 <PARAM name="GAMMA" value="2.2" />
1375 </OBJECT>
1376 <OBJECT height="3508" width="2480">
1377 <PARAM name="DPI" value="300" />
1378 <PARAM name="GAMMA" value="2.2" />
1379 </OBJECT>
1380 <OBJECT height="3508" width="2480">
1381 <PARAM name="DPI" value="300" />
1382 <PARAM name="GAMMA" value="2.2" />
1383 </OBJECT>
1384 </BODY>
1385 </DjVuXML>',
1386                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1387                         'fileExists' => true
1388                 ], $this->db->timestamp( '20010115123600' ), $user );
1389
1390                 return $this->createTeardownObject( $teardown, $nextTeardown );
1391         }
1392
1393         /**
1394          * Helper for database teardown, called from the teardown closure. Destroy
1395          * the database clone and fix up some things that CloneDatabase doesn't fix.
1396          *
1397          * @todo Move most things here to CloneDatabase
1398          */
1399         private function teardownDatabase() {
1400                 $this->checkSetupDone( 'setupDatabase' );
1401
1402                 $this->dbClone->destroy();
1403                 $this->databaseSetupDone = false;
1404
1405                 if ( $this->useTemporaryTables ) {
1406                         if ( $this->db->getType() == 'sqlite' ) {
1407                                 # Under SQLite the searchindex table is virtual and need
1408                                 # to be explicitly destroyed. See T31912
1409                                 # See also MediaWikiTestCase::destroyDB()
1410                                 wfDebug( __METHOD__ . " explicitly destroying sqlite virtual table parsertest_searchindex\n" );
1411                                 $this->db->query( "DROP TABLE `parsertest_searchindex`" );
1412                         }
1413                         # Don't need to do anything
1414                         return;
1415                 }
1416
1417                 $tables = $this->listTables();
1418
1419                 foreach ( $tables as $table ) {
1420                         if ( $this->db->getType() == 'oracle' ) {
1421                                 $this->db->query( "DROP TABLE pt_$table DROP CONSTRAINTS" );
1422                         } else {
1423                                 $this->db->query( "DROP TABLE `parsertest_$table`" );
1424                         }
1425                 }
1426
1427                 if ( $this->db->getType() == 'oracle' ) {
1428                         $this->db->query( 'BEGIN FILL_WIKI_INFO; END;' );
1429                 }
1430         }
1431
1432         /**
1433          * Upload test files to the backend created by createRepoGroup().
1434          *
1435          * @return callable The teardown callback
1436          */
1437         private function setupUploadBackend() {
1438                 global $IP;
1439
1440                 $repo = RepoGroup::singleton()->getLocalRepo();
1441                 $base = $repo->getZonePath( 'public' );
1442                 $backend = $repo->getBackend();
1443                 $backend->prepare( [ 'dir' => "$base/3/3a" ] );
1444                 $backend->store( [
1445                         'src' => "$IP/tests/phpunit/data/parser/headbg.jpg",
1446                         'dst' => "$base/3/3a/Foobar.jpg"
1447                 ] );
1448                 $backend->prepare( [ 'dir' => "$base/e/ea" ] );
1449                 $backend->store( [
1450                         'src' => "$IP/tests/phpunit/data/parser/wiki.png",
1451                         'dst' => "$base/e/ea/Thumb.png"
1452                 ] );
1453                 $backend->prepare( [ 'dir' => "$base/0/09" ] );
1454                 $backend->store( [
1455                         'src' => "$IP/tests/phpunit/data/parser/headbg.jpg",
1456                         'dst' => "$base/0/09/Bad.jpg"
1457                 ] );
1458                 $backend->prepare( [ 'dir' => "$base/5/5f" ] );
1459                 $backend->store( [
1460                         'src' => "$IP/tests/phpunit/data/parser/LoremIpsum.djvu",
1461                         'dst' => "$base/5/5f/LoremIpsum.djvu"
1462                 ] );
1463
1464                 // No helpful SVG file to copy, so make one ourselves
1465                 $data = '<?xml version="1.0" encoding="utf-8"?>' .
1466                         '<svg xmlns="http://www.w3.org/2000/svg"' .
1467                         ' version="1.1" width="240" height="180"/>';
1468
1469                 $backend->prepare( [ 'dir' => "$base/f/ff" ] );
1470                 $backend->quickCreate( [
1471                         'content' => $data, 'dst' => "$base/f/ff/Foobar.svg"
1472                 ] );
1473
1474                 return function () use ( $backend ) {
1475                         if ( $backend instanceof MockFileBackend ) {
1476                                 // In memory backend, so dont bother cleaning them up.
1477                                 return;
1478                         }
1479                         $this->teardownUploadBackend();
1480                 };
1481         }
1482
1483         /**
1484          * Remove the dummy uploads directory
1485          */
1486         private function teardownUploadBackend() {
1487                 if ( $this->keepUploads ) {
1488                         return;
1489                 }
1490
1491                 $repo = RepoGroup::singleton()->getLocalRepo();
1492                 $public = $repo->getZonePath( 'public' );
1493
1494                 $this->deleteFiles(
1495                         [
1496                                 "$public/3/3a/Foobar.jpg",
1497                                 "$public/e/ea/Thumb.png",
1498                                 "$public/0/09/Bad.jpg",
1499                                 "$public/5/5f/LoremIpsum.djvu",
1500                                 "$public/f/ff/Foobar.svg",
1501                                 "$public/0/00/Video.ogv",
1502                                 "$public/4/41/Audio.oga",
1503                         ]
1504                 );
1505         }
1506
1507         /**
1508          * Delete the specified files and their parent directories
1509          * @param array $files File backend URIs mwstore://...
1510          */
1511         private function deleteFiles( $files ) {
1512                 // Delete the files
1513                 $backend = RepoGroup::singleton()->getLocalRepo()->getBackend();
1514                 foreach ( $files as $file ) {
1515                         $backend->delete( [ 'src' => $file ], [ 'force' => 1 ] );
1516                 }
1517
1518                 // Delete the parent directories
1519                 foreach ( $files as $file ) {
1520                         $tmp = FileBackend::parentStoragePath( $file );
1521                         while ( $tmp ) {
1522                                 if ( !$backend->clean( [ 'dir' => $tmp ] )->isOK() ) {
1523                                         break;
1524                                 }
1525                                 $tmp = FileBackend::parentStoragePath( $tmp );
1526                         }
1527                 }
1528         }
1529
1530         /**
1531          * Add articles to the test DB.
1532          *
1533          * @param array $articles Article info array from TestFileReader
1534          */
1535         public function addArticles( $articles ) {
1536                 global $wgContLang;
1537                 $setup = [];
1538                 $teardown = [];
1539
1540                 // Be sure ParserTestRunner::addArticle has correct language set,
1541                 // so that system messages get into the right language cache
1542                 if ( $wgContLang->getCode() !== 'en' ) {
1543                         $setup['wgLanguageCode'] = 'en';
1544                         $setup['wgContLang'] = Language::factory( 'en' );
1545                 }
1546
1547                 // Add special namespaces, in case that hasn't been done by staticSetup() yet
1548                 $this->appendNamespaceSetup( $setup, $teardown );
1549
1550                 // wgCapitalLinks obviously needs initialisation
1551                 $setup['wgCapitalLinks'] = true;
1552
1553                 $teardown[] = $this->executeSetupSnippets( $setup );
1554
1555                 foreach ( $articles as $info ) {
1556                         $this->addArticle( $info['name'], $info['text'], $info['file'], $info['line'] );
1557                 }
1558
1559                 // Wipe WANObjectCache process cache, which is invalidated by article insertion
1560                 // due to T144706
1561                 ObjectCache::getMainWANInstance()->clearProcessCache();
1562
1563                 $this->executeSetupSnippets( $teardown );
1564         }
1565
1566         /**
1567          * Insert a temporary test article
1568          * @param string $name The title, including any prefix
1569          * @param string $text The article text
1570          * @param string $file The input file name
1571          * @param int|string $line The input line number, for reporting errors
1572          * @throws Exception
1573          * @throws MWException
1574          */
1575         private function addArticle( $name, $text, $file, $line ) {
1576                 $text = self::chomp( $text );
1577                 $name = self::chomp( $name );
1578
1579                 $title = Title::newFromText( $name );
1580                 wfDebug( __METHOD__ . ": adding $name" );
1581
1582                 if ( is_null( $title ) ) {
1583                         throw new MWException( "invalid title '$name' at $file:$line\n" );
1584                 }
1585
1586                 $page = WikiPage::factory( $title );
1587                 $page->loadPageData( 'fromdbmaster' );
1588
1589                 if ( $page->exists() ) {
1590                         throw new MWException( "duplicate article '$name' at $file:$line\n" );
1591                 }
1592
1593                 // Use mock parser, to make debugging of actual parser tests simpler.
1594                 // But initialise the MessageCache clone first, don't let MessageCache
1595                 // get a reference to the mock object.
1596                 MessageCache::singleton()->getParser();
1597                 $restore = $this->executeSetupSnippets( [ 'wgParser' => new ParserTestMockParser ] );
1598                 $status = $page->doEditContent(
1599                         ContentHandler::makeContent( $text, $title ),
1600                         '',
1601                         EDIT_NEW | EDIT_INTERNAL
1602                 );
1603                 $restore();
1604
1605                 if ( !$status->isOK() ) {
1606                         throw new MWException( $status->getWikiText( false, false, 'en' ) );
1607                 }
1608
1609                 // The RepoGroup cache is invalidated by the creation of file redirects
1610                 if ( $title->inNamespace( NS_FILE ) ) {
1611                         RepoGroup::singleton()->clearCache( $title );
1612                 }
1613         }
1614
1615         /**
1616          * Check if a hook is installed
1617          *
1618          * @param string $name
1619          * @return bool True if tag hook is present
1620          */
1621         public function requireHook( $name ) {
1622                 global $wgParser;
1623
1624                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1625                 if ( isset( $wgParser->mTagHooks[$name] ) ) {
1626                         return true;
1627                 } else {
1628                         $this->recorder->warning( "   This test suite requires the '$name' hook " .
1629                                 "extension, skipping." );
1630                         return false;
1631                 }
1632         }
1633
1634         /**
1635          * Check if a function hook is installed
1636          *
1637          * @param string $name
1638          * @return bool True if function hook is present
1639          */
1640         public function requireFunctionHook( $name ) {
1641                 global $wgParser;
1642
1643                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1644
1645                 if ( isset( $wgParser->mFunctionHooks[$name] ) ) {
1646                         return true;
1647                 } else {
1648                         $this->recorder->warning( "   This test suite requires the '$name' function " .
1649                                 "hook extension, skipping." );
1650                         return false;
1651                 }
1652         }
1653
1654         /**
1655          * Check if a transparent tag hook is installed
1656          *
1657          * @param string $name
1658          * @return bool True if function hook is present
1659          */
1660         public function requireTransparentHook( $name ) {
1661                 global $wgParser;
1662
1663                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1664
1665                 if ( isset( $wgParser->mTransparentTagHooks[$name] ) ) {
1666                         return true;
1667                 } else {
1668                         $this->recorder->warning( "   This test suite requires the '$name' transparent " .
1669                                 "hook extension, skipping.\n" );
1670                         return false;
1671                 }
1672         }
1673
1674         /**
1675          * Fake constant timestamp to make sure time-related parser
1676          * functions give a persistent value.
1677          *
1678          * - Parser::getVariableValue (via ParserGetVariableValueTs hook)
1679          * - Parser::preSaveTransform (via ParserOptions)
1680          */
1681         private function getFakeTimestamp() {
1682                 // parsed as '1970-01-01T00:02:03Z'
1683                 return 123;
1684         }
1685 }