tests/parser/ParserTestRunner.php

   1 <?php
   2 /**
   3  * Generic backend for the MediaWiki parser test suite, used by both the
   4  * standalone parserTests.php and the PHPUnit "parsertests" suite.
   5  *
   6  * Copyright © 2004, 2010 Brion Vibber <brion@pobox.com>
   7  * https://www.mediawiki.org/
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write to the Free Software Foundation, Inc.,
  21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  22  * http://www.gnu.org/copyleft/gpl.html
  23  *
  24  * @todo Make this more independent of the configuration (and if possible the database)
  25  * @file
  26  * @ingroup Testing
  27  */
  28 use Wikimedia\Rdbms\IDatabase;
  29 use MediaWiki\MediaWikiServices;
  30 use MediaWiki\Tidy\TidyDriverBase;
  31 use Wikimedia\ScopedCallback;
  32 use Wikimedia\TestingAccessWrapper;
  33
  34 /**
  35  * @ingroup Testing
  36  */
  37 class ParserTestRunner {
  38
  39         /**
  40          * MediaWiki core parser test files, paths
  41          * will be prefixed with __DIR__ . '/'
  42          *
  43          * @var array
  44          */
  45         private static $coreTestFiles = [
  46                 'parserTests.txt',
  47                 'extraParserTests.txt',
  48         ];
  49
  50         /**
  51          * @var bool $useTemporaryTables Use temporary tables for the temporary database
  52          */
  53         private $useTemporaryTables = true;
  54
  55         /**
  56          * @var array $setupDone The status of each setup function
  57          */
  58         private $setupDone = [
  59                 'staticSetup' => false,
  60                 'perTestSetup' => false,
  61                 'setupDatabase' => false,
  62                 'setDatabase' => false,
  63                 'setupUploads' => false,
  64         ];
  65
  66         /**
  67          * Our connection to the database
  68          * @var Database
  69          */
  70         private $db;
  71
  72         /**
  73          * Database clone helper
  74          * @var CloneDatabase
  75          */
  76         private $dbClone;
  77
  78         /**
  79          * @var TidyDriverBase
  80          */
  81         private $tidyDriver = null;
  82
  83         /**
  84          * @var TestRecorder
  85          */
  86         private $recorder;
  87
  88         /**
  89          * The upload directory, or null to not set up an upload directory
  90          *
  91          * @var string|null
  92          */
  93         private $uploadDir = null;
  94
  95         /**
  96          * The name of the file backend to use, or null to use MockFileBackend.
  97          * @var string|null
  98          */
  99         private $fileBackendName;
 100
 101         /**
 102          * A complete regex for filtering tests.
 103          * @var string
 104          */
 105         private $regex;
 106
 107         /**
 108          * A list of normalization functions to apply to the expected and actual
 109          * output.
 110          * @var array
 111          */
 112         private $normalizationFunctions = [];
 113
 114         /**
 115          * Run disabled parser tests
 116          * @var bool
 117          */
 118         private $runDisabled;
 119
 120         /**
 121          * Disable parse on article insertion
 122          * @var bool
 123          */
 124         private $disableSaveParse;
 125
 126         /**
 127          * Reuse upload directory
 128          * @var bool
 129          */
 130         private $keepUploads;
 131
 132         /**
 133          * @param TestRecorder $recorder
 134          * @param array $options
 135          */
 136         public function __construct( TestRecorder $recorder, $options = [] ) {
 137                 $this->recorder = $recorder;
 138
 139                 if ( isset( $options['norm'] ) ) {
 140                         foreach ( $options['norm'] as $func ) {
 141                                 if ( in_array( $func, [ 'removeTbody', 'trimWhitespace' ] ) ) {
 142                                         $this->normalizationFunctions[] = $func;
 143                                 } else {
 144                                         $this->recorder->warning(
 145                                                 "Warning: unknown normalization option \"$func\"\n" );
 146                                 }
 147                         }
 148                 }
 149
 150                 if ( isset( $options['regex'] ) && $options['regex'] !== false ) {
 151                         $this->regex = $options['regex'];
 152                 } else {
 153                         # Matches anything
 154                         $this->regex = '//';
 155                 }
 156
 157                 $this->keepUploads = !empty( $options['keep-uploads'] );
 158
 159                 $this->fileBackendName = $options['file-backend'] ?? false;
 160
 161                 $this->runDisabled = !empty( $options['run-disabled'] );
 162
 163                 $this->disableSaveParse = !empty( $options['disable-save-parse'] );
 164
 165                 if ( isset( $options['upload-dir'] ) ) {
 166                         $this->uploadDir = $options['upload-dir'];
 167                 }
 168         }
 169
 170         /**
 171          * Get list of filenames to extension and core parser tests
 172          *
 173          * @return array
 174          */
 175         public static function getParserTestFiles() {
 176                 global $wgParserTestFiles;
 177
 178                 // Add core test files
 179                 $files = array_map( function ( $item ) {
 180                         return __DIR__ . "/$item";
 181                 }, self::$coreTestFiles );
 182
 183                 // Plus legacy global files
 184                 $files = array_merge( $files, $wgParserTestFiles );
 185
 186                 // Auto-discover extension parser tests
 187                 $registry = ExtensionRegistry::getInstance();
 188                 foreach ( $registry->getAllThings() as $info ) {
 189                         $dir = dirname( $info['path'] ) . '/tests/parser';
 190                         if ( !file_exists( $dir ) ) {
 191                                 continue;
 192                         }
 193                         $counter = 1;
 194                         $dirIterator = new RecursiveIteratorIterator(
 195                                 new RecursiveDirectoryIterator( $dir )
 196                         );
 197                         foreach ( $dirIterator as $fileInfo ) {
 198                                 /** @var SplFileInfo $fileInfo */
 199                                 if ( substr( $fileInfo->getFilename(), -4 ) === '.txt' ) {
 200                                         $name = $info['name'] . $counter;
 201                                         while ( isset( $files[$name] ) ) {
 202                                                 $name = $info['name'] . '_' . $counter++;
 203                                         }
 204                                         $files[$name] = $fileInfo->getPathname();
 205                                 }
 206                         }
 207                 }
 208
 209                 return array_unique( $files );
 210         }
 211
 212         public function getRecorder() {
 213                 return $this->recorder;
 214         }
 215
 216         /**
 217          * Do any setup which can be done once for all tests, independent of test
 218          * options, except for database setup.
 219          *
 220          * Public setup functions in this class return a ScopedCallback object. When
 221          * this object is destroyed by going out of scope, teardown of the
 222          * corresponding test setup is performed.
 223          *
 224          * Teardown objects may be chained by passing a ScopedCallback from a
 225          * previous setup stage as the $nextTeardown parameter. This enforces the
 226          * convention that teardown actions are taken in reverse order to the
 227          * corresponding setup actions. When $nextTeardown is specified, a
 228          * ScopedCallback will be returned which first tears down the current
 229          * setup stage, and then tears down the previous setup stage which was
 230          * specified by $nextTeardown.
 231          *
 232          * @param ScopedCallback|null $nextTeardown
 233          * @return ScopedCallback
 234          */
 235         public function staticSetup( $nextTeardown = null ) {
 236                 // A note on coding style:
 237
 238                 // The general idea here is to keep setup code together with
 239                 // corresponding teardown code, in a fine-grained manner. We have two
 240                 // arrays: $setup and $teardown. The code snippets in the $setup array
 241                 // are executed at the end of the method, before it returns, and the
 242                 // code snippets in the $teardown array are executed in reverse order
 243                 // when the Wikimedia\ScopedCallback object is consumed.
 244
 245                 // Because it is a common operation to save, set and restore global
 246                 // variables, we have an additional convention: when the array key of
 247                 // $setup is a string, the string is taken to be the name of the global
 248                 // variable, and the element value is taken to be the desired new value.
 249
 250                 // It's acceptable to just do the setup immediately, instead of adding
 251                 // a closure to $setup, except when the setup action depends on global
 252                 // variable initialisation being done first. In this case, you have to
 253                 // append a closure to $setup after the global variable is appended.
 254
 255                 // When you add to setup functions in this class, please keep associated
 256                 // setup and teardown actions together in the source code, and please
 257                 // add comments explaining why the setup action is necessary.
 258
 259                 $setup = [];
 260                 $teardown = [];
 261
 262                 $teardown[] = $this->markSetupDone( 'staticSetup' );
 263
 264                 // Some settings which influence HTML output
 265                 $setup['wgSitename'] = 'MediaWiki';
 266                 $setup['wgServer'] = 'http://example.org';
 267                 $setup['wgServerName'] = 'example.org';
 268                 $setup['wgScriptPath'] = '';
 269                 $setup['wgScript'] = '/index.php';
 270                 $setup['wgResourceBasePath'] = '';
 271                 $setup['wgStylePath'] = '/skins';
 272                 $setup['wgExtensionAssetsPath'] = '/extensions';
 273                 $setup['wgArticlePath'] = '/wiki/$1';
 274                 $setup['wgActionPaths'] = [];
 275                 $setup['wgVariantArticlePath'] = false;
 276                 $setup['wgUploadNavigationUrl'] = false;
 277                 $setup['wgCapitalLinks'] = true;
 278                 $setup['wgNoFollowLinks'] = true;
 279                 $setup['wgNoFollowDomainExceptions'] = [ 'no-nofollow.org' ];
 280                 $setup['wgExternalLinkTarget'] = false;
 281                 $setup['wgLocaltimezone'] = 'UTC';
 282                 $setup['wgHtml5'] = true;
 283                 $setup['wgDisableLangConversion'] = false;
 284                 $setup['wgDisableTitleConversion'] = false;
 285
 286                 // "extra language links"
 287                 // see https://gerrit.wikimedia.org/r/111390
 288                 $setup['wgExtraInterlanguageLinkPrefixes'] = [ 'mul' ];
 289
 290                 // All FileRepo changes should be done here by injecting services,
 291                 // there should be no need to change global variables.
 292                 RepoGroup::setSingleton( $this->createRepoGroup() );
 293                 $teardown[] = function () {
 294                         RepoGroup::destroySingleton();
 295                 };
 296
 297                 // Set up null lock managers
 298                 $setup['wgLockManagers'] = [ [
 299                         'name' => 'fsLockManager',
 300                         'class' => NullLockManager::class,
 301                 ], [
 302                         'name' => 'nullLockManager',
 303                         'class' => NullLockManager::class,
 304                 ] ];
 305                 $reset = function () {
 306                         LockManagerGroup::destroySingletons();
 307                 };
 308                 $setup[] = $reset;
 309                 $teardown[] = $reset;
 310
 311                 // This allows article insertion into the prefixed DB
 312                 $setup['wgDefaultExternalStore'] = false;
 313
 314                 // This might slightly reduce memory usage
 315                 $setup['wgAdaptiveMessageCache'] = true;
 316
 317                 // This is essential and overrides disabling of database messages in TestSetup
 318                 $setup['wgUseDatabaseMessages'] = true;
 319                 $reset = function () {
 320                         MessageCache::destroyInstance();
 321                 };
 322                 $setup[] = $reset;
 323                 $teardown[] = $reset;
 324
 325                 // It's not necessary to actually convert any files
 326                 $setup['wgSVGConverter'] = 'null';
 327                 $setup['wgSVGConverters'] = [ 'null' => 'echo "1">$output' ];
 328
 329                 // Fake constant timestamp
 330                 Hooks::register( 'ParserGetVariableValueTs', function ( &$parser, &$ts ) {
 331                         $ts = $this->getFakeTimestamp();
 332                         return true;
 333                 } );
 334                 $teardown[] = function () {
 335                         Hooks::clear( 'ParserGetVariableValueTs' );
 336                 };
 337
 338                 $this->appendNamespaceSetup( $setup, $teardown );
 339
 340                 // Set up interwikis and append teardown function
 341                 $teardown[] = $this->setupInterwikis();
 342
 343                 // This affects title normalization in links. It invalidates
 344                 // MediaWikiTitleCodec objects.
 345                 $setup['wgLocalInterwikis'] = [ 'local', 'mi' ];
 346                 $reset = function () {
 347                         $this->resetTitleServices();
 348                 };
 349                 $setup[] = $reset;
 350                 $teardown[] = $reset;
 351
 352                 // Set up a mock MediaHandlerFactory
 353                 MediaWikiServices::getInstance()->disableService( 'MediaHandlerFactory' );
 354                 MediaWikiServices::getInstance()->redefineService(
 355                         'MediaHandlerFactory',
 356                         function ( MediaWikiServices $services ) {
 357                                 $handlers = $services->getMainConfig()->get( 'ParserTestMediaHandlers' );
 358                                 return new MediaHandlerFactory( $handlers );
 359                         }
 360                 );
 361                 $teardown[] = function () {
 362                         MediaWikiServices::getInstance()->resetServiceForTesting( 'MediaHandlerFactory' );
 363                 };
 364
 365                 // SqlBagOStuff broke when using temporary tables on r40209 (T17892).
 366                 // It seems to have been fixed since (r55079?), but regressed at some point before r85701.
 367                 // This works around it for now...
 368                 global $wgObjectCaches;
 369                 $setup['wgObjectCaches'] = [ CACHE_DB => $wgObjectCaches['hash'] ] + $wgObjectCaches;
 370                 if ( isset( ObjectCache::$instances[CACHE_DB] ) ) {
 371                         $savedCache = ObjectCache::$instances[CACHE_DB];
 372                         ObjectCache::$instances[CACHE_DB] = new HashBagOStuff;
 373                         $teardown[] = function () use ( $savedCache ) {
 374                                 ObjectCache::$instances[CACHE_DB] = $savedCache;
 375                         };
 376                 }
 377
 378                 $teardown[] = $this->executeSetupSnippets( $setup );
 379
 380                 // Schedule teardown snippets in reverse order
 381                 return $this->createTeardownObject( $teardown, $nextTeardown );
 382         }
 383
 384         private function appendNamespaceSetup( &$setup, &$teardown ) {
 385                 // Add a namespace shadowing a interwiki link, to test
 386                 // proper precedence when resolving links. (T53680)
 387                 $setup['wgExtraNamespaces'] = [
 388                         100 => 'MemoryAlpha',
 389                         101 => 'MemoryAlpha_talk'
 390                 ];
 391                 // Changing wgExtraNamespaces invalidates caches in MWNamespace and
 392                 // any live Language object, both on setup and teardown
 393                 $reset = function () {
 394                         MWNamespace::clearCaches();
 395                         MediaWikiServices::getInstance()->getContentLanguage()->resetNamespaces();
 396                 };
 397                 $setup[] = $reset;
 398                 $teardown[] = $reset;
 399         }
 400
 401         /**
 402          * Create a RepoGroup object appropriate for the current configuration
 403          * @return RepoGroup
 404          */
 405         protected function createRepoGroup() {
 406                 if ( $this->uploadDir ) {
 407                         if ( $this->fileBackendName ) {
 408                                 throw new MWException( 'You cannot specify both use-filebackend and upload-dir' );
 409                         }
 410                         $backend = new FSFileBackend( [
 411                                 'name' => 'local-backend',
 412                                 'wikiId' => wfWikiID(),
 413                                 'basePath' => $this->uploadDir,
 414                                 'tmpDirectory' => wfTempDir()
 415                         ] );
 416                 } elseif ( $this->fileBackendName ) {
 417                         global $wgFileBackends;
 418                         $name = $this->fileBackendName;
 419                         $useConfig = false;
 420                         foreach ( $wgFileBackends as $conf ) {
 421                                 if ( $conf['name'] === $name ) {
 422                                         $useConfig = $conf;
 423                                 }
 424                         }
 425                         if ( $useConfig === false ) {
 426                                 throw new MWException( "Unable to find file backend \"$name\"" );
 427                         }
 428                         $useConfig['name'] = 'local-backend'; // swap name
 429                         unset( $useConfig['lockManager'] );
 430                         unset( $useConfig['fileJournal'] );
 431                         $class = $useConfig['class'];
 432                         $backend = new $class( $useConfig );
 433                 } else {
 434                         # Replace with a mock. We do not care about generating real
 435                         # files on the filesystem, just need to expose the file
 436                         # informations.
 437                         $backend = new MockFileBackend( [
 438                                 'name' => 'local-backend',
 439                                 'wikiId' => wfWikiID()
 440                         ] );
 441                 }
 442
 443                 return new RepoGroup(
 444                         [
 445                                 'class' => MockLocalRepo::class,
 446                                 'name' => 'local',
 447                                 'url' => 'http://example.com/images',
 448                                 'hashLevels' => 2,
 449                                 'transformVia404' => false,
 450                                 'backend' => $backend
 451                         ],
 452                         []
 453                 );
 454         }
 455
 456         /**
 457          * Execute an array in which elements with integer keys are taken to be
 458          * callable objects, and other elements are taken to be global variable
 459          * set operations, with the key giving the variable name and the value
 460          * giving the new global variable value. A closure is returned which, when
 461          * executed, sets the global variables back to the values they had before
 462          * this function was called.
 463          *
 464          * @see staticSetup
 465          *
 466          * @param array $setup
 467          * @return closure
 468          */
 469         protected function executeSetupSnippets( $setup ) {
 470                 $saved = [];
 471                 foreach ( $setup as $name => $value ) {
 472                         if ( is_int( $name ) ) {
 473                                 $value();
 474                         } else {
 475                                 $saved[$name] = $GLOBALS[$name] ?? null;
 476                                 $GLOBALS[$name] = $value;
 477                         }
 478                 }
 479                 return function () use ( $saved ) {
 480                         $this->executeSetupSnippets( $saved );
 481                 };
 482         }
 483
 484         /**
 485          * Take a setup array in the same format as the one given to
 486          * executeSetupSnippets(), and return a ScopedCallback which, when consumed,
 487          * executes the snippets in the setup array in reverse order. This is used
 488          * to create "teardown objects" for the public API.
 489          *
 490          * @see staticSetup
 491          *
 492          * @param array $teardown The snippet array
 493          * @param ScopedCallback|null $nextTeardown A ScopedCallback to consume
 494          * @return ScopedCallback
 495          */
 496         protected function createTeardownObject( $teardown, $nextTeardown = null ) {
 497                 return new ScopedCallback( function () use ( $teardown, $nextTeardown ) {
 498                         // Schedule teardown snippets in reverse order
 499                         $teardown = array_reverse( $teardown );
 500
 501                         $this->executeSetupSnippets( $teardown );
 502                         if ( $nextTeardown ) {
 503                                 ScopedCallback::consume( $nextTeardown );
 504                         }
 505                 } );
 506         }
 507
 508         /**
 509          * Set a setupDone flag to indicate that setup has been done, and return
 510          * the teardown closure. If the flag was already set, throw an exception.
 511          *
 512          * @param string $funcName The setup function name
 513          * @return closure
 514          */
 515         protected function markSetupDone( $funcName ) {
 516                 if ( $this->setupDone[$funcName] ) {
 517                         throw new MWException( "$funcName is already done" );
 518                 }
 519                 $this->setupDone[$funcName] = true;
 520                 return function () use ( $funcName ) {
 521                         $this->setupDone[$funcName] = false;
 522                 };
 523         }
 524
 525         /**
 526          * Ensure a given setup stage has been done, throw an exception if it has
 527          * not.
 528          * @param string $funcName
 529          * @param string|null $funcName2
 530          */
 531         protected function checkSetupDone( $funcName, $funcName2 = null ) {
 532                 if ( !$this->setupDone[$funcName]
 533                         && ( $funcName === null || !$this->setupDone[$funcName2] )
 534                 ) {
 535                         throw new MWException( "$funcName must be called before calling " .
 536                                 wfGetCaller() );
 537                 }
 538         }
 539
 540         /**
 541          * Determine whether a particular setup function has been run
 542          *
 543          * @param string $funcName
 544          * @return bool
 545          */
 546         public function isSetupDone( $funcName ) {
 547                 return $this->setupDone[$funcName] ?? false;
 548         }
 549
 550         /**
 551          * Insert hardcoded interwiki in the lookup table.
 552          *
 553          * This function insert a set of well known interwikis that are used in
 554          * the parser tests. They can be considered has fixtures are injected in
 555          * the interwiki cache by using the 'InterwikiLoadPrefix' hook.
 556          * Since we are not interested in looking up interwikis in the database,
 557          * the hook completely replace the existing mechanism (hook returns false).
 558          *
 559          * @return closure for teardown
 560          */
 561         private function setupInterwikis() {
 562                 # Hack: insert a few Wikipedia in-project interwiki prefixes,
 563                 # for testing inter-language links
 564                 Hooks::register( 'InterwikiLoadPrefix', function ( $prefix, &$iwData ) {
 565                         static $testInterwikis = [
 566                                 'local' => [
 567                                         'iw_url' => 'http://doesnt.matter.org/$1',
 568                                         'iw_api' => '',
 569                                         'iw_wikiid' => '',
 570                                         'iw_local' => 0 ],
 571                                 'wikipedia' => [
 572                                         'iw_url' => 'http://en.wikipedia.org/wiki/$1',
 573                                         'iw_api' => '',
 574                                         'iw_wikiid' => '',
 575                                         'iw_local' => 0 ],
 576                                 'meatball' => [
 577                                         'iw_url' => 'http://www.usemod.com/cgi-bin/mb.pl?$1',
 578                                         'iw_api' => '',
 579                                         'iw_wikiid' => '',
 580                                         'iw_local' => 0 ],
 581                                 'memoryalpha' => [
 582                                         'iw_url' => 'http://www.memory-alpha.org/en/index.php/$1',
 583                                         'iw_api' => '',
 584                                         'iw_wikiid' => '',
 585                                         'iw_local' => 0 ],
 586                                 'zh' => [
 587                                         'iw_url' => 'http://zh.wikipedia.org/wiki/$1',
 588                                         'iw_api' => '',
 589                                         'iw_wikiid' => '',
 590                                         'iw_local' => 1 ],
 591                                 'es' => [
 592                                         'iw_url' => 'http://es.wikipedia.org/wiki/$1',
 593                                         'iw_api' => '',
 594                                         'iw_wikiid' => '',
 595                                         'iw_local' => 1 ],
 596                                 'fr' => [
 597                                         'iw_url' => 'http://fr.wikipedia.org/wiki/$1',
 598                                         'iw_api' => '',
 599                                         'iw_wikiid' => '',
 600                                         'iw_local' => 1 ],
 601                                 'ru' => [
 602                                         'iw_url' => 'http://ru.wikipedia.org/wiki/$1',
 603                                         'iw_api' => '',
 604                                         'iw_wikiid' => '',
 605                                         'iw_local' => 1 ],
 606                                 'mi' => [
 607                                         'iw_url' => 'http://mi.wikipedia.org/wiki/$1',
 608                                         'iw_api' => '',
 609                                         'iw_wikiid' => '',
 610                                         'iw_local' => 1 ],
 611                                 'mul' => [
 612                                         'iw_url' => 'http://wikisource.org/wiki/$1',
 613                                         'iw_api' => '',
 614                                         'iw_wikiid' => '',
 615                                         'iw_local' => 1 ],
 616                         ];
 617                         if ( array_key_exists( $prefix, $testInterwikis ) ) {
 618                                 $iwData = $testInterwikis[$prefix];
 619                         }
 620
 621                         // We only want to rely on the above fixtures
 622                         return false;
 623                 } );// hooks::register
 624
 625                 // Reset the service in case any other tests already cached some prefixes.
 626                 MediaWikiServices::getInstance()->resetServiceForTesting( 'InterwikiLookup' );
 627
 628                 return function () {
 629                         // Tear down
 630                         Hooks::clear( 'InterwikiLoadPrefix' );
 631                         MediaWikiServices::getInstance()->resetServiceForTesting( 'InterwikiLookup' );
 632                 };
 633         }
 634
 635         /**
 636          * Reset the Title-related services that need resetting
 637          * for each test
 638          */
 639         private function resetTitleServices() {
 640                 $services = MediaWikiServices::getInstance();
 641                 $services->resetServiceForTesting( 'TitleFormatter' );
 642                 $services->resetServiceForTesting( 'TitleParser' );
 643                 $services->resetServiceForTesting( '_MediaWikiTitleCodec' );
 644                 $services->resetServiceForTesting( 'LinkRenderer' );
 645                 $services->resetServiceForTesting( 'LinkRendererFactory' );
 646         }
 647
 648         /**
 649          * Remove last character if it is a newline
 650          * @param string $s
 651          * @return string
 652          */
 653         public static function chomp( $s ) {
 654                 if ( substr( $s, -1 ) === "\n" ) {
 655                         return substr( $s, 0, -1 );
 656                 } else {
 657                         return $s;
 658                 }
 659         }
 660
 661         /**
 662          * Run a series of tests listed in the given text files.
 663          * Each test consists of a brief description, wikitext input,
 664          * and the expected HTML output.
 665          *
 666          * Prints status updates on stdout and counts up the total
 667          * number and percentage of passed tests.
 668          *
 669          * Handles all setup and teardown.
 670          *
 671          * @param array $filenames Array of strings
 672          * @return bool True if passed all tests, false if any tests failed.
 673          */
 674         public function runTestsFromFiles( $filenames ) {
 675                 $ok = false;
 676
 677                 $teardownGuard = $this->staticSetup();
 678                 $teardownGuard = $this->setupDatabase( $teardownGuard );
 679                 $teardownGuard = $this->setupUploads( $teardownGuard );
 680
 681                 $this->recorder->start();
 682                 try {
 683                         $ok = true;
 684
 685                         foreach ( $filenames as $filename ) {
 686                                 $testFileInfo = TestFileReader::read( $filename, [
 687                                         'runDisabled' => $this->runDisabled,
 688                                         'regex' => $this->regex ] );
 689
 690                                 // Don't start the suite if there are no enabled tests in the file
 691                                 if ( !$testFileInfo['tests'] ) {
 692                                         continue;
 693                                 }
 694
 695                                 $this->recorder->startSuite( $filename );
 696                                 $ok = $this->runTests( $testFileInfo ) && $ok;
 697                                 $this->recorder->endSuite( $filename );
 698                         }
 699
 700                         $this->recorder->report();
 701                 } catch ( DBError $e ) {
 702                         $this->recorder->warning( $e->getMessage() );
 703                 }
 704                 $this->recorder->end();
 705
 706                 ScopedCallback::consume( $teardownGuard );
 707
 708                 return $ok;
 709         }
 710
 711         /**
 712          * Determine whether the current parser has the hooks registered in it
 713          * that are required by a file read by TestFileReader.
 714          * @param array $requirements
 715          * @return bool
 716          */
 717         public function meetsRequirements( $requirements ) {
 718                 foreach ( $requirements as $requirement ) {
 719                         switch ( $requirement['type'] ) {
 720                                 case 'hook':
 721                                         $ok = $this->requireHook( $requirement['name'] );
 722                                         break;
 723                                 case 'functionHook':
 724                                         $ok = $this->requireFunctionHook( $requirement['name'] );
 725                                         break;
 726                                 case 'transparentHook':
 727                                         $ok = $this->requireTransparentHook( $requirement['name'] );
 728                                         break;
 729                         }
 730                         if ( !$ok ) {
 731                                 return false;
 732                         }
 733                 }
 734                 return true;
 735         }
 736
 737         /**
 738          * Run the tests from a single file. staticSetup() and setupDatabase()
 739          * must have been called already.
 740          *
 741          * @param array $testFileInfo Parsed file info returned by TestFileReader
 742          * @return bool True if passed all tests, false if any tests failed.
 743          */
 744         public function runTests( $testFileInfo ) {
 745                 $ok = true;
 746
 747                 $this->checkSetupDone( 'staticSetup' );
 748
 749                 // Don't add articles from the file if there are no enabled tests from the file
 750                 if ( !$testFileInfo['tests'] ) {
 751                         return true;
 752                 }
 753
 754                 // If any requirements are not met, mark all tests from the file as skipped
 755                 if ( !$this->meetsRequirements( $testFileInfo['requirements'] ) ) {
 756                         foreach ( $testFileInfo['tests'] as $test ) {
 757                                 $this->recorder->startTest( $test );
 758                                 $this->recorder->skipped( $test, 'required extension not enabled' );
 759                         }
 760                         return true;
 761                 }
 762
 763                 // Add articles
 764                 $this->addArticles( $testFileInfo['articles'] );
 765
 766                 // Run tests
 767                 foreach ( $testFileInfo['tests'] as $test ) {
 768                         $this->recorder->startTest( $test );
 769                         $result =
 770                                 $this->runTest( $test );
 771                         if ( $result !== false ) {
 772                                 $ok = $ok && $result->isSuccess();
 773                                 $this->recorder->record( $test, $result );
 774                         }
 775                 }
 776
 777                 return $ok;
 778         }
 779
 780         /**
 781          * Get a Parser object
 782          *
 783          * @param string|null $preprocessor
 784          * @return Parser
 785          */
 786         function getParser( $preprocessor = null ) {
 787                 global $wgParserConf;
 788
 789                 $class = $wgParserConf['class'];
 790                 $parser = new $class( [ 'preprocessorClass' => $preprocessor ] + $wgParserConf );
 791                 ParserTestParserHook::setup( $parser );
 792
 793                 return $parser;
 794         }
 795
 796         /**
 797          * Run a given wikitext input through a freshly-constructed wiki parser,
 798          * and compare the output against the expected results.
 799          * Prints status and explanatory messages to stdout.
 800          *
 801          * staticSetup() and setupWikiData() must be called before this function
 802          * is entered.
 803          *
 804          * @param array $test The test parameters:
 805          *  - test: The test name
 806          *  - desc: The subtest description
 807          *  - input: Wikitext to try rendering
 808          *  - options: Array of test options
 809          *  - config: Overrides for global variables, one per line
 810          *
 811          * @return ParserTestResult|false false if skipped
 812          */
 813         public function runTest( $test ) {
 814                 wfDebug( __METHOD__ . ": running {$test['desc']}" );
 815                 $opts = $this->parseOptions( $test['options'] );
 816                 $teardownGuard = $this->perTestSetup( $test );
 817
 818                 $context = RequestContext::getMain();
 819                 $user = $context->getUser();
 820                 $options = ParserOptions::newFromContext( $context );
 821                 $options->setTimestamp( $this->getFakeTimestamp() );
 822
 823                 if ( isset( $opts['tidy'] ) ) {
 824                         $options->setTidy( true );
 825                 }
 826
 827                 if ( isset( $opts['title'] ) ) {
 828                         $titleText = $opts['title'];
 829                 } else {
 830                         $titleText = 'Parser test';
 831                 }
 832
 833                 if ( isset( $opts['maxincludesize'] ) ) {
 834                         $options->setMaxIncludeSize( $opts['maxincludesize'] );
 835                 }
 836                 if ( isset( $opts['maxtemplatedepth'] ) ) {
 837                         $options->setMaxTemplateDepth( $opts['maxtemplatedepth'] );
 838                 }
 839
 840                 $local = isset( $opts['local'] );
 841                 $preprocessor = $opts['preprocessor'] ?? null;
 842                 $parser = $this->getParser( $preprocessor );
 843                 $title = Title::newFromText( $titleText );
 844
 845                 if ( isset( $opts['styletag'] ) ) {
 846                         // For testing the behavior of <style> (including those deduplicated
 847                         // into <link> tags), add tag hooks to allow them to be generated.
 848                         $parser->setHook( 'style', function ( $content, $attributes, $parser ) {
 849                                 $marker = Parser::MARKER_PREFIX . '-style-' . md5( $content ) . Parser::MARKER_SUFFIX;
 850                                 $parser->mStripState->addNoWiki( $marker, $content );
 851                                 return Html::inlineStyle( $marker, 'all', $attributes );
 852                         } );
 853                         $parser->setHook( 'link', function ( $content, $attributes, $parser ) {
 854                                 return Html::element( 'link', $attributes );
 855                         } );
 856                 }
 857
 858                 if ( isset( $opts['pst'] ) ) {
 859                         $out = $parser->preSaveTransform( $test['input'], $title, $user, $options );
 860                         $output = $parser->getOutput();
 861                 } elseif ( isset( $opts['msg'] ) ) {
 862                         $out = $parser->transformMsg( $test['input'], $options, $title );
 863                 } elseif ( isset( $opts['section'] ) ) {
 864                         $section = $opts['section'];
 865                         $out = $parser->getSection( $test['input'], $section );
 866                 } elseif ( isset( $opts['replace'] ) ) {
 867                         $section = $opts['replace'][0];
 868                         $replace = $opts['replace'][1];
 869                         $out = $parser->replaceSection( $test['input'], $section, $replace );
 870                 } elseif ( isset( $opts['comment'] ) ) {
 871                         $out = Linker::formatComment( $test['input'], $title, $local );
 872                 } elseif ( isset( $opts['preload'] ) ) {
 873                         $out = $parser->getPreloadText( $test['input'], $title, $options );
 874                 } else {
 875                         $output = $parser->parse( $test['input'], $title, $options, true, true, 1337 );
 876                         $out = $output->getText( [
 877                                 'allowTOC' => !isset( $opts['notoc'] ),
 878                                 'unwrap' => !isset( $opts['wrap'] ),
 879                         ] );
 880                         if ( isset( $opts['tidy'] ) ) {
 881                                 $out = preg_replace( '/\s+$/', '', $out );
 882                         }
 883
 884                         if ( isset( $opts['showtitle'] ) ) {
 885                                 if ( $output->getTitleText() ) {
 886                                         $title = $output->getTitleText();
 887                                 }
 888
 889                                 $out = "$title\n$out";
 890                         }
 891
 892                         if ( isset( $opts['showindicators'] ) ) {
 893                                 $indicators = '';
 894                                 foreach ( $output->getIndicators() as $id => $content ) {
 895                                         $indicators .= "$id=$content\n";
 896                                 }
 897                                 $out = $indicators . $out;
 898                         }
 899
 900                         if ( isset( $opts['ill'] ) ) {
 901                                 $out = implode( ' ', $output->getLanguageLinks() );
 902                         } elseif ( isset( $opts['cat'] ) ) {
 903                                 $out = '';
 904                                 foreach ( $output->getCategories() as $name => $sortkey ) {
 905                                         if ( $out !== '' ) {
 906                                                 $out .= "\n";
 907                                         }
 908                                         $out .= "cat=$name sort=$sortkey";
 909                                 }
 910                         }
 911                 }
 912
 913                 if ( isset( $output ) && isset( $opts['showflags'] ) ) {
 914                         $actualFlags = array_keys( TestingAccessWrapper::newFromObject( $output )->mFlags );
 915                         sort( $actualFlags );
 916                         $out .= "\nflags=" . implode( ', ', $actualFlags );
 917                 }
 918
 919                 ScopedCallback::consume( $teardownGuard );
 920
 921                 $expected = $test['result'];
 922                 if ( count( $this->normalizationFunctions ) ) {
 923                         $expected = ParserTestResultNormalizer::normalize(
 924                                 $test['expected'], $this->normalizationFunctions );
 925                         $out = ParserTestResultNormalizer::normalize( $out, $this->normalizationFunctions );
 926                 }
 927
 928                 $testResult = new ParserTestResult( $test, $expected, $out );
 929                 return $testResult;
 930         }
 931
 932         /**
 933          * Use a regex to find out the value of an option
 934          * @param string $key Name of option val to retrieve
 935          * @param array $opts Options array to look in
 936          * @param mixed $default Default value returned if not found
 937          * @return mixed
 938          */
 939         private static function getOptionValue( $key, $opts, $default ) {
 940                 $key = strtolower( $key );
 941
 942                 if ( isset( $opts[$key] ) ) {
 943                         return $opts[$key];
 944                 } else {
 945                         return $default;
 946                 }
 947         }
 948
 949         /**
 950          * Given the options string, return an associative array of options.
 951          * @todo Move this to TestFileReader
 952          *
 953          * @param string $instring
 954          * @return array
 955          */
 956         private function parseOptions( $instring ) {
 957                 $opts = [];
 958                 // foo
 959                 // foo=bar
 960                 // foo="bar baz"
 961                 // foo=[[bar baz]]
 962                 // foo=bar,"baz quux"
 963                 // foo={...json...}
 964                 $defs = '(?(DEFINE)
 965                         (?<qstr>                                        # Quoted string
 966                                 "
 967                                 (?:[^\\\\"] | \\\\.)*
 968                                 "
 969                         )
 970                         (?<json>
 971                                 \{              # Open bracket
 972                                 (?:
 973                                         [^"{}] |                                # Not a quoted string or object, or
 974                                         (?&qstr) |                              # A quoted string, or
 975                                         (?&json)                                # A json object (recursively)
 976                                 )*
 977                                 \}              # Close bracket
 978                         )
 979                         (?<value>
 980                                 (?:
 981                                         (?&qstr)                        # Quoted val
 982                                 |
 983                                         \[\[
 984                                                 [^]]*                   # Link target
 985                                         \]\]
 986                                 |
 987                                         [\w-]+                          # Plain word
 988                                 |
 989                                         (?&json)                        # JSON object
 990                                 )
 991                         )
 992                 )';
 993                 $regex = '/' . $defs . '\b
 994                         (?<k>[\w-]+)                            # Key
 995                         \b
 996                         (?:\s*
 997                                 =                                               # First sub-value
 998                                 \s*
 999                                 (?<v>
1000                                         (?&value)
1001                                         (?:\s*
1002                                                 ,                               # Sub-vals 1..N
1003                                                 \s*
1004                                                 (?&value)
1005                                         )*
1006                                 )
1007                         )?
1008                         /x';
1009                 $valueregex = '/' . $defs . '(?&value)/x';
1010
1011                 if ( preg_match_all( $regex, $instring, $matches, PREG_SET_ORDER ) ) {
1012                         foreach ( $matches as $bits ) {
1013                                 $key = strtolower( $bits['k'] );
1014                                 if ( !isset( $bits['v'] ) ) {
1015                                         $opts[$key] = true;
1016                                 } else {
1017                                         preg_match_all( $valueregex, $bits['v'], $vmatches );
1018                                         $opts[$key] = array_map( [ $this, 'cleanupOption' ], $vmatches[0] );
1019                                         if ( count( $opts[$key] ) == 1 ) {
1020                                                 $opts[$key] = $opts[$key][0];
1021                                         }
1022                                 }
1023                         }
1024                 }
1025                 return $opts;
1026         }
1027
1028         private function cleanupOption( $opt ) {
1029                 if ( substr( $opt, 0, 1 ) == '"' ) {
1030                         return stripcslashes( substr( $opt, 1, -1 ) );
1031                 }
1032
1033                 if ( substr( $opt, 0, 2 ) == '[[' ) {
1034                         return substr( $opt, 2, -2 );
1035                 }
1036
1037                 if ( substr( $opt, 0, 1 ) == '{' ) {
1038                         return FormatJson::decode( $opt, true );
1039                 }
1040                 return $opt;
1041         }
1042
1043         /**
1044          * Do any required setup which is dependent on test options.
1045          *
1046          * @see staticSetup() for more information about setup/teardown
1047          *
1048          * @param array $test Test info supplied by TestFileReader
1049          * @param callable|null $nextTeardown
1050          * @return ScopedCallback
1051          */
1052         public function perTestSetup( $test, $nextTeardown = null ) {
1053                 $teardown = [];
1054
1055                 $this->checkSetupDone( 'setupDatabase', 'setDatabase' );
1056                 $teardown[] = $this->markSetupDone( 'perTestSetup' );
1057
1058                 $opts = $this->parseOptions( $test['options'] );
1059                 $config = $test['config'];
1060
1061                 // Find out values for some special options.
1062                 $langCode =
1063                         self::getOptionValue( 'language', $opts, 'en' );
1064                 $variant =
1065                         self::getOptionValue( 'variant', $opts, false );
1066                 $maxtoclevel =
1067                         self::getOptionValue( 'wgMaxTocLevel', $opts, 999 );
1068                 $linkHolderBatchSize =
1069                         self::getOptionValue( 'wgLinkHolderBatchSize', $opts, 1000 );
1070
1071                 // Default to fallback skin, but allow it to be overridden
1072                 $skin = self::getOptionValue( 'skin', $opts, 'fallback' );
1073
1074                 $setup = [
1075                         'wgEnableUploads' => self::getOptionValue( 'wgEnableUploads', $opts, true ),
1076                         'wgLanguageCode' => $langCode,
1077                         'wgRawHtml' => self::getOptionValue( 'wgRawHtml', $opts, false ),
1078                         'wgNamespacesWithSubpages' => array_fill_keys(
1079                                 MWNamespace::getValidNamespaces(), isset( $opts['subpage'] )
1080                         ),
1081                         'wgMaxTocLevel' => $maxtoclevel,
1082                         'wgAllowExternalImages' => self::getOptionValue( 'wgAllowExternalImages', $opts, true ),
1083                         'wgThumbLimits' => [ self::getOptionValue( 'thumbsize', $opts, 180 ) ],
1084                         'wgDefaultLanguageVariant' => $variant,
1085                         'wgLinkHolderBatchSize' => $linkHolderBatchSize,
1086                         // Set as a JSON object like:
1087                         // wgEnableMagicLinks={"ISBN":false, "PMID":false, "RFC":false}
1088                         'wgEnableMagicLinks' => self::getOptionValue( 'wgEnableMagicLinks', $opts, [] )
1089                                 + [ 'ISBN' => true, 'PMID' => true, 'RFC' => true ],
1090                         // Test with legacy encoding by default until HTML5 is very stable and default
1091                         'wgFragmentMode' => [ 'legacy' ],
1092                 ];
1093
1094                 $nonIncludable = self::getOptionValue( 'wgNonincludableNamespaces', $opts, false );
1095                 if ( $nonIncludable !== false ) {
1096                         $setup['wgNonincludableNamespaces'] = [ $nonIncludable ];
1097                 }
1098
1099                 if ( $config ) {
1100                         $configLines = explode( "\n", $config );
1101
1102                         foreach ( $configLines as $line ) {
1103                                 list( $var, $value )  = explode( '=', $line, 2 );
1104                                 $setup[$var] = eval( "return $value;" );
1105                         }
1106                 }
1107
1108                 /** @since 1.20 */
1109                 Hooks::run( 'ParserTestGlobals', [ &$setup ] );
1110
1111                 // Create tidy driver
1112                 if ( isset( $opts['tidy'] ) ) {
1113                         // Cache a driver instance
1114                         if ( $this->tidyDriver === null ) {
1115                                 $this->tidyDriver = MWTidy::factory();
1116                         }
1117                         $tidy = $this->tidyDriver;
1118                 } else {
1119                         $tidy = false;
1120                 }
1121
1122                 # Suppress warnings about running tests without tidy
1123                 Wikimedia\suppressWarnings();
1124                 wfDeprecated( 'disabling tidy' );
1125                 wfDeprecated( 'MWTidy::setInstance' );
1126                 Wikimedia\restoreWarnings();
1127
1128                 MWTidy::setInstance( $tidy );
1129                 $teardown[] = function () {
1130                         MWTidy::destroySingleton();
1131                 };
1132
1133                 // Set content language. This invalidates the magic word cache and title services
1134                 $lang = Language::factory( $langCode );
1135                 $lang->resetNamespaces();
1136                 $setup['wgContLang'] = $lang;
1137                 $setup[] = function () use ( $lang ) {
1138                         MediaWikiServices::getInstance()->disableService( 'ContentLanguage' );
1139                         MediaWikiServices::getInstance()->redefineService(
1140                                 'ContentLanguage',
1141                                 function () use ( $lang ) {
1142                                         return $lang;
1143                                 }
1144                         );
1145                 };
1146                 $teardown[] = function () {
1147                         MediaWikiServices::getInstance()->resetServiceForTesting( 'ContentLanguage' );
1148                 };
1149                 $reset = function () {
1150                         MediaWikiServices::getInstance()->resetServiceForTesting( 'MagicWordFactory' );
1151                         $this->resetTitleServices();
1152                 };
1153                 $setup[] = $reset;
1154                 $teardown[] = $reset;
1155
1156                 // Make a user object with the same language
1157                 $user = new User;
1158                 $user->setOption( 'language', $langCode );
1159                 $setup['wgLang'] = $lang;
1160
1161                 // We (re)set $wgThumbLimits to a single-element array above.
1162                 $user->setOption( 'thumbsize', 0 );
1163
1164                 $setup['wgUser'] = $user;
1165
1166                 // And put both user and language into the context
1167                 $context = RequestContext::getMain();
1168                 $context->setUser( $user );
1169                 $context->setLanguage( $lang );
1170                 // And the skin!
1171                 $oldSkin = $context->getSkin();
1172                 $skinFactory = MediaWikiServices::getInstance()->getSkinFactory();
1173                 $context->setSkin( $skinFactory->makeSkin( $skin ) );
1174                 $context->setOutput( new OutputPage( $context ) );
1175                 $setup['wgOut'] = $context->getOutput();
1176                 $teardown[] = function () use ( $context, $oldSkin ) {
1177                         // Clear language conversion tables
1178                         $wrapper = TestingAccessWrapper::newFromObject(
1179                                 $context->getLanguage()->getConverter()
1180                         );
1181                         $wrapper->reloadTables();
1182                         // Reset context to the restored globals
1183                         $context->setUser( $GLOBALS['wgUser'] );
1184                         $context->setLanguage( $GLOBALS['wgContLang'] );
1185                         $context->setSkin( $oldSkin );
1186                         $context->setOutput( $GLOBALS['wgOut'] );
1187                 };
1188
1189                 $teardown[] = $this->executeSetupSnippets( $setup );
1190
1191                 return $this->createTeardownObject( $teardown, $nextTeardown );
1192         }
1193
1194         /**
1195          * List of temporary tables to create, without prefix.
1196          * Some of these probably aren't necessary.
1197          * @return array
1198          */
1199         private function listTables() {
1200                 global $wgActorTableSchemaMigrationStage;
1201
1202                 $tables = [ 'user', 'user_properties', 'user_former_groups', 'page', 'page_restrictions',
1203                         'protected_titles', 'revision', 'ip_changes', 'text', 'pagelinks', 'imagelinks',
1204                         'categorylinks', 'templatelinks', 'externallinks', 'langlinks', 'iwlinks',
1205                         'site_stats', 'ipblocks', 'image', 'oldimage',
1206                         'recentchanges', 'watchlist', 'interwiki', 'logging', 'log_search',
1207                         'querycache', 'objectcache', 'job', 'l10n_cache', 'redirect', 'querycachetwo',
1208                         'archive', 'user_groups', 'page_props', 'category',
1209                         'slots', 'content', 'slot_roles', 'content_models',
1210                         'comment', 'revision_comment_temp',
1211                 ];
1212
1213                 if ( $wgActorTableSchemaMigrationStage & SCHEMA_COMPAT_WRITE_NEW ) {
1214                         // The new tables for actors are in use
1215                         $tables[] = 'actor';
1216                         $tables[] = 'revision_actor_temp';
1217                 }
1218
1219                 if ( in_array( $this->db->getType(), [ 'mysql', 'sqlite', 'oracle' ] ) ) {
1220                         array_push( $tables, 'searchindex' );
1221                 }
1222
1223                 // Allow extensions to add to the list of tables to duplicate;
1224                 // may be necessary if they hook into page save or other code
1225                 // which will require them while running tests.
1226                 Hooks::run( 'ParserTestTables', [ &$tables ] );
1227
1228                 return $tables;
1229         }
1230
1231         public function setDatabase( IDatabase $db ) {
1232                 $this->db = $db;
1233                 $this->setupDone['setDatabase'] = true;
1234         }
1235
1236         /**
1237          * Set up temporary DB tables.
1238          *
1239          * For best performance, call this once only for all tests. However, it can
1240          * be called at the start of each test if more isolation is desired.
1241          *
1242          * @todo This is basically an unrefactored copy of
1243          * MediaWikiTestCase::setupAllTestDBs. They should be factored out somehow.
1244          *
1245          * Do not call this function from a MediaWikiTestCase subclass, since
1246          * MediaWikiTestCase does its own DB setup. Instead use setDatabase().
1247          *
1248          * @see staticSetup() for more information about setup/teardown
1249          *
1250          * @param ScopedCallback|null $nextTeardown The next teardown object
1251          * @return ScopedCallback The teardown object
1252          */
1253         public function setupDatabase( $nextTeardown = null ) {
1254                 global $wgDBprefix;
1255
1256                 $this->db = wfGetDB( DB_MASTER );
1257                 $dbType = $this->db->getType();
1258
1259                 if ( $dbType == 'oracle' ) {
1260                         $suspiciousPrefixes = [ 'pt_', MediaWikiTestCase::ORA_DB_PREFIX ];
1261                 } else {
1262                         $suspiciousPrefixes = [ 'parsertest_', MediaWikiTestCase::DB_PREFIX ];
1263                 }
1264                 if ( in_array( $wgDBprefix, $suspiciousPrefixes ) ) {
1265                         throw new MWException( "\$wgDBprefix=$wgDBprefix suggests DB setup is already done" );
1266                 }
1267
1268                 $teardown = [];
1269
1270                 $teardown[] = $this->markSetupDone( 'setupDatabase' );
1271
1272                 # CREATE TEMPORARY TABLE breaks if there is more than one server
1273                 if ( MediaWikiServices::getInstance()->getDBLoadBalancer()->getServerCount() != 1 ) {
1274                         $this->useTemporaryTables = false;
1275                 }
1276
1277                 $temporary = $this->useTemporaryTables || $dbType == 'postgres';
1278                 $prefix = $dbType != 'oracle' ? 'parsertest_' : 'pt_';
1279
1280                 $this->dbClone = new CloneDatabase( $this->db, $this->listTables(), $prefix );
1281                 $this->dbClone->useTemporaryTables( $temporary );
1282                 $this->dbClone->cloneTableStructure();
1283                 CloneDatabase::changePrefix( $prefix );
1284
1285                 if ( $dbType == 'oracle' ) {
1286                         $this->db->query( 'BEGIN FILL_WIKI_INFO; END;' );
1287                         # Insert 0 user to prevent FK violations
1288
1289                         # Anonymous user
1290                         $this->db->insert( 'user', [
1291                                 'user_id' => 0,
1292                                 'user_name' => 'Anonymous' ] );
1293                 }
1294
1295                 $teardown[] = function () {
1296                         $this->teardownDatabase();
1297                 };
1298
1299                 // Wipe some DB query result caches on setup and teardown
1300                 $reset = function () {
1301                         MediaWikiServices::getInstance()->getLinkCache()->clear();
1302
1303                         // Clear the message cache
1304                         MessageCache::singleton()->clear();
1305                 };
1306                 $reset();
1307                 $teardown[] = $reset;
1308                 return $this->createTeardownObject( $teardown, $nextTeardown );
1309         }
1310
1311         /**
1312          * Add data about uploads to the new test DB, and set up the upload
1313          * directory. This should be called after either setDatabase() or
1314          * setupDatabase().
1315          *
1316          * @param ScopedCallback|null $nextTeardown The next teardown object
1317          * @return ScopedCallback The teardown object
1318          */
1319         public function setupUploads( $nextTeardown = null ) {
1320                 $teardown = [];
1321
1322                 $this->checkSetupDone( 'setupDatabase', 'setDatabase' );
1323                 $teardown[] = $this->markSetupDone( 'setupUploads' );
1324
1325                 // Create the files in the upload directory (or pretend to create them
1326                 // in a MockFileBackend). Append teardown callback.
1327                 $teardown[] = $this->setupUploadBackend();
1328
1329                 // Create a user
1330                 $user = User::createNew( 'WikiSysop' );
1331
1332                 // Register the uploads in the database
1333
1334                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Foobar.jpg' ) );
1335                 # note that the size/width/height/bits/etc of the file
1336                 # are actually set by inspecting the file itself; the arguments
1337                 # to recordUpload2 have no effect.  That said, we try to make things
1338                 # match up so it is less confusing to readers of the code & tests.
1339                 $image->recordUpload2( '', 'Upload of some lame file', 'Some lame file', [
1340                         'size' => 7881,
1341                         'width' => 1941,
1342                         'height' => 220,
1343                         'bits' => 8,
1344                         'media_type' => MEDIATYPE_BITMAP,
1345                         'mime' => 'image/jpeg',
1346                         'metadata' => serialize( [] ),
1347                         'sha1' => Wikimedia\base_convert( '1', 16, 36, 31 ),
1348                         'fileExists' => true
1349                 ], $this->db->timestamp( '20010115123500' ), $user );
1350
1351                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Thumb.png' ) );
1352                 # again, note that size/width/height below are ignored; see above.
1353                 $image->recordUpload2( '', 'Upload of some lame thumbnail', 'Some lame thumbnail', [
1354                         'size' => 22589,
1355                         'width' => 135,
1356                         'height' => 135,
1357                         'bits' => 8,
1358                         'media_type' => MEDIATYPE_BITMAP,
1359                         'mime' => 'image/png',
1360                         'metadata' => serialize( [] ),
1361                         'sha1' => Wikimedia\base_convert( '2', 16, 36, 31 ),
1362                         'fileExists' => true
1363                 ], $this->db->timestamp( '20130225203040' ), $user );
1364
1365                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Foobar.svg' ) );
1366                 $image->recordUpload2( '', 'Upload of some lame SVG', 'Some lame SVG', [
1367                                 'size'        => 12345,
1368                                 'width'       => 240,
1369                                 'height'      => 180,
1370                                 'bits'        => 0,
1371                                 'media_type'  => MEDIATYPE_DRAWING,
1372                                 'mime'        => 'image/svg+xml',
1373                                 'metadata'    => serialize( [
1374                                         'version'        => SvgHandler::SVG_METADATA_VERSION,
1375                                         'width'          => 240,
1376                                         'height'         => 180,
1377                                         'originalWidth'  => '100%',
1378                                         'originalHeight' => '100%',
1379                                         'translations'   => [
1380                                                 'en' => SVGReader::LANG_FULL_MATCH,
1381                                                 'ru' => SVGReader::LANG_FULL_MATCH,
1382                                         ],
1383                                 ] ),
1384                                 'sha1'        => Wikimedia\base_convert( '', 16, 36, 31 ),
1385                                 'fileExists'  => true
1386                 ], $this->db->timestamp( '20010115123500' ), $user );
1387
1388                 # This image will be blacklisted in [[MediaWiki:Bad image list]]
1389                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Bad.jpg' ) );
1390                 $image->recordUpload2( '', 'zomgnotcensored', 'Borderline image', [
1391                         'size' => 12345,
1392                         'width' => 320,
1393                         'height' => 240,
1394                         'bits' => 24,
1395                         'media_type' => MEDIATYPE_BITMAP,
1396                         'mime' => 'image/jpeg',
1397                         'metadata' => serialize( [] ),
1398                         'sha1' => Wikimedia\base_convert( '3', 16, 36, 31 ),
1399                         'fileExists' => true
1400                 ], $this->db->timestamp( '20010115123500' ), $user );
1401
1402                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Video.ogv' ) );
1403                 $image->recordUpload2( '', 'A pretty movie', 'Will it play', [
1404                         'size' => 12345,
1405                         'width' => 320,
1406                         'height' => 240,
1407                         'bits' => 0,
1408                         'media_type' => MEDIATYPE_VIDEO,
1409                         'mime' => 'application/ogg',
1410                         'metadata' => serialize( [] ),
1411                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1412                         'fileExists' => true
1413                 ], $this->db->timestamp( '20010115123500' ), $user );
1414
1415                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Audio.oga' ) );
1416                 $image->recordUpload2( '', 'An awesome hitsong', 'Will it play', [
1417                         'size' => 12345,
1418                         'width' => 0,
1419                         'height' => 0,
1420                         'bits' => 0,
1421                         'media_type' => MEDIATYPE_AUDIO,
1422                         'mime' => 'application/ogg',
1423                         'metadata' => serialize( [] ),
1424                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1425                         'fileExists' => true
1426                 ], $this->db->timestamp( '20010115123500' ), $user );
1427
1428                 # A DjVu file
1429                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'LoremIpsum.djvu' ) );
1430                 $image->recordUpload2( '', 'Upload a DjVu', 'A DjVu', [
1431                         'size' => 3249,
1432                         'width' => 2480,
1433                         'height' => 3508,
1434                         'bits' => 0,
1435                         'media_type' => MEDIATYPE_BITMAP,
1436                         'mime' => 'image/vnd.djvu',
1437                         'metadata' => '<?xml version="1.0" ?>
1438 <!DOCTYPE DjVuXML PUBLIC "-//W3C//DTD DjVuXML 1.1//EN" "pubtext/DjVuXML-s.dtd">
1439 <DjVuXML>
1440 <HEAD></HEAD>
1441 <BODY><OBJECT height="3508" width="2480">
1442 <PARAM name="DPI" value="300" />
1443 <PARAM name="GAMMA" value="2.2" />
1444 </OBJECT>
1445 <OBJECT height="3508" width="2480">
1446 <PARAM name="DPI" value="300" />
1447 <PARAM name="GAMMA" value="2.2" />
1448 </OBJECT>
1449 <OBJECT height="3508" width="2480">
1450 <PARAM name="DPI" value="300" />
1451 <PARAM name="GAMMA" value="2.2" />
1452 </OBJECT>
1453 <OBJECT height="3508" width="2480">
1454 <PARAM name="DPI" value="300" />
1455 <PARAM name="GAMMA" value="2.2" />
1456 </OBJECT>
1457 <OBJECT height="3508" width="2480">
1458 <PARAM name="DPI" value="300" />
1459 <PARAM name="GAMMA" value="2.2" />
1460 </OBJECT>
1461 </BODY>
1462 </DjVuXML>',
1463                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1464                         'fileExists' => true
1465                 ], $this->db->timestamp( '20010115123600' ), $user );
1466
1467                 return $this->createTeardownObject( $teardown, $nextTeardown );
1468         }
1469
1470         /**
1471          * Helper for database teardown, called from the teardown closure. Destroy
1472          * the database clone and fix up some things that CloneDatabase doesn't fix.
1473          *
1474          * @todo Move most things here to CloneDatabase
1475          */
1476         private function teardownDatabase() {
1477                 $this->checkSetupDone( 'setupDatabase' );
1478
1479                 $this->dbClone->destroy();
1480
1481                 if ( $this->useTemporaryTables ) {
1482                         if ( $this->db->getType() == 'sqlite' ) {
1483                                 # Under SQLite the searchindex table is virtual and need
1484                                 # to be explicitly destroyed. See T31912
1485                                 # See also MediaWikiTestCase::destroyDB()
1486                                 wfDebug( __METHOD__ . " explicitly destroying sqlite virtual table parsertest_searchindex\n" );
1487                                 $this->db->query( "DROP TABLE `parsertest_searchindex`" );
1488                         }
1489                         # Don't need to do anything
1490                         return;
1491                 }
1492
1493                 $tables = $this->listTables();
1494
1495                 foreach ( $tables as $table ) {
1496                         if ( $this->db->getType() == 'oracle' ) {
1497                                 $this->db->query( "DROP TABLE pt_$table DROP CONSTRAINTS" );
1498                         } else {
1499                                 $this->db->query( "DROP TABLE `parsertest_$table`" );
1500                         }
1501                 }
1502
1503                 if ( $this->db->getType() == 'oracle' ) {
1504                         $this->db->query( 'BEGIN FILL_WIKI_INFO; END;' );
1505                 }
1506         }
1507
1508         /**
1509          * Upload test files to the backend created by createRepoGroup().
1510          *
1511          * @return callable The teardown callback
1512          */
1513         private function setupUploadBackend() {
1514                 global $IP;
1515
1516                 $repo = RepoGroup::singleton()->getLocalRepo();
1517                 $base = $repo->getZonePath( 'public' );
1518                 $backend = $repo->getBackend();
1519                 $backend->prepare( [ 'dir' => "$base/3/3a" ] );
1520                 $backend->store( [
1521                         'src' => "$IP/tests/phpunit/data/parser/headbg.jpg",
1522                         'dst' => "$base/3/3a/Foobar.jpg"
1523                 ] );
1524                 $backend->prepare( [ 'dir' => "$base/e/ea" ] );
1525                 $backend->store( [
1526                         'src' => "$IP/tests/phpunit/data/parser/wiki.png",
1527                         'dst' => "$base/e/ea/Thumb.png"
1528                 ] );
1529                 $backend->prepare( [ 'dir' => "$base/0/09" ] );
1530                 $backend->store( [
1531                         'src' => "$IP/tests/phpunit/data/parser/headbg.jpg",
1532                         'dst' => "$base/0/09/Bad.jpg"
1533                 ] );
1534                 $backend->prepare( [ 'dir' => "$base/5/5f" ] );
1535                 $backend->store( [
1536                         'src' => "$IP/tests/phpunit/data/parser/LoremIpsum.djvu",
1537                         'dst' => "$base/5/5f/LoremIpsum.djvu"
1538                 ] );
1539
1540                 // No helpful SVG file to copy, so make one ourselves
1541                 $data = '<?xml version="1.0" encoding="utf-8"?>' .
1542                         '<svg xmlns="http://www.w3.org/2000/svg"' .
1543                         ' version="1.1" width="240" height="180"/>';
1544
1545                 $backend->prepare( [ 'dir' => "$base/f/ff" ] );
1546                 $backend->quickCreate( [
1547                         'content' => $data, 'dst' => "$base/f/ff/Foobar.svg"
1548                 ] );
1549
1550                 return function () use ( $backend ) {
1551                         if ( $backend instanceof MockFileBackend ) {
1552                                 // In memory backend, so dont bother cleaning them up.
1553                                 return;
1554                         }
1555                         $this->teardownUploadBackend();
1556                 };
1557         }
1558
1559         /**
1560          * Remove the dummy uploads directory
1561          */
1562         private function teardownUploadBackend() {
1563                 if ( $this->keepUploads ) {
1564                         return;
1565                 }
1566
1567                 $repo = RepoGroup::singleton()->getLocalRepo();
1568                 $public = $repo->getZonePath( 'public' );
1569
1570                 $this->deleteFiles(
1571                         [
1572                                 "$public/3/3a/Foobar.jpg",
1573                                 "$public/e/ea/Thumb.png",
1574                                 "$public/0/09/Bad.jpg",
1575                                 "$public/5/5f/LoremIpsum.djvu",
1576                                 "$public/f/ff/Foobar.svg",
1577                                 "$public/0/00/Video.ogv",
1578                                 "$public/4/41/Audio.oga",
1579                         ]
1580                 );
1581         }
1582
1583         /**
1584          * Delete the specified files and their parent directories
1585          * @param array $files File backend URIs mwstore://...
1586          */
1587         private function deleteFiles( $files ) {
1588                 // Delete the files
1589                 $backend = RepoGroup::singleton()->getLocalRepo()->getBackend();
1590                 foreach ( $files as $file ) {
1591                         $backend->delete( [ 'src' => $file ], [ 'force' => 1 ] );
1592                 }
1593
1594                 // Delete the parent directories
1595                 foreach ( $files as $file ) {
1596                         $tmp = FileBackend::parentStoragePath( $file );
1597                         while ( $tmp ) {
1598                                 if ( !$backend->clean( [ 'dir' => $tmp ] )->isOK() ) {
1599                                         break;
1600                                 }
1601                                 $tmp = FileBackend::parentStoragePath( $tmp );
1602                         }
1603                 }
1604         }
1605
1606         /**
1607          * Add articles to the test DB.
1608          *
1609          * @param array $articles Article info array from TestFileReader
1610          */
1611         public function addArticles( $articles ) {
1612                 $setup = [];
1613                 $teardown = [];
1614
1615                 // Be sure ParserTestRunner::addArticle has correct language set,
1616                 // so that system messages get into the right language cache
1617                 if ( MediaWikiServices::getInstance()->getContentLanguage()->getCode() !== 'en' ) {
1618                         $setup['wgLanguageCode'] = 'en';
1619                         $lang = Language::factory( 'en' );
1620                         $setup['wgContLang'] = $lang;
1621                         $setup[] = function () use ( $lang ) {
1622                                 $services = MediaWikiServices::getInstance();
1623                                 $services->disableService( 'ContentLanguage' );
1624                                 $services->redefineService( 'ContentLanguage', function () use ( $lang ) {
1625                                         return $lang;
1626                                 } );
1627                         };
1628                         $teardown[] = function () {
1629                                 MediaWikiServices::getInstance()->resetServiceForTesting( 'ContentLanguage' );
1630                         };
1631                 }
1632
1633                 // Add special namespaces, in case that hasn't been done by staticSetup() yet
1634                 $this->appendNamespaceSetup( $setup, $teardown );
1635
1636                 // wgCapitalLinks obviously needs initialisation
1637                 $setup['wgCapitalLinks'] = true;
1638
1639                 $teardown[] = $this->executeSetupSnippets( $setup );
1640
1641                 foreach ( $articles as $info ) {
1642                         $this->addArticle( $info['name'], $info['text'], $info['file'], $info['line'] );
1643                 }
1644
1645                 // Wipe WANObjectCache process cache, which is invalidated by article insertion
1646                 // due to T144706
1647                 ObjectCache::getMainWANInstance()->clearProcessCache();
1648
1649                 $this->executeSetupSnippets( $teardown );
1650         }
1651
1652         /**
1653          * Insert a temporary test article
1654          * @param string $name The title, including any prefix
1655          * @param string $text The article text
1656          * @param string $file The input file name
1657          * @param int|string $line The input line number, for reporting errors
1658          * @throws Exception
1659          * @throws MWException
1660          */
1661         private function addArticle( $name, $text, $file, $line ) {
1662                 $text = self::chomp( $text );
1663                 $name = self::chomp( $name );
1664
1665                 $title = Title::newFromText( $name );
1666                 wfDebug( __METHOD__ . ": adding $name" );
1667
1668                 if ( is_null( $title ) ) {
1669                         throw new MWException( "invalid title '$name' at $file:$line\n" );
1670                 }
1671
1672                 $newContent = ContentHandler::makeContent( $text, $title );
1673
1674                 $page = WikiPage::factory( $title );
1675                 $page->loadPageData( 'fromdbmaster' );
1676
1677                 if ( $page->exists() ) {
1678                         $content = $page->getContent( Revision::RAW );
1679                         // Only reject the title, if the content/content model is different.
1680                         // This makes it easier to create Template:(( or Template:)) in different extensions
1681                         if ( $newContent->equals( $content ) ) {
1682                                 return;
1683                         }
1684                         throw new MWException(
1685                                 "duplicate article '$name' with different content at $file:$line\n"
1686                         );
1687                 }
1688
1689                 // Optionally use mock parser, to make debugging of actual parser tests simpler.
1690                 // But initialise the MessageCache clone first, don't let MessageCache
1691                 // get a reference to the mock object.
1692                 if ( $this->disableSaveParse ) {
1693                         MessageCache::singleton()->getParser();
1694                         $restore = $this->executeSetupSnippets( [ 'wgParser' => new ParserTestMockParser ] );
1695                 } else {
1696                         $restore = false;
1697                 }
1698                 try {
1699                         $status = $page->doEditContent(
1700                                 $newContent,
1701                                 '',
1702                                 EDIT_NEW | EDIT_INTERNAL
1703                         );
1704                 } finally {
1705                         if ( $restore ) {
1706                                 $restore();
1707                         }
1708                 }
1709
1710                 if ( !$status->isOK() ) {
1711                         throw new MWException( $status->getWikiText( false, false, 'en' ) );
1712                 }
1713
1714                 // The RepoGroup cache is invalidated by the creation of file redirects
1715                 if ( $title->inNamespace( NS_FILE ) ) {
1716                         RepoGroup::singleton()->clearCache( $title );
1717                 }
1718         }
1719
1720         /**
1721          * Check if a hook is installed
1722          *
1723          * @param string $name
1724          * @return bool True if tag hook is present
1725          */
1726         public function requireHook( $name ) {
1727                 global $wgParser;
1728
1729                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1730                 if ( isset( $wgParser->mTagHooks[$name] ) ) {
1731                         return true;
1732                 } else {
1733                         $this->recorder->warning( "   This test suite requires the '$name' hook " .
1734                                 "extension, skipping." );
1735                         return false;
1736                 }
1737         }
1738
1739         /**
1740          * Check if a function hook is installed
1741          *
1742          * @param string $name
1743          * @return bool True if function hook is present
1744          */
1745         public function requireFunctionHook( $name ) {
1746                 global $wgParser;
1747
1748                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1749
1750                 if ( isset( $wgParser->mFunctionHooks[$name] ) ) {
1751                         return true;
1752                 } else {
1753                         $this->recorder->warning( "   This test suite requires the '$name' function " .
1754                                 "hook extension, skipping." );
1755                         return false;
1756                 }
1757         }
1758
1759         /**
1760          * Check if a transparent tag hook is installed
1761          *
1762          * @param string $name
1763          * @return bool True if function hook is present
1764          */
1765         public function requireTransparentHook( $name ) {
1766                 global $wgParser;
1767
1768                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1769
1770                 if ( isset( $wgParser->mTransparentTagHooks[$name] ) ) {
1771                         return true;
1772                 } else {
1773                         $this->recorder->warning( "   This test suite requires the '$name' transparent " .
1774                                 "hook extension, skipping.\n" );
1775                         return false;
1776                 }
1777         }
1778
1779         /**
1780          * Fake constant timestamp to make sure time-related parser
1781          * functions give a persistent value.
1782          *
1783          * - Parser::getVariableValue (via ParserGetVariableValueTs hook)
1784          * - Parser::preSaveTransform (via ParserOptions)
1785          */
1786         private function getFakeTimestamp() {
1787                 // parsed as '1970-01-01T00:02:03Z'
1788                 return 123;
1789         }
1790 }