X-Git-Url: http://git.heureux-cyclage.org/?a=blobdiff_plain;f=maintenance%2FparserTests.inc;h=5c7ed473604e3f4439f8d2374ea44841fd69d8c6;hb=d57050f8fca2a9465e7e2254d277648d7e095dee;hp=01409c174b71d2d6c54a458c7a5895248555c6c7;hpb=213a310d097a96ab75c8134b4d1d4728e3897243;p=lhc%2Fweb%2Fwiklou.git diff --git a/maintenance/parserTests.inc b/maintenance/parserTests.inc index 01409c174b..5c7ed47360 100644 --- a/maintenance/parserTests.inc +++ b/maintenance/parserTests.inc @@ -26,7 +26,7 @@ /** */ $options = array( 'quick', 'color', 'quiet', 'help', 'show-output', 'record' ); -$optionsWithArgs = array( 'regex' ); +$optionsWithArgs = array( 'regex', 'seed' ); require_once( 'commandLine.inc' ); require_once( "$IP/maintenance/parserTestsParserHook.php" ); @@ -62,6 +62,10 @@ class ParserTest { */ private $oldTablePrefix; + private $maxFuzzTestLength = 300; + private $fuzzSeed = 0; + private $memoryLimit = 50; + /** * Sets terminal colorization and diff/quick modes depending on OS and * command-line options (--color and --quick). @@ -117,6 +121,10 @@ class ParserTest { } $this->keepUploads = isset( $options['keep-uploads'] ); + if ( isset( $options['seed'] ) ) { + $this->fuzzSeed = intval( $options['seed'] ) - 1; + } + $this->hooks = array(); $this->functionHooks = array(); } @@ -133,6 +141,119 @@ class ParserTest { } } + /** + * Run a fuzz test series + * Draw input from a set of test files + */ + function fuzzTest( $filenames ) { + $dict = $this->getFuzzInput( $filenames ); + $dictSize = strlen( $dict ); + $logMaxLength = log( $this->maxFuzzTestLength ); + $this->setupDatabase(); + ini_set( 'memory_limit', $this->memoryLimit * 1048576 ); + + $numTotal = 0; + $numSuccess = 0; + $user = new User; + $opts = ParserOptions::newFromUser( $user ); + $title = Title::makeTitle( NS_MAIN, 'Parser_test' ); + + while ( true ) { + // Generate test input + mt_srand( ++$this->fuzzSeed ); + $totalLength = mt_rand( 1, $this->maxFuzzTestLength ); + $input = ''; + while ( strlen( $input ) < $totalLength ) { + $logHairLength = mt_rand( 0, 1000000 ) / 1000000 * $logMaxLength; + $hairLength = min( intval( exp( $logHairLength ) ), $dictSize ); + $offset = mt_rand( 0, $dictSize - $hairLength ); + $input .= substr( $dict, $offset, $hairLength ); + } + + $this->setupGlobals(); + $parser = $this->getParser(); + // Run the test + try { + $parser->parse( $input, $title, $opts ); + $fail = false; + } catch ( Exception $exception ) { + $fail = true; + } + + if ( $fail ) { + echo "Test failed with seed {$this->fuzzSeed}\n"; + echo "Input:\n"; + var_dump( $input ); + echo "\n\n"; + echo "$exception\n"; + } else { + $numSuccess++; + } + $numTotal++; + $this->teardownGlobals(); + $parser->__destruct(); + + if ( $numTotal % 100 == 0 ) { + $usage = intval( memory_get_usage( true ) / $this->memoryLimit / 1048576 * 100 ); + echo "{$this->fuzzSeed}: $numSuccess/$numTotal (mem: $usage%)\n"; + if ( $usage > 90 ) { + echo "Out of memory:\n"; + $memStats = $this->getMemoryBreakdown(); + foreach ( $memStats as $name => $usage ) { + echo "$name: $usage\n"; + } + $this->abort(); + } + } + } + } + + /** + * Get an input dictionary from a set of parser test files + */ + function getFuzzInput( $filenames ) { + $dict = ''; + foreach( $filenames as $filename ) { + $contents = file_get_contents( $filename ); + preg_match_all( '/!!\s*input\n(.*?)\n!!\s*result/s', $contents, $matches ); + foreach ( $matches[1] as $match ) { + $dict .= $match . "\n"; + } + } + return $dict; + } + + /** + * Get a memory usage breakdown + */ + function getMemoryBreakdown() { + $memStats = array(); + foreach ( $GLOBALS as $name => $value ) { + $memStats['$'.$name] = strlen( serialize( $value ) ); + } + $classes = get_declared_classes(); + foreach ( $classes as $class ) { + $rc = new ReflectionClass( $class ); + $props = $rc->getStaticProperties(); + $memStats[$class] = strlen( serialize( $props ) ); + $methods = $rc->getMethods(); + foreach ( $methods as $method ) { + $memStats[$class] += strlen( serialize( $method->getStaticVariables() ) ); + } + } + $functions = get_defined_functions(); + foreach ( $functions['user'] as $function ) { + $rf = new ReflectionFunction( $function ); + $memStats["$function()"] = strlen( serialize( $rf->getStaticVariables() ) ); + } + asort( $memStats ); + return $memStats; + } + + function abort() { + $this->abort(); + } + /** * Run a series of tests listed in the given text files. * Each test consists of a brief description, wikitext input, @@ -266,6 +387,24 @@ class ParserTest { return $ok; } + /** + * Get a Parser object + */ + function getParser() { + global $wgParserConf; + $class = $wgParserConf['class']; + $parser = new $class( $wgParserConf ); + foreach( $this->hooks as $tag => $callback ) { + $parser->setHook( $tag, $callback ); + } + foreach( $this->functionHooks as $tag => $bits ) { + list( $callback, $flags ) = $bits; + $parser->setFunctionHook( $tag, $callback, $flags ); + } + wfRunHooks( 'ParserTestParser', array( &$parser ) ); + return $parser; + } + /** * Run a given wikitext input through a freshly-constructed wiki parser, * and compare the output against the expected results. @@ -276,7 +415,6 @@ class ParserTest { * @return bool */ private function runTest( $desc, $input, $result, $opts ) { - global $wgParserConf; if( $this->showProgress ) { $this->showTesting( $desc ); } @@ -300,18 +438,7 @@ class ParserTest { } $noxml = (bool)preg_match( '~\\b noxml \\b~x', $opts ); - - $class = $wgParserConf['class']; - $parser = new $class( $wgParserConf ); - foreach( $this->hooks as $tag => $callback ) { - $parser->setHook( $tag, $callback ); - } - foreach( $this->functionHooks as $tag => $bits ) { - list( $callback, $flags ) = $bits; - $parser->setFunctionHook( $tag, $callback, $flags ); - } - wfRunHooks( 'ParserTestParser', array( &$parser ) ); - + $parser = $this->getParser(); $title =& Title::makeTitle( NS_MAIN, $titleText ); $matches = array(); @@ -387,6 +514,8 @@ class ParserTest { self::getOptionValue( '/variant=([a-z]+(?:-[a-z]+)?)/', $opts, false ); $maxtoclevel = self::getOptionValue( '/wgMaxTocLevel=(\d+)/', $opts, 999 ); + $linkHolderBatchSize = + self::getOptionValue( '/wgLinkHolderBatchSize=(\d+)/', $opts, 1000 ); $settings = array( 'wgServer' => 'http://localhost', @@ -432,6 +561,7 @@ class ParserTest { ) ), 'wgDefaultExternalStore' => array(), 'wgForeignFileRepos' => array(), + 'wgLinkHolderBatchSize' => $linkHolderBatchSize, ); $this->savedGlobals = array(); foreach( $settings as $var => $val ) { @@ -441,6 +571,7 @@ class ParserTest { $langObj = Language::factory( $lang ); $GLOBALS['wgLang'] = $langObj; $GLOBALS['wgContLang'] = $langObj; + $GLOBALS['wgMemc'] = new FakeMemCachedClient; //$GLOBALS['wgMessageCache'] = new MessageCache( new BagOStuff(), false, 0, $GLOBALS['wgDBname'] ); @@ -491,7 +622,9 @@ class ParserTest { $this->oldTablePrefix = $wgDBprefix; # CREATE TEMPORARY TABLE breaks if there is more than one server - if ( wfGetLB()->getServerCount() != 1 ) { + # FIXME: r40209 makes temporary tables break even with just one server + # FIXME: (bug 15892); disabling the feature entirely as a temporary fix + if ( true || wfGetLB()->getServerCount() != 1 ) { $this->useTemporaryTables = false; } @@ -551,10 +684,10 @@ class ParserTest { # Hack: insert a few Wikipedia in-project interwiki prefixes, # for testing inter-language links $db->insert( 'interwiki', array( - array( 'iw_prefix' => 'Wikipedia', + array( 'iw_prefix' => 'wikipedia', 'iw_url' => 'http://en.wikipedia.org/wiki/$1', 'iw_local' => 0 ), - array( 'iw_prefix' => 'MeatBall', + array( 'iw_prefix' => 'meatball', 'iw_url' => 'http://www.usemod.com/cgi-bin/mb.pl?$1', 'iw_local' => 0 ), array( 'iw_prefix' => 'zh', @@ -621,11 +754,12 @@ class ParserTest { return; } + /* $tables = $this->listTables(); $db = wfGetDB( DB_MASTER ); foreach ( $tables as $table ) { $db->query( "DROP TABLE `parsertest_$table`" ); - } + }*/ } /** @@ -645,9 +779,11 @@ class ParserTest { } wfDebug( "Creating upload directory $dir\n" ); - mkdir( $dir ); - mkdir( $dir . '/3' ); - mkdir( $dir . '/3/3a' ); + if ( file_exists( $dir ) ) { + wfDebug( "Already exists!\n" ); + return $dir; + } + wfMkdirParents( $dir . '/3/3a' ); copy( "$IP/skins/monobook/headbg.jpg", "$dir/3/3a/Foobar.jpg" ); return $dir; } @@ -658,6 +794,7 @@ class ParserTest { */ private function teardownGlobals() { RepoGroup::destroySingleton(); + LinkCache::singleton()->clear(); foreach( $this->savedGlobals as $var => $val ) { $GLOBALS[$var] = $val; }