Explored some ideas for HipHop optimisation. Made a preprocessor implementation,...
authorTim Starling <tstarling@users.mediawiki.org>
Thu, 2 Jun 2011 02:44:33 +0000 (02:44 +0000)
committerTim Starling <tstarling@users.mediawiki.org>
Thu, 2 Jun 2011 02:44:33 +0000 (02:44 +0000)
* Set EnableHipHopSyntax=true to enable string and integer type hints. I gave the file a .hphp extension to avoid false alarms in syntax checking scripts.
* Made sure almost all the local variables in preprocessToObj() have a specific type, instead of being variants. This is useful for integers, but has the largest impact for objects, since dynamic method calls can be avoided.
* Stopped using extract() since it forces all local variables to be variants, and adds some hashtable initialisation overhead.
* Found a way to cast a variant to a specific object class, by abusing argument type hinting. The method does not require special syntax; it is harmless in Zend PHP.
* Wrapped various internal function calls with type casts. strspn() and substr() need to be wrapped with intval() and strval() respectively, since they return a variant to support special error return values. HipHop isn't smart enough to know whether the error case will be triggered.
* Replaced most instances of double-equals with triple-equals. Profiling indicates that this makes a very large difference when comparing strings, much more so than in Zend.

includes/AutoLoader.php
includes/parser/Preprocessor_HipHop.hphp [new file with mode: 0644]
maintenance/hiphop/compiler.conf

index 4f612ea..4848902 100644 (file)
@@ -569,35 +569,47 @@ $wgAutoloadLocalClasses = array(
        'LinkHolderArray' => 'includes/parser/LinkHolderArray.php',
        'LinkMarkerReplacer' => 'includes/parser/Parser_LinkHooks.php',
        'MWTidy' => 'includes/parser/Tidy.php',
-       'Parser' => 'includes/parser/Parser.php',
-       'Parser_DiffTest' => 'includes/parser/Parser_DiffTest.php',
-       'Parser_LinkHooks' => 'includes/parser/Parser_LinkHooks.php',
-       'ParserCache' => 'includes/parser/ParserCache.php',
-       'ParserOptions' => 'includes/parser/ParserOptions.php',
-       'ParserOutput' => 'includes/parser/ParserOutput.php',
        'PPCustomFrame_DOM' => 'includes/parser/Preprocessor_DOM.php',
        'PPCustomFrame_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'PPCustomFrame_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
        'PPDAccum_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'PPDAccum_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
        'PPDPart' => 'includes/parser/Preprocessor_DOM.php',
        'PPDPart_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'PPDPart_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
        'PPDStack' => 'includes/parser/Preprocessor_DOM.php',
-       'PPDStack_Hash' => 'includes/parser/Preprocessor_Hash.php',
        'PPDStackElement' => 'includes/parser/Preprocessor_DOM.php',
        'PPDStackElement_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'PPDStackElement_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
+       'PPDStack_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'PPDStack_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
        'PPFrame' => 'includes/parser/Preprocessor.php',
        'PPFrame_DOM' => 'includes/parser/Preprocessor_DOM.php',
        'PPFrame_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'PPFrame_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
        'PPNode' => 'includes/parser/Preprocessor.php',
        'PPNode_DOM' => 'includes/parser/Preprocessor_DOM.php',
        'PPNode_Hash_Array' => 'includes/parser/Preprocessor_Hash.php',
        'PPNode_Hash_Attr' => 'includes/parser/Preprocessor_Hash.php',
        'PPNode_Hash_Text' => 'includes/parser/Preprocessor_Hash.php',
        'PPNode_Hash_Tree' => 'includes/parser/Preprocessor_Hash.php',
+       'PPNode_HipHop_Array' => 'includes/parser/Preprocessor_HipHop.hphp',
+       'PPNode_HipHop_Attr' => 'includes/parser/Preprocessor_HipHop.hphp',
+       'PPNode_HipHop_Text' => 'includes/parser/Preprocessor_HipHop.hphp',
+       'PPNode_HipHop_Tree' => 'includes/parser/Preprocessor_HipHop.hphp',
        'PPTemplateFrame_DOM' => 'includes/parser/Preprocessor_DOM.php',
        'PPTemplateFrame_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'PPTemplateFrame_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
+       'Parser' => 'includes/parser/Parser.php',
+       'ParserCache' => 'includes/parser/ParserCache.php',
+       'ParserOptions' => 'includes/parser/ParserOptions.php',
+       'ParserOutput' => 'includes/parser/ParserOutput.php',
+       'Parser_DiffTest' => 'includes/parser/Parser_DiffTest.php',
+       'Parser_LinkHooks' => 'includes/parser/Parser_LinkHooks.php',
        'Preprocessor' => 'includes/parser/Preprocessor.php',
        'Preprocessor_DOM' => 'includes/parser/Preprocessor_DOM.php',
        'Preprocessor_Hash' => 'includes/parser/Preprocessor_Hash.php',
+       'Preprocessor_HipHop' => 'includes/parser/Preprocessor_HipHop.hphp',
        'StripState' => 'includes/parser/StripState.php',
 
        # includes/profiler
diff --git a/includes/parser/Preprocessor_HipHop.hphp b/includes/parser/Preprocessor_HipHop.hphp
new file mode 100644 (file)
index 0000000..dc404f7
--- /dev/null
@@ -0,0 +1,1941 @@
+<?php
+/**
+ * A preprocessor optimised for HipHop, using HipHop-specific syntax.
+ * vim: ft=php
+ *
+ * @file
+ * @ingroup Parser
+ */
+
+/**
+ * @ingroup Parser
+ */
+class Preprocessor_HipHop implements Preprocessor {
+       /**
+        * @var Parser
+        */
+       var $parser;
+
+       const CACHE_VERSION = 1;
+
+       function __construct( $parser ) {
+               $this->parser = $parser;
+       }
+
+       /**
+        * @return PPFrame_HipHop
+        */
+       function newFrame() {
+               return new PPFrame_HipHop( $this );
+       }
+
+       /**
+        * @param $args
+        * @return PPCustomFrame_HipHop
+        */
+       function newCustomFrame( array $args ) {
+               return new PPCustomFrame_HipHop( $this, $args );
+       }
+
+       /**
+        * @param $values array
+        * @return PPNode_HipHop_Array
+        */
+       function newPartNodeArray( $values ) {
+               $list = array();
+
+               foreach ( $values as $k => $val ) {
+                       $partNode = new PPNode_HipHop_Tree( 'part' );
+                       $nameNode = new PPNode_HipHop_Tree( 'name' );
+
+                       if ( is_int( $k ) ) {
+                               $nameNode->addChild( new PPNode_HipHop_Attr( 'index', $k ) );
+                               $partNode->addChild( $nameNode );
+                       } else {
+                               $nameNode->addChild( new PPNode_HipHop_Text( $k ) );
+                               $partNode->addChild( $nameNode );
+                               $partNode->addChild( new PPNode_HipHop_Text( '=' ) );
+                       }
+
+                       $valueNode = new PPNode_HipHop_Tree( 'value' );
+                       $valueNode->addChild( new PPNode_HipHop_Text( $val ) );
+                       $partNode->addChild( $valueNode );
+
+                       $list[] = $partNode;
+               }
+
+               $node = new PPNode_HipHop_Array( $list );
+               return $node;
+       }
+
+       /**
+        * Preprocess some wikitext and return the document tree.
+        * This is the ghost of Parser::replace_variables().
+        *
+        * @param $text String: the text to parse
+        * @param $flags Integer: bitwise combination of:
+        *          Parser::PTD_FOR_INCLUSION    Handle <noinclude>/<includeonly> as if the text is being
+        *                                     included. Default is to assume a direct page view.
+        *
+        * The generated DOM tree must depend only on the input text and the flags.
+        * The DOM tree must be the same in OT_HTML and OT_WIKI mode, to avoid a regression of bug 4899.
+        *
+        * Any flag added to the $flags parameter here, or any other parameter liable to cause a
+        * change in the DOM tree for a given text, must be passed through the section identifier
+        * in the section edit link and thus back to extractSections().
+        *
+        * The output of this function is currently only cached in process memory, but a persistent
+        * cache may be implemented at a later date which takes further advantage of these strict
+        * dependency requirements.
+        *
+        * @return PPNode_HipHop_Tree
+        */
+       function preprocessToObj( string $text, int $flags = 0 ) {
+               wfProfileIn( __METHOD__ );
+
+               // Check cache.
+               global $wgMemc, $wgPreprocessorCacheThreshold;
+
+               $cacheable = ($wgPreprocessorCacheThreshold !== false && strlen( $text ) > $wgPreprocessorCacheThreshold);
+               if ( $cacheable ) {
+                       wfProfileIn( __METHOD__.'-cacheable' );
+
+                       $cacheKey = strval( wfMemcKey( 'preprocess-hash', md5($text), $flags ) );
+                       $cacheValue = strval( $wgMemc->get( $cacheKey ) );
+                       if ( $cacheValue !== '' ) {
+                               $version = substr( $cacheValue, 0, 8 );
+                               if ( intval( $version ) == self::CACHE_VERSION ) {
+                                       $hash = unserialize( substr( $cacheValue, 8 ) );
+                                       // From the cache
+                                       wfDebugLog( "Preprocessor",
+                                               "Loaded preprocessor hash from memcached (key $cacheKey)" );
+                                       wfProfileOut( __METHOD__.'-cacheable' );
+                                       wfProfileOut( __METHOD__ );
+                                       return $hash;
+                               }
+                       }
+                       wfProfileIn( __METHOD__.'-cache-miss' );
+               }
+
+               $rules = array(
+                       '{' => array(
+                               'end' => '}',
+                               'names' => array(
+                                       2 => 'template',
+                                       3 => 'tplarg',
+                               ),
+                               'min' => 2,
+                               'max' => 3,
+                       ),
+                       '[' => array(
+                               'end' => ']',
+                               'names' => array( 2 => 'LITERAL' ),
+                               'min' => 2,
+                               'max' => 2,
+                       )
+               );
+
+               $forInclusion = (bool)( $flags & Parser::PTD_FOR_INCLUSION );
+
+               $xmlishElements = (array)$this->parser->getStripList();
+               $enableOnlyinclude = false;
+               if ( $forInclusion ) {
+                       $ignoredTags = array( 'includeonly', '/includeonly' );
+                       $ignoredElements = array( 'noinclude' );
+                       $xmlishElements[] = 'noinclude';
+                       if ( strpos( $text, '<onlyinclude>' ) !== false && strpos( $text, '</onlyinclude>' ) !== false ) {
+                               $enableOnlyinclude = true;
+                       }
+               } else if ( $this->parser->ot['wiki'] ) {
+                       $ignoredTags = array( 'noinclude', '/noinclude', 'onlyinclude', '/onlyinclude', 'includeonly', '/includeonly' );
+                       $ignoredElements = array();
+               } else {
+                       $ignoredTags = array( 'noinclude', '/noinclude', 'onlyinclude', '/onlyinclude' );
+                       $ignoredElements = array( 'includeonly' );
+                       $xmlishElements[] = 'includeonly';
+               }
+               $xmlishRegex = implode( '|', array_merge( $xmlishElements, $ignoredTags ) );
+
+               // Use "A" modifier (anchored) instead of "^", because ^ doesn't work with an offset
+               $elementsRegex = "~($xmlishRegex)(?:\s|\/>|>)|(!--)~iA";
+
+               $stack = new PPDStack_HipHop;
+
+               $searchBase = "[{<\n";
+               $revText = strrev( $text ); // For fast reverse searches
+
+               $i = 0;                     # Input pointer, starts out pointing to a pseudo-newline before the start
+               $accum = $stack->getAccum();   # Current accumulator
+               $headingIndex = 1;
+               $stackFlags = array(
+                       'findPipe' => false, # True to take notice of pipe characters
+                       'findEquals' => false, # True to find equals signs in arguments
+                       'inHeading' => false, # True if $i is inside a possible heading
+               );
+               $noMoreGT = false;         # True if there are no more greater-than (>) signs right of $i
+               $findOnlyinclude = $enableOnlyinclude; # True to ignore all input up to the next <onlyinclude>
+               $fakeLineStart = true;     # Do a line-start run without outputting an LF character
+
+               while ( true ) {
+                       //$this->memCheck();
+
+                       if ( $findOnlyinclude ) {
+                               // Ignore all input up to the next <onlyinclude>
+                               $variantStartPos = strpos( $text, '<onlyinclude>', $i );
+                               if ( $variantStartPos === false ) {
+                                       // Ignored section runs to the end
+                                       $accum->addNodeWithText( 'ignore', strval( substr( $text, $i ) ) );
+                                       break;
+                               }
+                               $startPos1 = intval( $variantStartPos );
+                               $tagEndPos = $startPos1 + strlen( '<onlyinclude>' ); // past-the-end
+                               $accum->addNodeWithText( 'ignore', strval( substr( $text, $i, $tagEndPos - $i ) ) );
+                               $i = $tagEndPos;
+                               $findOnlyinclude = false;
+                       }
+
+                       if ( $fakeLineStart ) {
+                               $found = 'line-start';
+                               $curChar = '';
+                       } else {
+                               # Find next opening brace, closing brace or pipe
+                               $search = $searchBase;
+                               if ( $stack->top === false ) {
+                                       $currentClosing = '';
+                               } else {
+                                       $currentClosing = strval( $stack->getTop()->close );
+                                       $search .= $currentClosing;
+                               }
+                               if ( $stackFlags['findPipe'] ) {
+                                       $search .= '|';
+                               }
+                               if ( $stackFlags['findEquals'] ) {
+                                       // First equals will be for the template
+                                       $search .= '=';
+                               }
+                               $rule = null;
+                               # Output literal section, advance input counter
+                               $literalLength = intval( strcspn( $text, $search, $i ) );
+                               if ( $literalLength > 0 ) {
+                                       $accum->addLiteral( strval( substr( $text, $i, $literalLength ) ) );
+                                       $i += $literalLength;
+                               }
+                               if ( $i >= strlen( $text ) ) {
+                                       if ( $currentClosing === "\n" ) {
+                                               // Do a past-the-end run to finish off the heading
+                                               $curChar = '';
+                                               $found = 'line-end';
+                                       } else {
+                                               # All done
+                                               break;
+                                       }
+                               } else {
+                                       $curChar = $text[$i];
+                                       if ( $curChar === '|' ) {
+                                               $found = 'pipe';
+                                       } elseif ( $curChar === '=' ) {
+                                               $found = 'equals';
+                                       } elseif ( $curChar === '<' ) {
+                                               $found = 'angle';
+                                       } elseif ( $curChar === "\n" ) {
+                                               if ( $stackFlags['inHeading'] ) {
+                                                       $found = 'line-end';
+                                               } else {
+                                                       $found = 'line-start';
+                                               }
+                                       } elseif ( $curChar === $currentClosing ) {
+                                               $found = 'close';
+                                       } elseif ( isset( $rules[$curChar] ) ) {
+                                               $found = 'open';
+                                               $rule = $rules[$curChar];
+                                       } else {
+                                               # Some versions of PHP have a strcspn which stops on null characters
+                                               # Ignore and continue
+                                               ++$i;
+                                               continue;
+                                       }
+                               }
+                       }
+
+                       if ( $found === 'angle' ) {
+                               $matches = false;
+                               // Handle </onlyinclude>
+                               if ( $enableOnlyinclude 
+                                       && substr( $text, $i, strlen( '</onlyinclude>' ) ) === '</onlyinclude>' ) 
+                               {
+                                       $findOnlyinclude = true;
+                                       continue;
+                               }
+
+                               // Determine element name
+                               if ( !preg_match( $elementsRegex, $text, $matches, 0, $i + 1 ) ) {
+                                       // Element name missing or not listed
+                                       $accum->addLiteral( '<' );
+                                       ++$i;
+                                       continue;
+                               }
+                               // Handle comments
+                               if ( isset( $matches[2] ) && $matches[2] === '!--' ) {
+                                       // To avoid leaving blank lines, when a comment is both preceded
+                                       // and followed by a newline (ignoring spaces), trim leading and
+                                       // trailing spaces and one of the newlines.
+
+                                       // Find the end
+                                       $variantEndPos = strpos( $text, '-->', $i + 4 );
+                                       if ( $variantEndPos === false ) {
+                                               // Unclosed comment in input, runs to end
+                                               $inner = strval( substr( $text, $i ) );
+                                               $accum->addNodeWithText( 'comment', $inner );
+                                               $i = strlen( $text );
+                                       } else {
+                                               $endPos = intval( $variantEndPos );
+                                               // Search backwards for leading whitespace
+                                               if ( $i ) {
+                                                       $wsStart = $i - intval( strspn( $revText, ' ', strlen( $text ) - $i ) );
+                                               } else {
+                                                       $wsStart = 0;
+                                               }
+                                               // Search forwards for trailing whitespace
+                                               // $wsEnd will be the position of the last space (or the '>' if there's none)
+                                               $wsEnd = $endPos + 2 + intval( strspn( $text, ' ', $endPos + 3 ) );
+                                               // Eat the line if possible
+                                               // TODO: This could theoretically be done if $wsStart == 0, i.e. for comments at
+                                               // the overall start. That's not how Sanitizer::removeHTMLcomments() did it, but
+                                               // it's a possible beneficial b/c break.
+                                               if ( $wsStart > 0 && substr( $text, $wsStart - 1, 1 ) === "\n"
+                                                       && substr( $text, $wsEnd + 1, 1 ) === "\n" )
+                                               {
+                                                       $startPos2 = $wsStart;
+                                                       $endPos = $wsEnd + 1;
+                                                       // Remove leading whitespace from the end of the accumulator
+                                                       // Sanity check first though
+                                                       $wsLength = $i - $wsStart;
+                                                       if ( $wsLength > 0
+                                                               && $accum->lastNode instanceof PPNode_HipHop_Text
+                                                               && substr( $accum->lastNode->value, -$wsLength ) === str_repeat( ' ', $wsLength ) )
+                                                       {
+                                                               $accum->lastNode->value = strval( substr( $accum->lastNode->value, 0, -$wsLength ) );
+                                                       }
+                                                       // Do a line-start run next time to look for headings after the comment
+                                                       $fakeLineStart = true;
+                                               } else {
+                                                       // No line to eat, just take the comment itself
+                                                       $startPos2 = $i;
+                                                       $endPos += 2;
+                                               }
+
+                                               if ( $stack->top ) {
+                                                       $part = $stack->getTop()->getCurrentPart();
+                                                       if ( ! (isset( $part->commentEnd ) && $part->commentEnd == $wsStart - 1 )) {
+                                                               $part->visualEnd = $wsStart;
+                                                       }
+                                                       // Else comments abutting, no change in visual end
+                                                       $part->commentEnd = $endPos;
+                                               }
+                                               $i = $endPos + 1;
+                                               $inner = strval( substr( $text, $startPos2, $endPos - $startPos2 + 1 ) );
+                                               $accum->addNodeWithText( 'comment', $inner );
+                                       }
+                                       continue;
+                               }
+                               $name = strval( $matches[1] );
+                               $lowerName = strtolower( $name );
+                               $attrStart = $i + strlen( $name ) + 1;
+
+                               // Find end of tag
+                               $variantTagEndPos = $noMoreGT ? false : strpos( $text, '>', $attrStart );
+                               if ( $variantTagEndPos === false ) {
+                                       // Infinite backtrack
+                                       // Disable tag search to prevent worst-case O(N^2) performance
+                                       $noMoreGT = true;
+                                       $accum->addLiteral( '<' );
+                                       ++$i;
+                                       continue;
+                               }
+                               $tagEndPos = intval( $variantTagEndPos );
+
+                               // Handle ignored tags
+                               if ( in_array( $lowerName, $ignoredTags ) ) {
+                                       $accum->addNodeWithText( 'ignore', strval( substr( $text, $i, $tagEndPos - $i + 1 ) ) );
+                                       $i = $tagEndPos + 1;
+                                       continue;
+                               }
+
+                               $tagStartPos = $i;
+                               $inner = $close = '';
+                               if ( $text[$tagEndPos-1] === '/' ) {
+                                       // Short end tag
+                                       $attrEnd = $tagEndPos - 1;
+                                       $shortEnd = true;
+                                       $inner = '';
+                                       $i = $tagEndPos + 1;
+                                       $haveClose = false;
+                               } else {
+                                       $attrEnd = $tagEndPos;
+                                       $shortEnd = false;
+                                       // Find closing tag
+                                       if ( preg_match( "/<\/" . preg_quote( $name, '/' ) . "\s*>/i",
+                                                       $text, $matches, PREG_OFFSET_CAPTURE, $tagEndPos + 1 ) )
+                                       {
+                                               $inner = strval( substr( $text, $tagEndPos + 1, $matches[0][1] - $tagEndPos - 1 ) );
+                                               $i = intval( $matches[0][1] ) + strlen( $matches[0][0] );
+                                               $close = strval( $matches[0][0] );
+                                               $haveClose = true;
+                                       } else {
+                                               // No end tag -- let it run out to the end of the text.
+                                               $inner = strval( substr( $text, $tagEndPos + 1 ) );
+                                               $i = strlen( $text );
+                                               $haveClose = false;
+                                       }
+                               }
+                               // <includeonly> and <noinclude> just become <ignore> tags
+                               if ( in_array( $lowerName, $ignoredElements ) ) {
+                                       $accum->addNodeWithText(  'ignore', strval( substr( $text, $tagStartPos, $i - $tagStartPos ) ) );
+                                       continue;
+                               }
+
+                               if ( $attrEnd <= $attrStart ) {
+                                       $attr = '';
+                               } else {
+                                       // Note that the attr element contains the whitespace between name and attribute,
+                                       // this is necessary for precise reconstruction during pre-save transform.
+                                       $attr = strval( substr( $text, $attrStart, $attrEnd - $attrStart ) );
+                               }
+
+                               $extNode = new PPNode_HipHop_Tree( 'ext' );
+                               $extNode->addChild( PPNode_HipHop_Tree::newWithText( 'name', $name ) );
+                               $extNode->addChild( PPNode_HipHop_Tree::newWithText( 'attr', $attr ) );
+                               if ( !$shortEnd ) {
+                                       $extNode->addChild( PPNode_HipHop_Tree::newWithText( 'inner', $inner ) );
+                               }
+                               if ( $haveClose ) {
+                                       $extNode->addChild( PPNode_HipHop_Tree::newWithText( 'close', $close ) );
+                               }
+                               $accum->addNode( $extNode );
+                       }
+
+                       elseif ( $found === 'line-start' ) {
+                               // Is this the start of a heading?
+                               // Line break belongs before the heading element in any case
+                               if ( $fakeLineStart ) {
+                                       $fakeLineStart = false;
+                               } else {
+                                       $accum->addLiteral( $curChar );
+                                       $i++;
+                               }
+
+                               $count = intval( strspn( $text, '=', $i, 6 ) );
+                               if ( $count == 1 && $stackFlags['findEquals'] ) {
+                                       // DWIM: This looks kind of like a name/value separator
+                                       // Let's let the equals handler have it and break the potential heading
+                                       // This is heuristic, but AFAICT the methods for completely correct disambiguation are very complex.
+                               } elseif ( $count > 0 ) {
+                                       $partData = array(
+                                               'open' => "\n",
+                                               'close' => "\n",
+                                               'parts' => array( new PPDPart_HipHop( str_repeat( '=', $count ) ) ),
+                                               'startPos' => $i,
+                                               'count' => $count );
+                                       $stack->push( $partData );
+                                       $accum = $stack->getAccum();
+                                       $stackFlags = $stack->getFlags();
+                                       $i += $count;
+                               }
+                       } elseif ( $found === 'line-end' ) {
+                               $piece = $stack->getTop();
+                               // A heading must be open, otherwise \n wouldn't have been in the search list
+                               assert( $piece->open === "\n" );
+                               $part = $piece->getCurrentPart();
+                               // Search back through the input to see if it has a proper close
+                               // Do this using the reversed string since the other solutions (end anchor, etc.) are inefficient
+                               $wsLength = intval( strspn( $revText, " \t", strlen( $text ) - $i ) );
+                               $searchStart = $i - $wsLength;
+                               if ( isset( $part->commentEnd ) && $searchStart - 1 == $part->commentEnd ) {
+                                       // Comment found at line end
+                                       // Search for equals signs before the comment
+                                       $searchStart = intval( $part->visualEnd );
+                                       $searchStart -= intval( strspn( $revText, " \t", strlen( $text ) - $searchStart ) );
+                               }
+                               $count = intval( $piece->count );
+                               $equalsLength = intval( strspn( $revText, '=', strlen( $text ) - $searchStart ) );
+                               $isTreeNode = false;
+                               $resultAccum = $accum;
+                               if ( $equalsLength > 0 ) {
+                                       if ( $searchStart - $equalsLength == $piece->startPos ) {
+                                               // This is just a single string of equals signs on its own line
+                                               // Replicate the doHeadings behaviour /={count}(.+)={count}/
+                                               // First find out how many equals signs there really are (don't stop at 6)
+                                               $count = $equalsLength;
+                                               if ( $count < 3 ) {
+                                                       $count = 0;
+                                               } else {
+                                                       $count = intval( ( $count - 1 ) / 2 );
+                                                       if ( $count > 6 ) {
+                                                               $count = 6;
+                                                       }
+                                               }
+                                       } else {
+                                               if ( $count > $equalsLength ) {
+                                                       $count = $equalsLength;
+                                               }
+                                       }
+                                       if ( $count > 0 ) {
+                                               // Normal match, output <h>
+                                               $tree = new PPNode_HipHop_Tree( 'possible-h' );
+                                               $tree->addChild( new PPNode_HipHop_Attr( 'level', $count ) );
+                                               $tree->addChild( new PPNode_HipHop_Attr( 'i', $headingIndex++ ) );
+                                               $tree->lastChild->nextSibling = $accum->firstNode;
+                                               $tree->lastChild = $accum->lastNode;
+                                               $isTreeNode = true;
+                                       } else {
+                                               // Single equals sign on its own line, count=0
+                                               // Output $resultAccum
+                                       }
+                               } else {
+                                       // No match, no <h>, just pass down the inner text
+                                       // Output $resultAccum
+                               }
+                               // Unwind the stack
+                               $stack->pop();
+                               $accum = $stack->getAccum();
+                               $stackFlags = $stack->getFlags();
+
+                               // Append the result to the enclosing accumulator
+                               if ( $isTreeNode ) {
+                                       $accum->addNode( $tree );
+                               } else {
+                                       $accum->addAccum( $resultAccum );
+                               }
+                               // Note that we do NOT increment the input pointer.
+                               // This is because the closing linebreak could be the opening linebreak of
+                               // another heading. Infinite loops are avoided because the next iteration MUST
+                               // hit the heading open case above, which unconditionally increments the
+                               // input pointer.
+                       } elseif ( $found === 'open' ) {
+                               # count opening brace characters
+                               $count = intval( strspn( $text, $curChar, $i ) );
+
+                               # we need to add to stack only if opening brace count is enough for one of the rules
+                               if ( $count >= $rule['min'] ) {
+                                       # Add it to the stack
+                                       $partData = array(
+                                               'open' => $curChar,
+                                               'close' => $rule['end'],
+                                               'count' => $count,
+                                               'lineStart' => ($i == 0 || $text[$i-1] === "\n"),
+                                       );
+
+                                       $stack->push( $partData );
+                                       $accum = $stack->getAccum();
+                                       $stackFlags = $stack->getFlags();
+                               } else {
+                                       # Add literal brace(s)
+                                       $accum->addLiteral( str_repeat( $curChar, $count ) );
+                               }
+                               $i += $count;
+                       } elseif ( $found === 'close' ) {
+                               $piece = $stack->getTop();
+                               # lets check if there are enough characters for closing brace
+                               $maxCount = intval( $piece->count );
+                               $count = intval( strspn( $text, $curChar, $i, $maxCount ) );
+
+                               # check for maximum matching characters (if there are 5 closing
+                               # characters, we will probably need only 3 - depending on the rules)
+                               $rule = $rules[$piece->open];
+                               if ( $count > $rule['max'] ) {
+                                       # The specified maximum exists in the callback array, unless the caller
+                                       # has made an error
+                                       $matchingCount = intval( $rule['max'] );
+                               } else {
+                                       # Count is less than the maximum
+                                       # Skip any gaps in the callback array to find the true largest match
+                                       # Need to use array_key_exists not isset because the callback can be null
+                                       $matchingCount = $count;
+                                       while ( $matchingCount > 0 && !array_key_exists( $matchingCount, $rule['names'] ) ) {
+                                               --$matchingCount;
+                                       }
+                               }
+
+                               if ($matchingCount <= 0) {
+                                       # No matching element found in callback array
+                                       # Output a literal closing brace and continue
+                                       $accum->addLiteral( str_repeat( $curChar, $count ) );
+                                       $i += $count;
+                                       continue;
+                               }
+                               $name = strval( $rule['names'][$matchingCount] );
+                               $isTreeNode = false;
+                               if ( $name === 'LITERAL' ) {
+                                       // No element, just literal text
+                                       $resultAccum = $piece->breakSyntax( $matchingCount );
+                                       $resultAccum->addLiteral( str_repeat( $rule['end'], $matchingCount ) );
+                               } else {
+                                       # Create XML element
+                                       # Note: $parts is already XML, does not need to be encoded further
+                                       $isTreeNode = true;
+                                       $parts = $piece->parts;
+                                       $titleAccum = PPDAccum_HipHop::cast( $parts[0]->out );
+                                       unset( $parts[0] );
+
+                                       $tree = new PPNode_HipHop_Tree( $name );
+
+                                       # The invocation is at the start of the line if lineStart is set in
+                                       # the stack, and all opening brackets are used up.
+                                       if ( $maxCount == $matchingCount && !empty( $piece->lineStart ) ) {
+                                               $tree->addChild( new PPNode_HipHop_Attr( 'lineStart', 1 ) );
+                                       }
+                                       $titleNode = new PPNode_HipHop_Tree( 'title' );
+                                       $titleNode->firstChild = $titleAccum->firstNode;
+                                       $titleNode->lastChild = $titleAccum->lastNode;
+                                       $tree->addChild( $titleNode );
+                                       $argIndex = 1;
+                                       foreach ( $parts as $variantPart ) {
+                                               $part = PPDPart_HipHop::cast( $variantPart );
+                                               if ( isset( $part->eqpos ) ) {
+                                                       // Find equals
+                                                       $lastNode = false;
+                                                       for ( $node = $part->out->firstNode; $node; $node = $node->nextSibling ) {
+                                                               if ( $node === $part->eqpos ) {
+                                                                       break;
+                                                               }
+                                                               $lastNode = $node;
+                                                       }
+                                                       if ( !$node ) {
+                                                               throw new MWException( __METHOD__. ': eqpos not found' );
+                                                       }
+                                                       if ( $node->name !== 'equals' ) {
+                                                               throw new MWException( __METHOD__ .': eqpos is not equals' );
+                                                       }
+                                                       $equalsNode = $node;
+
+                                                       // Construct name node
+                                                       $nameNode = new PPNode_HipHop_Tree( 'name' );
+                                                       if ( $lastNode !== false ) {
+                                                               $lastNode->nextSibling = false;
+                                                               $nameNode->firstChild = $part->out->firstNode;
+                                                               $nameNode->lastChild = $lastNode;
+                                                       }
+
+                                                       // Construct value node
+                                                       $valueNode = new PPNode_HipHop_Tree( 'value' );
+                                                       if ( $equalsNode->nextSibling !== false ) {
+                                                               $valueNode->firstChild = $equalsNode->nextSibling;
+                                                               $valueNode->lastChild = $part->out->lastNode;
+                                                       }
+                                                       $partNode = new PPNode_HipHop_Tree( 'part' );
+                                                       $partNode->addChild( $nameNode );
+                                                       $partNode->addChild( $equalsNode->firstChild );
+                                                       $partNode->addChild( $valueNode );
+                                                       $tree->addChild( $partNode );
+                                               } else {
+                                                       $partNode = new PPNode_HipHop_Tree( 'part' );
+                                                       $nameNode = new PPNode_HipHop_Tree( 'name' );
+                                                       $nameNode->addChild( new PPNode_HipHop_Attr( 'index', $argIndex++ ) );
+                                                       $valueNode = new PPNode_HipHop_Tree( 'value' );
+                                                       $valueNode->firstChild = $part->out->firstNode;
+                                                       $valueNode->lastChild = $part->out->lastNode;
+                                                       $partNode->addChild( $nameNode );
+                                                       $partNode->addChild( $valueNode );
+                                                       $tree->addChild( $partNode );
+                                               }
+                                       }
+                               }
+
+                               # Advance input pointer
+                               $i += $matchingCount;
+
+                               # Unwind the stack
+                               $stack->pop();
+                               $accum = $stack->getAccum();
+
+                               # Re-add the old stack element if it still has unmatched opening characters remaining
+                               if ($matchingCount < $piece->count) {
+                                       $piece->parts = array( new PPDPart_HipHop );
+                                       $piece->count -= $matchingCount;
+                                       # do we still qualify for any callback with remaining count?
+                                       $names = $rules[$piece->open]['names'];
+                                       $skippedBraces = 0;
+                                       $enclosingAccum = $accum;
+                                       while ( $piece->count ) {
+                                               if ( array_key_exists( $piece->count, $names ) ) {
+                                                       $stack->push( $piece );
+                                                       $accum = $stack->getAccum();
+                                                       break;
+                                               }
+                                               --$piece->count;
+                                               $skippedBraces ++;
+                                       }
+                                       $enclosingAccum->addLiteral( str_repeat( $piece->open, $skippedBraces ) );
+                               }
+
+                               $stackFlags = $stack->getFlags();
+
+                               # Add XML element to the enclosing accumulator
+                               if ( $isTreeNode ) {
+                                       $accum->addNode( $tree );
+                               } else {
+                                       $accum->addAccum( $resultAccum );
+                               }
+                       } elseif ( $found === 'pipe' ) {
+                               $stackFlags['findEquals'] = true; // shortcut for getFlags()
+                               $stack->addPart();
+                               $accum = $stack->getAccum();
+                               ++$i;
+                       } elseif ( $found === 'equals' ) {
+                               $stackFlags['findEquals'] = false; // shortcut for getFlags()
+                               $accum->addNodeWithText( 'equals', '=' );
+                               $stack->getCurrentPart()->eqpos = $accum->lastNode;
+                               ++$i;
+                       }
+               }
+
+               # Output any remaining unclosed brackets
+               foreach ( $stack->stack as $variantPiece ) {
+                       $piece = PPDStackElement_HipHop::cast( $variantPiece );
+                       $stack->rootAccum->addAccum( $piece->breakSyntax() );
+               }
+
+               # Enable top-level headings
+               for ( $node = $stack->rootAccum->firstNode; $node; $node = $node->nextSibling ) {
+                       if ( isset( $node->name ) && $node->name === 'possible-h' ) {
+                               $node->name = 'h';
+                       }
+               }
+
+               $rootNode = new PPNode_HipHop_Tree( 'root' );
+               $rootNode->firstChild = $stack->rootAccum->firstNode;
+               $rootNode->lastChild = $stack->rootAccum->lastNode;
+
+               // Cache
+               if ($cacheable) {
+                       $cacheValue = sprintf( "%08d", self::CACHE_VERSION ) . serialize( $rootNode );
+                       $wgMemc->set( $cacheKey, $cacheValue, 86400 );
+                       wfProfileOut( __METHOD__.'-cache-miss' );
+                       wfProfileOut( __METHOD__.'-cacheable' );
+                       wfDebugLog( "Preprocessor", "Saved preprocessor Hash to memcached (key $cacheKey)" );
+               }
+
+               wfProfileOut( __METHOD__ );
+               return $rootNode;
+       }
+}
+
+
+
+/**
+ * Stack class to help Preprocessor::preprocessToObj()
+ * @ingroup Parser
+ */
+class PPDStack_HipHop {
+       var $stack, $rootAccum;
+
+       /**
+        * @var PPDStack
+        */
+       var $top;
+       var $out;
+
+       static $false = false;
+
+       function __construct() {
+               $this->stack = array();
+               $this->top = false;
+               $this->rootAccum = new PPDAccum_HipHop;
+               $this->accum = $this->rootAccum;
+       }
+
+       /**
+        * @return int
+        */
+       function count() {
+               return count( $this->stack );
+       }
+
+       function getAccum() {
+               return PPDAccum_HipHop::cast( $this->accum );
+       }
+
+       function getCurrentPart() {
+               return $this->getTop()->getCurrentPart();
+       }
+
+       function getTop() {
+               return PPDStackElement_HipHop::cast( $this->top );
+       }
+
+       function push( $data ) {
+               if ( $data instanceof PPDStackElement_HipHop ) {
+                       $this->stack[] = $data;
+               } else {
+                       $this->stack[] = new PPDStackElement_HipHop( $data );
+               }
+               $this->top = $this->stack[ count( $this->stack ) - 1 ];
+               $this->accum = $this->top->getAccum();
+       }
+
+       function pop() {
+               if ( !count( $this->stack ) ) {
+                       throw new MWException( __METHOD__.': no elements remaining' );
+               }
+               $temp = array_pop( $this->stack );
+
+               if ( count( $this->stack ) ) {
+                       $this->top = $this->stack[ count( $this->stack ) - 1 ];
+                       $this->accum = $this->top->getAccum();
+               } else {
+                       $this->top = self::$false;
+                       $this->accum = $this->rootAccum;
+               }
+               return $temp;
+       }
+
+       function addPart( $s = '' ) {
+               $this->top->addPart( $s );
+               $this->accum = $this->top->getAccum();
+       }
+
+       /**
+        * @return array
+        */
+       function getFlags() {
+               if ( !count( $this->stack ) ) {
+                       return array(
+                               'findEquals' => false,
+                               'findPipe' => false,
+                               'inHeading' => false,
+                       );
+               } else {
+                       return $this->top->getFlags();
+               }
+       }
+}
+
+/**
+ * @ingroup Parser
+ */
+class PPDStackElement_HipHop {
+       var $open,                      // Opening character (\n for heading)
+               $close,             // Matching closing character
+               $count,             // Number of opening characters found (number of "=" for heading)
+               $parts,             // Array of PPDPart objects describing pipe-separated parts.
+               $lineStart;         // True if the open char appeared at the start of the input line. Not set for headings.
+
+       static function cast( PPDStackElement_HipHop $obj ) {
+               return $obj;
+       }
+
+       function __construct( $data = array() ) {
+               $this->parts = array( new PPDPart_HipHop );
+
+               foreach ( $data as $name => $value ) {
+                       $this->$name = $value;
+               }
+       }
+
+       function getAccum() {
+               return PPDAccum_HipHop::cast( $this->parts[count($this->parts) - 1]->out );
+       }
+
+       function addPart( $s = '' ) {
+               $this->parts[] = new PPDPart_HipHop( $s );
+       }
+
+       function getCurrentPart() {
+               return PPDPart_HipHop::cast( $this->parts[count($this->parts) - 1] );
+       }
+
+       /**
+        * @return array
+        */
+       function getFlags() {
+               $partCount = count( $this->parts );
+               $findPipe = $this->open !== "\n" && $this->open !== '[';
+               return array(
+                       'findPipe' => $findPipe,
+                       'findEquals' => $findPipe && $partCount > 1 && !isset( $this->parts[$partCount - 1]->eqpos ),
+                       'inHeading' => $this->open === "\n",
+               );
+       }
+
+       /**
+        * Get the accumulator that would result if the close is not found.
+        *
+        * @return PPDAccum_HipHop
+        */
+       function breakSyntax( $openingCount = false ) {
+               if ( $this->open === "\n" ) {
+                       $accum = PPDAccum_HipHop::cast( $this->parts[0]->out );
+               } else {
+                       if ( $openingCount === false ) {
+                               $openingCount = $this->count;
+                       }
+                       $accum = new PPDAccum_HipHop;
+                       $accum->addLiteral( str_repeat( $this->open, $openingCount ) );
+                       $first = true;
+                       foreach ( $this->parts as $part ) {
+                               if ( $first ) {
+                                       $first = false;
+                               } else {
+                                       $accum->addLiteral( '|' );
+                               }
+                               $accum->addAccum( $part->out );
+                       }
+               }
+               return $accum;
+       }
+}
+
+/**
+ * @ingroup Parser
+ */
+class PPDPart_HipHop {
+       var $out; // Output accumulator object
+
+       // Optional member variables:
+       //   eqpos        Position of equals sign in output accumulator
+       //   commentEnd   Past-the-end input pointer for the last comment encountered
+       //   visualEnd    Past-the-end input pointer for the end of the accumulator minus comments
+
+       function __construct( $out = '' ) {
+               $this->out = new PPDAccum_HipHop;
+               if ( $out !== '' ) {
+                       $this->out->addLiteral( $out );
+               }
+       }
+
+       static function cast( PPDPart_HipHop $obj ) {
+               return $obj;
+       }
+}
+
+/**
+ * @ingroup Parser
+ */
+class PPDAccum_HipHop {
+       var $firstNode, $lastNode;
+
+       function __construct() {
+               $this->firstNode = $this->lastNode = false;
+       }
+
+       static function cast( PPDAccum_HipHop $obj ) {
+               return $obj;
+       }
+
+       /**
+        * Append a string literal
+        */
+       function addLiteral( string $s ) {
+               if ( $this->lastNode === false ) {
+                       $this->firstNode = $this->lastNode = new PPNode_HipHop_Text( $s );
+               } elseif ( $this->lastNode instanceof PPNode_HipHop_Text ) {
+                       $this->lastNode->value .= $s;
+               } else {
+                       $this->lastNode->nextSibling = new PPNode_HipHop_Text( $s );
+                       $this->lastNode = $this->lastNode->nextSibling;
+               }
+       }
+
+       /**
+        * Append a PPNode
+        */
+       function addNode( PPNode $node ) {
+               if ( $this->lastNode === false ) {
+                       $this->firstNode = $this->lastNode = $node;
+               } else {
+                       $this->lastNode->nextSibling = $node;
+                       $this->lastNode = $node;
+               }
+       }
+
+       /**
+        * Append a tree node with text contents
+        */
+       function addNodeWithText( string $name, string $value ) {
+               $node = PPNode_HipHop_Tree::newWithText( $name, $value );
+               $this->addNode( $node );
+       }
+
+       /**
+        * Append a PPDAccum_HipHop
+        * Takes over ownership of the nodes in the source argument. These nodes may
+        * subsequently be modified, especially nextSibling.
+        */
+       function addAccum( PPDAccum_HipHop $accum ) {
+               if ( $accum->lastNode === false ) {
+                       // nothing to add
+               } elseif ( $this->lastNode === false ) {
+                       $this->firstNode = $accum->firstNode;
+                       $this->lastNode = $accum->lastNode;
+               } else {
+                       $this->lastNode->nextSibling = $accum->firstNode;
+                       $this->lastNode = $accum->lastNode;
+               }
+       }
+}
+
+/**
+ * An expansion frame, used as a context to expand the result of preprocessToObj()
+ * @ingroup Parser
+ */
+class PPFrame_HipHop implements PPFrame {
+
+       /**
+        * @var Parser
+        */
+       var $parser;
+
+       /**
+        * @var Preprocessor
+        */
+       var $preprocessor;
+
+       /**
+        * @var Title
+        */
+       var $title;
+       var $titleCache;
+
+       /**
+        * Hashtable listing templates which are disallowed for expansion in this frame,
+        * having been encountered previously in parent frames.
+        */
+       var $loopCheckHash;
+
+       /**
+        * Recursion depth of this frame, top = 0
+        * Note that this is NOT the same as expansion depth in expand()
+        */
+       var $depth;
+
+
+       /**
+        * Construct a new preprocessor frame.
+        * @param $preprocessor Preprocessor: the parent preprocessor
+        */
+       function __construct( $preprocessor ) {
+               $this->preprocessor = $preprocessor;
+               $this->parser = $preprocessor->parser;
+               $this->title = $this->parser->mTitle;
+               $this->titleCache = array( $this->title ? $this->title->getPrefixedDBkey() : false );
+               $this->loopCheckHash = array();
+               $this->depth = 0;
+       }
+
+       /**
+        * Create a new child frame
+        * $args is optionally a multi-root PPNode or array containing the template arguments
+        *
+        * @param $args PPNode_HipHop_Array|array
+        * @param $title Title|false
+        *
+        * @return PPTemplateFrame_HipHop
+        */
+       function newChild( $args = false, $title = false ) {
+               $namedArgs = array();
+               $numberedArgs = array();
+               if ( $title === false ) {
+                       $title = $this->title;
+               }
+               if ( $args !== false ) {
+                       if ( $args instanceof PPNode_HipHop_Array ) {
+                               $args = $args->value;
+                       } elseif ( !is_array( $args ) ) {
+                               throw new MWException( __METHOD__ . ': $args must be array or PPNode_HipHop_Array' );
+                       }
+                       foreach ( $args as $arg ) {
+                               $bits = $arg->splitArg();
+                               if ( $bits['index'] !== '' ) {
+                                       // Numbered parameter
+                                       $numberedArgs[$bits['index']] = $bits['value'];
+                                       unset( $namedArgs[$bits['index']] );
+                               } else {
+                                       // Named parameter
+                                       $name = trim( $this->expand( $bits['name'], PPFrame::STRIP_COMMENTS ) );
+                                       $namedArgs[$name] = $bits['value'];
+                                       unset( $numberedArgs[$name] );
+                               }
+                       }
+               }
+               return new PPTemplateFrame_HipHop( $this->preprocessor, $this, $numberedArgs, $namedArgs, $title );
+       }
+
+       /**
+        * @throws MWException
+        * @param $root
+        * @param $flags int
+        * @return string
+        */
+       function expand( $root, $flags = 0 ) {
+               static $expansionDepth = 0;
+               if ( is_string( $root ) ) {
+                       return $root;
+               }
+
+               if ( ++$this->parser->mPPNodeCount > $this->parser->mOptions->getMaxPPNodeCount() ) {
+                       return '<span class="error">Node-count limit exceeded</span>';
+               }
+               if ( $expansionDepth > $this->parser->mOptions->getMaxPPExpandDepth() ) {
+                       return '<span class="error">Expansion depth limit exceeded</span>';
+               }
+               ++$expansionDepth;
+
+               $outStack = array( '', '' );
+               $iteratorStack = array( false, $root );
+               $indexStack = array( 0, 0 );
+
+               while ( count( $iteratorStack ) > 1 ) {
+                       $level = count( $outStack ) - 1;
+                       $iteratorNode =& $iteratorStack[ $level ];
+                       $out =& $outStack[$level];
+                       $index =& $indexStack[$level];
+
+                       if ( is_array( $iteratorNode ) ) {
+                               if ( $index >= count( $iteratorNode ) ) {
+                                       // All done with this iterator
+                                       $iteratorStack[$level] = false;
+                                       $contextNode = false;
+                               } else {
+                                       $contextNode = $iteratorNode[$index];
+                                       $index++;
+                               }
+                       } elseif ( $iteratorNode instanceof PPNode_HipHop_Array ) {
+                               if ( $index >= $iteratorNode->getLength() ) {
+                                       // All done with this iterator
+                                       $iteratorStack[$level] = false;
+                                       $contextNode = false;
+                               } else {
+                                       $contextNode = $iteratorNode->item( $index );
+                                       $index++;
+                               }
+                       } else {
+                               // Copy to $contextNode and then delete from iterator stack,
+                               // because this is not an iterator but we do have to execute it once
+                               $contextNode = $iteratorStack[$level];
+                               $iteratorStack[$level] = false;
+                       }
+
+                       $newIterator = false;
+
+                       if ( $contextNode === false ) {
+                               // nothing to do
+                       } elseif ( is_string( $contextNode ) ) {
+                               $out .= $contextNode;
+                       } elseif ( is_array( $contextNode ) || $contextNode instanceof PPNode_HipHop_Array ) {
+                               $newIterator = $contextNode;
+                       } elseif ( $contextNode instanceof PPNode_HipHop_Attr ) {
+                               // No output
+                       } elseif ( $contextNode instanceof PPNode_HipHop_Text ) {
+                               $out .= $contextNode->value;
+                       } elseif ( $contextNode instanceof PPNode_HipHop_Tree ) {
+                               if ( $contextNode->name === 'template' ) {
+                                       # Double-brace expansion
+                                       $bits = $contextNode->splitTemplate();
+                                       if ( $flags & PPFrame::NO_TEMPLATES ) {
+                                               $newIterator = $this->virtualBracketedImplode( '{{', '|', '}}', $bits['title'], $bits['parts'] );
+                                       } else {
+                                               $ret = $this->parser->braceSubstitution( $bits, $this );
+                                               if ( isset( $ret['object'] ) ) {
+                                                       $newIterator = $ret['object'];
+                                               } else {
+                                                       $out .= $ret['text'];
+                                               }
+                                       }
+                               } elseif ( $contextNode->name === 'tplarg' ) {
+                                       # Triple-brace expansion
+                                       $bits = $contextNode->splitTemplate();
+                                       if ( $flags & PPFrame::NO_ARGS ) {
+                                               $newIterator = $this->virtualBracketedImplode( '{{{', '|', '}}}', $bits['title'], $bits['parts'] );
+                                       } else {
+                                               $ret = $this->parser->argSubstitution( $bits, $this );
+                                               if ( isset( $ret['object'] ) ) {
+                                                       $newIterator = $ret['object'];
+                                               } else {
+                                                       $out .= $ret['text'];
+                                               }
+                                       }
+                               } elseif ( $contextNode->name === 'comment' ) {
+                                       # HTML-style comment
+                                       # Remove it in HTML, pre+remove and STRIP_COMMENTS modes
+                                       if ( $this->parser->ot['html']
+                                               || ( $this->parser->ot['pre'] && $this->parser->mOptions->getRemoveComments() )
+                                               || ( $flags & PPFrame::STRIP_COMMENTS ) )
+                                       {
+                                               $out .= '';
+                                       }
+                                       # Add a strip marker in PST mode so that pstPass2() can run some old-fashioned regexes on the result
+                                       # Not in RECOVER_COMMENTS mode (extractSections) though
+                                       elseif ( $this->parser->ot['wiki'] && ! ( $flags & PPFrame::RECOVER_COMMENTS ) ) {
+                                               $out .= $this->parser->insertStripItem( $contextNode->firstChild->value );
+                                       }
+                                       # Recover the literal comment in RECOVER_COMMENTS and pre+no-remove
+                                       else {
+                                               $out .= $contextNode->firstChild->value;
+                                       }
+                               } elseif ( $contextNode->name === 'ignore' ) {
+                                       # Output suppression used by <includeonly> etc.
+                                       # OT_WIKI will only respect <ignore> in substed templates.
+                                       # The other output types respect it unless NO_IGNORE is set.
+                                       # extractSections() sets NO_IGNORE and so never respects it.
+                                       if ( ( !isset( $this->parent ) && $this->parser->ot['wiki'] ) || ( $flags & PPFrame::NO_IGNORE ) ) {
+                                               $out .= $contextNode->firstChild->value;
+                                       } else {
+                                               //$out .= '';
+                                       }
+                               } elseif ( $contextNode->name === 'ext' ) {
+                                       # Extension tag
+                                       $bits = $contextNode->splitExt() + array( 'attr' => null, 'inner' => null, 'close' => null );
+                                       $out .= $this->parser->extensionSubstitution( $bits, $this );
+                               } elseif ( $contextNode->name === 'h' ) {
+                                       # Heading
+                                       if ( $this->parser->ot['html'] ) {
+                                               # Expand immediately and insert heading index marker
+                                               $s = '';
+                                               for ( $node = $contextNode->firstChild; $node; $node = $node->nextSibling ) {
+                                                       $s .= $this->expand( $node, $flags );
+                                               }
+
+                                               $bits = $contextNode->splitHeading();
+                                               $titleText = $this->title->getPrefixedDBkey();
+                                               $this->parser->mHeadings[] = array( $titleText, $bits['i'] );
+                                               $serial = count( $this->parser->mHeadings ) - 1;
+                                               $marker = "{$this->parser->mUniqPrefix}-h-$serial-" . Parser::MARKER_SUFFIX;
+                                               $s = substr( $s, 0, $bits['level'] ) . $marker . substr( $s, $bits['level'] );
+                                               $this->parser->mStripState->addGeneral( $marker, '' );
+                                               $out .= $s;
+                                       } else {
+                                               # Expand in virtual stack
+                                               $newIterator = $contextNode->getChildren();
+                                       }
+                               } else {
+                                       # Generic recursive expansion
+                                       $newIterator = $contextNode->getChildren();
+                               }
+                       } else {
+                               throw new MWException( __METHOD__.': Invalid parameter type' );
+                       }
+
+                       if ( $newIterator !== false ) {
+                               $outStack[] = '';
+                               $iteratorStack[] = $newIterator;
+                               $indexStack[] = 0;
+                       } elseif ( $iteratorStack[$level] === false ) {
+                               // Return accumulated value to parent
+                               // With tail recursion
+                               while ( $iteratorStack[$level] === false && $level > 0 ) {
+                                       $outStack[$level - 1] .= $out;
+                                       array_pop( $outStack );
+                                       array_pop( $iteratorStack );
+                                       array_pop( $indexStack );
+                                       $level--;
+                               }
+                       }
+               }
+               --$expansionDepth;
+               return $outStack[0];
+       }
+
+       /**
+        * @param $sep
+        * @param $flags
+        * @return string
+        */
+       function implodeWithFlags( $sep, $flags /*, ... */ ) {
+               $args = array_slice( func_get_args(), 2 );
+
+               $first = true;
+               $s = '';
+               foreach ( $args as $root ) {
+                       if ( $root instanceof PPNode_HipHop_Array ) {
+                               $root = $root->value;
+                       }
+                       if ( !is_array( $root ) ) {
+                               $root = array( $root );
+                       }
+                       foreach ( $root as $node ) {
+                               if ( $first ) {
+                                       $first = false;
+                               } else {
+                                       $s .= $sep;
+                               }
+                               $s .= $this->expand( $node, $flags );
+                       }
+               }
+               return $s;
+       }
+
+       /**
+        * Implode with no flags specified
+        * This previously called implodeWithFlags but has now been inlined to reduce stack depth
+        * @return string
+        */
+       function implode( $sep /*, ... */ ) {
+               $args = array_slice( func_get_args(), 1 );
+
+               $first = true;
+               $s = '';
+               foreach ( $args as $root ) {
+                       if ( $root instanceof PPNode_HipHop_Array ) {
+                               $root = $root->value;
+                       }
+                       if ( !is_array( $root ) ) {
+                               $root = array( $root );
+                       }
+                       foreach ( $root as $node ) {
+                               if ( $first ) {
+                                       $first = false;
+                               } else {
+                                       $s .= $sep;
+                               }
+                               $s .= $this->expand( $node );
+                       }
+               }
+               return $s;
+       }
+
+       /**
+        * Makes an object that, when expand()ed, will be the same as one obtained
+        * with implode()
+        *
+        * @return PPNode_HipHop_Array
+        */
+       function virtualImplode( $sep /*, ... */ ) {
+               $args = array_slice( func_get_args(), 1 );
+               $out = array();
+               $first = true;
+
+               foreach ( $args as $root ) {
+                       if ( $root instanceof PPNode_HipHop_Array ) {
+                               $root = $root->value;
+                       }
+                       if ( !is_array( $root ) ) {
+                               $root = array( $root );
+                       }
+                       foreach ( $root as $node ) {
+                               if ( $first ) {
+                                       $first = false;
+                               } else {
+                                       $out[] = $sep;
+                               }
+                               $out[] = $node;
+                       }
+               }
+               return new PPNode_HipHop_Array( $out );
+       }
+
+       /**
+        * Virtual implode with brackets
+        *
+        * @return PPNode_HipHop_Array
+        */
+       function virtualBracketedImplode( $start, $sep, $end /*, ... */ ) {
+               $args = array_slice( func_get_args(), 3 );
+               $out = array( $start );
+               $first = true;
+
+               foreach ( $args as $root ) {
+                       if ( $root instanceof PPNode_HipHop_Array ) {
+                               $root = $root->value;
+                       }
+                       if ( !is_array( $root ) ) {
+                               $root = array( $root );
+                       }
+                       foreach ( $root as $node ) {
+                               if ( $first ) {
+                                       $first = false;
+                               } else {
+                                       $out[] = $sep;
+                               }
+                               $out[] = $node;
+                       }
+               }
+               $out[] = $end;
+               return new PPNode_HipHop_Array( $out );
+       }
+
+       function __toString() {
+               return 'frame{}';
+       }
+
+       /**
+        * @param $level bool
+        * @return array|bool|String
+        */
+       function getPDBK( $level = false ) {
+               if ( $level === false ) {
+                       return $this->title->getPrefixedDBkey();
+               } else {
+                       return isset( $this->titleCache[$level] ) ? $this->titleCache[$level] : false;
+               }
+       }
+
+       /**
+        * @return array
+        */
+       function getArguments() {
+               return array();
+       }
+
+       /**
+        * @return array
+        */
+       function getNumberedArguments() {
+               return array();
+       }
+
+       /**
+        * @return array
+        */
+       function getNamedArguments() {
+               return array();
+       }
+
+       /**
+        * Returns true if there are no arguments in this frame
+        *
+        * @return bool
+        */
+       function isEmpty() {
+               return true;
+       }
+
+       /**
+        * @param $name
+        * @return bool
+        */
+       function getArgument( $name ) {
+               return false;
+       }
+
+       /**
+        * Returns true if the infinite loop check is OK, false if a loop is detected
+        *
+        * @param $title Title
+        *
+        * @return bool
+        */
+       function loopCheck( $title ) {
+               return !isset( $this->loopCheckHash[$title->getPrefixedDBkey()] );
+       }
+
+       /**
+        * Return true if the frame is a template frame
+        *
+        * @return bool
+        */
+       function isTemplate() {
+               return false;
+       }
+}
+
+/**
+ * Expansion frame with template arguments
+ * @ingroup Parser
+ */
+class PPTemplateFrame_HipHop extends PPFrame_HipHop {
+       var $numberedArgs, $namedArgs, $parent;
+       var $numberedExpansionCache, $namedExpansionCache;
+
+       /**
+        * @param $preprocessor
+        * @param $parent
+        * @param $numberedArgs array
+        * @param $namedArgs array
+        * @param $title Title
+        */
+       function __construct( $preprocessor, $parent = false, $numberedArgs = array(), $namedArgs = array(), $title = false ) {
+               parent::__construct( $preprocessor );
+
+               $this->parent = $parent;
+               $this->numberedArgs = $numberedArgs;
+               $this->namedArgs = $namedArgs;
+               $this->title = $title;
+               $pdbk = $title ? $title->getPrefixedDBkey() : false;
+               $this->titleCache = $parent->titleCache;
+               $this->titleCache[] = $pdbk;
+               $this->loopCheckHash = /*clone*/ $parent->loopCheckHash;
+               if ( $pdbk !== false ) {
+                       $this->loopCheckHash[$pdbk] = true;
+               }
+               $this->depth = $parent->depth + 1;
+               $this->numberedExpansionCache = $this->namedExpansionCache = array();
+       }
+
+       function __toString() {
+               $s = 'tplframe{';
+               $first = true;
+               $args = $this->numberedArgs + $this->namedArgs;
+               foreach ( $args as $name => $value ) {
+                       if ( $first ) {
+                               $first = false;
+                       } else {
+                               $s .= ', ';
+                       }
+                       $s .= "\"$name\":\"" .
+                               str_replace( '"', '\\"', $value->__toString() ) . '"';
+               }
+               $s .= '}';
+               return $s;
+       }
+       /**
+        * Returns true if there are no arguments in this frame
+        *
+        * @return bool
+        */
+       function isEmpty() {
+               return !count( $this->numberedArgs ) && !count( $this->namedArgs );
+       }
+
+       /**
+        * @return array
+        */
+       function getArguments() {
+               $arguments = array();
+               foreach ( array_merge(
+                               array_keys($this->numberedArgs),
+                               array_keys($this->namedArgs)) as $key ) {
+                       $arguments[$key] = $this->getArgument($key);
+               }
+               return $arguments;
+       }
+
+       /**
+        * @return array
+        */
+       function getNumberedArguments() {
+               $arguments = array();
+               foreach ( array_keys($this->numberedArgs) as $key ) {
+                       $arguments[$key] = $this->getArgument($key);
+               }
+               return $arguments;
+       }
+
+       /**
+        * @return array
+        */
+       function getNamedArguments() {
+               $arguments = array();
+               foreach ( array_keys($this->namedArgs) as $key ) {
+                       $arguments[$key] = $this->getArgument($key);
+               }
+               return $arguments;
+       }
+
+       /**
+        * @param $index
+        * @return array|bool
+        */
+       function getNumberedArgument( $index ) {
+               if ( !isset( $this->numberedArgs[$index] ) ) {
+                       return false;
+               }
+               if ( !isset( $this->numberedExpansionCache[$index] ) ) {
+                       # No trimming for unnamed arguments
+                       $this->numberedExpansionCache[$index] = $this->parent->expand( $this->numberedArgs[$index], PPFrame::STRIP_COMMENTS );
+               }
+               return $this->numberedExpansionCache[$index];
+       }
+
+       /**
+        * @param $name
+        * @return bool
+        */
+       function getNamedArgument( $name ) {
+               if ( !isset( $this->namedArgs[$name] ) ) {
+                       return false;
+               }
+               if ( !isset( $this->namedExpansionCache[$name] ) ) {
+                       # Trim named arguments post-expand, for backwards compatibility
+                       $this->namedExpansionCache[$name] = trim(
+                               $this->parent->expand( $this->namedArgs[$name], PPFrame::STRIP_COMMENTS ) );
+               }
+               return $this->namedExpansionCache[$name];
+       }
+
+       /**
+        * @param $name
+        * @return array|bool
+        */
+       function getArgument( $name ) {
+               $text = $this->getNumberedArgument( $name );
+               if ( $text === false ) {
+                       $text = $this->getNamedArgument( $name );
+               }
+               return $text;
+       }
+
+       /**
+        * Return true if the frame is a template frame
+        *
+        * @return bool
+        */
+       function isTemplate() {
+               return true;
+       }
+}
+
+/**
+ * Expansion frame with custom arguments
+ * @ingroup Parser
+ */
+class PPCustomFrame_HipHop extends PPFrame_HipHop {
+       var $args;
+
+       function __construct( $preprocessor, $args ) {
+               parent::__construct( $preprocessor );
+               $this->args = $args;
+       }
+
+       function __toString() {
+               $s = 'cstmframe{';
+               $first = true;
+               foreach ( $this->args as $name => $value ) {
+                       if ( $first ) {
+                               $first = false;
+                       } else {
+                               $s .= ', ';
+                       }
+                       $s .= "\"$name\":\"" .
+                               str_replace( '"', '\\"', $value->__toString() ) . '"';
+               }
+               $s .= '}';
+               return $s;
+       }
+
+       /**
+        * @return bool
+        */
+       function isEmpty() {
+               return !count( $this->args );
+       }
+
+       /**
+        * @param $index
+        * @return bool
+        */
+       function getArgument( $index ) {
+               if ( !isset( $this->args[$index] ) ) {
+                       return false;
+               }
+               return $this->args[$index];
+       }
+}
+
+/**
+ * @ingroup Parser
+ */
+class PPNode_HipHop_Tree implements PPNode {
+       var $name, $firstChild, $lastChild, $nextSibling;
+
+       function __construct( $name ) {
+               $this->name = $name;
+               $this->firstChild = $this->lastChild = $this->nextSibling = false;
+       }
+
+       function __toString() {
+               $inner = '';
+               $attribs = '';
+               for ( $node = $this->firstChild; $node; $node = $node->nextSibling ) {
+                       if ( $node instanceof PPNode_HipHop_Attr ) {
+                               $attribs .= ' ' . $node->name . '="' . htmlspecialchars( $node->value ) . '"';
+                       } else {
+                               $inner .= $node->__toString();
+                       }
+               }
+               if ( $inner === '' ) {
+                       return "<{$this->name}$attribs/>";
+               } else {
+                       return "<{$this->name}$attribs>$inner</{$this->name}>";
+               }
+       }
+
+       /**
+        * @param $name
+        * @param $text
+        * @return PPNode_HipHop_Tree
+        */
+       static function newWithText( $name, $text ) {
+               $obj = new self( $name );
+               $obj->addChild( new PPNode_HipHop_Text( $text ) );
+               return $obj;
+       }
+
+       function addChild( $node ) {
+               if ( $this->lastChild === false ) {
+                       $this->firstChild = $this->lastChild = $node;
+               } else {
+                       $this->lastChild->nextSibling = $node;
+                       $this->lastChild = $node;
+               }
+       }
+
+       /**
+        * @return PPNode_HipHop_Array
+        */
+       function getChildren() {
+               $children = array();
+               for ( $child = $this->firstChild; $child; $child = $child->nextSibling ) {
+                       $children[] = $child;
+               }
+               return new PPNode_HipHop_Array( $children );
+       }
+
+       function getFirstChild() {
+               return $this->firstChild;
+       }
+
+       function getNextSibling() {
+               return $this->nextSibling;
+       }
+
+       function getChildrenOfType( $name ) {
+               $children = array();
+               for ( $child = $this->firstChild; $child; $child = $child->nextSibling ) {
+                       if ( isset( $child->name ) && $child->name === $name ) {
+                               $children[] = $name;
+                       }
+               }
+               return $children;
+       }
+
+       /**
+        * @return bool
+        */
+       function getLength() {
+               return false;
+       }
+
+       /**
+        * @param  $i
+        * @return bool
+        */
+       function item( $i ) {
+               return false;
+       }
+
+       /**
+        * @return string
+        */
+       function getName() {
+               return $this->name;
+       }
+
+       /**
+        * Split a <part> node into an associative array containing:
+        *    name          PPNode name
+        *    index         String index
+        *    value         PPNode value
+        *
+        * @return array
+        */
+       function splitArg() {
+               $bits = array();
+               for ( $child = $this->firstChild; $child; $child = $child->nextSibling ) {
+                       if ( !isset( $child->name ) ) {
+                               continue;
+                       }
+                       if ( $child->name === 'name' ) {
+                               $bits['name'] = $child;
+                               if ( $child->firstChild instanceof PPNode_HipHop_Attr
+                                       && $child->firstChild->name === 'index' )
+                               {
+                                       $bits['index'] = $child->firstChild->value;
+                               }
+                       } elseif ( $child->name === 'value' ) {
+                               $bits['value'] = $child;
+                       }
+               }
+
+               if ( !isset( $bits['name'] ) ) {
+                       throw new MWException( 'Invalid brace node passed to ' . __METHOD__ );
+               }
+               if ( !isset( $bits['index'] ) ) {
+                       $bits['index'] = '';
+               }
+               return $bits;
+       }
+
+       /**
+        * Split an <ext> node into an associative array containing name, attr, inner and close
+        * All values in the resulting array are PPNodes. Inner and close are optional.
+        *
+        * @return array
+        */
+       function splitExt() {
+               $bits = array();
+               for ( $child = $this->firstChild; $child; $child = $child->nextSibling ) {
+                       if ( !isset( $child->name ) ) {
+                               continue;
+                       }
+                       if ( $child->name === 'name' ) {
+                               $bits['name'] = $child;
+                       } elseif ( $child->name === 'attr' ) {
+                               $bits['attr'] = $child;
+                       } elseif ( $child->name === 'inner' ) {
+                               $bits['inner'] = $child;
+                       } elseif ( $child->name === 'close' ) {
+                               $bits['close'] = $child;
+                       }
+               }
+               if ( !isset( $bits['name'] ) ) {
+                       throw new MWException( 'Invalid ext node passed to ' . __METHOD__ );
+               }
+               return $bits;
+       }
+
+       /**
+        * Split an <h> node
+        *
+        * @return array
+        */
+       function splitHeading() {
+               if ( $this->name !== 'h' ) {
+                       throw new MWException( 'Invalid h node passed to ' . __METHOD__ );
+               }
+               $bits = array();
+               for ( $child = $this->firstChild; $child; $child = $child->nextSibling ) {
+                       if ( !isset( $child->name ) ) {
+                               continue;
+                       }
+                       if ( $child->name === 'i' ) {
+                               $bits['i'] = $child->value;
+                       } elseif ( $child->name === 'level' ) {
+                               $bits['level'] = $child->value;
+                       }
+               }
+               if ( !isset( $bits['i'] ) ) {
+                       throw new MWException( 'Invalid h node passed to ' . __METHOD__ );
+               }
+               return $bits;
+       }
+
+       /**
+        * Split a <template> or <tplarg> node
+        *
+        * @return array
+        */
+       function splitTemplate() {
+               $parts = array();
+               $bits = array( 'lineStart' => '' );
+               for ( $child = $this->firstChild; $child; $child = $child->nextSibling ) {
+                       if ( !isset( $child->name ) ) {
+                               continue;
+                       }
+                       if ( $child->name === 'title' ) {
+                               $bits['title'] = $child;
+                       }
+                       if ( $child->name === 'part' ) {
+                               $parts[] = $child;
+                       }
+                       if ( $child->name === 'lineStart' ) {
+                               $bits['lineStart'] = '1';
+                       }
+               }
+               if ( !isset( $bits['title'] ) ) {
+                       throw new MWException( 'Invalid node passed to ' . __METHOD__ );
+               }
+               $bits['parts'] = new PPNode_HipHop_Array( $parts );
+               return $bits;
+       }
+}
+
+/**
+ * @ingroup Parser
+ */
+class PPNode_HipHop_Text implements PPNode {
+       var $value, $nextSibling;
+
+       function __construct( $value ) {
+               if ( is_object( $value ) ) {
+                       throw new MWException( __CLASS__ . ' given object instead of string' );
+               }
+               $this->value = $value;
+       }
+
+       function __toString() {
+               return htmlspecialchars( $this->value );
+       }
+
+       function getNextSibling() {
+               return $this->nextSibling;
+       }
+
+       function getChildren() { return false; }
+       function getFirstChild() { return false; }
+       function getChildrenOfType( $name ) { return false; }
+       function getLength() { return false; }
+       function item( $i ) { return false; }
+       function getName() { return '#text'; }
+       function splitArg() { throw new MWException( __METHOD__ . ': not supported' ); }
+       function splitExt() { throw new MWException( __METHOD__ . ': not supported' ); }
+       function splitHeading() { throw new MWException( __METHOD__ . ': not supported' ); }
+}
+
+/**
+ * @ingroup Parser
+ */
+class PPNode_HipHop_Array implements PPNode {
+       var $value, $nextSibling;
+
+       function __construct( $value ) {
+               $this->value = $value;
+       }
+
+       function __toString() {
+               return var_export( $this, true );
+       }
+
+       function getLength() {
+               return count( $this->value );
+       }
+
+       function item( $i ) {
+               return $this->value[$i];
+       }
+
+       function getName() { return '#nodelist'; }
+
+       function getNextSibling() {
+               return $this->nextSibling;
+       }
+
+       function getChildren() { return false; }
+       function getFirstChild() { return false; }
+       function getChildrenOfType( $name ) { return false; }
+       function splitArg() { throw new MWException( __METHOD__ . ': not supported' ); }
+       function splitExt() { throw new MWException( __METHOD__ . ': not supported' ); }
+       function splitHeading() { throw new MWException( __METHOD__ . ': not supported' ); }
+}
+
+/**
+ * @ingroup Parser
+ */
+class PPNode_HipHop_Attr implements PPNode {
+       var $name, $value, $nextSibling;
+
+       function __construct( $name, $value ) {
+               $this->name = $name;
+               $this->value = $value;
+       }
+
+       function __toString() {
+               return "<@{$this->name}>" . htmlspecialchars( $this->value ) . "</@{$this->name}>";
+       }
+
+       function getName() {
+               return $this->name;
+       }
+
+       function getNextSibling() {
+               return $this->nextSibling;
+       }
+
+       function getChildren() { return false; }
+       function getFirstChild() { return false; }
+       function getChildrenOfType( $name ) { return false; }
+       function getLength() { return false; }
+       function item( $i ) { return false; }
+       function splitArg() { throw new MWException( __METHOD__ . ': not supported' ); }
+       function splitExt() { throw new MWException( __METHOD__ . ': not supported' ); }
+       function splitHeading() { throw new MWException( __METHOD__ . ': not supported' ); }
+}
index 122b7b5..3e01640 100644 (file)
@@ -1,3 +1,5 @@
 GenerateSourceInfo = true
 EnableEval = 2
 AllDynamic = true
+EnableHipHopSyntax = true
+EnableHipHopExperimentalSyntax = true