comment out entries that are the same as in en, use wfMsgNoDb( )
[lhc/web/wiklou.git] / includes / Parser.php
index f7c602a..79aa4e7 100644 (file)
@@ -1,9 +1,12 @@
 <?php
 
-include_once('Tokenizer.php');
+require_once('Tokenizer.php');
 
 if( $GLOBALS['wgUseWikiHiero'] ){
-       include_once('wikihiero.php');
+       require_once('extensions/wikihiero/wikihiero.php');
+}
+if( $GLOBALS['wgUseTimeline'] ){
+       require_once('extensions/timeline/Timeline.php');
 }
 
 # PHP Parser
@@ -41,6 +44,12 @@ define( "OT_HTML", 1 );
 define( "OT_WIKI", 2 );
 define( "OT_MSG", 3 );
 
+# string parameter for extractTags which will cause it
+# to strip HTML comments in addition to regular
+# <XML>-style tags. This should not be anything we
+# may want to use in wikisyntax
+define( "STRIP_COMMENTS", "HTMLCommentStrip" );
+
 # prefix for escaping, used in two functions at least
 define( "UNIQ_PREFIX", "NaodW29");
 
@@ -77,6 +86,7 @@ class Parser
        #
        function parse( $text, &$title, $options, $linestart = true, $clearState = true )
        {
+               global $wgUseTidy;
                $fname = "Parser::parse";
                wfProfileIn( $fname );
 
@@ -93,20 +103,29 @@ class Parser
                $text = $this->internalParse( $text, $linestart );
                $text = $this->unstrip( $text, $this->mStripState );
                # Clean up special characters, only run once, next-to-last before doBlockLevels
-               $fixtags = array(
-                       "/<hr *>/i" => '<hr/>',
-                       "/<br *>/i" => '<br/>',
-                       "/<center *>/i"=>'<div class="center">',
-                       "/<\\/center *>/i" => '</div>',
-                       # Clean up spare ampersands; note that we probably ought to be
-                       # more careful about named entities.
-                       '/&(?!:amp;|#[Xx][0-9A-fa-f]+;|#[0-9]+;|[a-zA-Z0-9]+;)/' => '&amp;'
-               );
-               $text = preg_replace( array_keys($fixtags), array_values($fixtags), $text );
-
+               if(!$wgUseTidy) {
+                       $fixtags = array(
+                               "/<hr *>/i" => '<hr/>',
+                               "/<br *>/i" => '<br/>',
+                               "/<center *>/i"=>'<div class="center">',
+                               "/<\\/center *>/i" => '</div>',
+                               # Clean up spare ampersands; note that we probably ought to be
+                               # more careful about named entities.
+                               '/&(?!:amp;|#[Xx][0-9A-fa-f]+;|#[0-9]+;|[a-zA-Z0-9]+;)/' => '&amp;'
+                       );
+                       $text = preg_replace( array_keys($fixtags), array_values($fixtags), $text );
+               } else {
+                       $fixtags = array(
+                               "/<center *>/i"=>'<div class="center">',
+                               "/<\\/center *>/i" => '</div>'
+                       );
+                       $text = preg_replace( array_keys($fixtags), array_values($fixtags), $text );
+               }
                # only once and last
                $text = $this->doBlockLevels( $text, $linestart );
-
+               if($wgUseTidy) {
+                       $text = $this->tidy($text);
+               }
                $this->mOutput->setText( $text );
                wfProfileOut( $fname );
                return $this->mOutput;
@@ -124,6 +143,9 @@ class Parser
 
        # If $content is already set, the additional entries will be appended
 
+       # If $tag is set to STRIP_COMMENTS, the function will extract
+       # <!-- HTML comments -->
+
        /* static */ function extractTags($tag, $text, &$content, $uniq_prefix = ""){
                $rnd = $uniq_prefix . '-' . $tag . Parser::getRandomString();
                if ( !$content ) {
@@ -133,12 +155,20 @@ class Parser
                $stripped = "";
 
                while ( "" != $text ) {
-                       $p = preg_split( "/<\\s*$tag\\s*>/i", $text, 2 );
+                       if($tag==STRIP_COMMENTS) {
+                               $p = preg_split( "/<!--/i", $text, 2 );
+                       } else {
+                               $p = preg_split( "/<\\s*$tag\\s*>/i", $text, 2 );
+                       }
                        $stripped .= $p[0];
                        if ( ( count( $p ) < 2 ) || ( "" == $p[1] ) ) {
                                $text = "";
                        } else {
-                               $q = preg_split( "/<\\/\\s*$tag\\s*>/i", $p[1], 2 );
+                               if($tag==STRIP_COMMENTS) {
+                                       $q = preg_split( "/-->/i", $p[1], 2 );
+                               } else {
+                                       $q = preg_split( "/<\\/\\s*$tag\\s*>/i", $p[1], 2 );
+                               }
                                $marker = $rnd . sprintf("%08X", $n++);
                                $content[$marker] = $q[0];
                                $stripped .= $marker;
@@ -148,18 +178,23 @@ class Parser
                return $stripped;
        }
 
-       # Strips <nowiki>, <pre> and <math>
+       # Strips and renders <nowiki>, <pre>, <math>, <hiero>
+       # If $render is set, performs necessary rendering operations on plugins
        # Returns the text, and fills an array with data needed in unstrip()
        # If the $state is already a valid strip state, it adds to the state
-       #
-       function strip( $text, &$state )
+
+       # When $stripcomments is set, HTML comments <!-- like this -->
+       # will be stripped in addition to other tags. This is important
+       # for section editing, where these comments cause confusion when
+       # counting the sections in the wikisource
+       function strip( $text, &$state, $stripcomments = false )
        {
                $render = ($this->mOutputType == OT_HTML);
                $nowiki_content = array();
                $hiero_content = array();
                $math_content = array();
                $pre_content = array();
-               $item_content = array();
+               $comment_content = array();
 
                # Replace any instances of the placeholders
                $uniq_prefix = UNIQ_PREFIX;
@@ -174,25 +209,25 @@ class Parser
                        }
                }
 
-               if( $GLOBALS['wgUseWikiHiero'] ){
-                       $text = Parser::extractTags("hiero", $text, $hiero_content, $uniq_prefix);
-                       foreach( $hiero_content as $marker => $content ){
-                               if( $render ){
-                                       $hiero_content[$marker] = WikiHiero( $content, WH_MODE_HTML);
-                               } else {
-                                       $hiero_content[$marker] = "<hiero>$content</hiero>";
-                               }
+               $text = Parser::extractTags("hiero", $text, $hiero_content, $uniq_prefix);
+               foreach( $hiero_content as $marker => $content ){
+                       if( $render && $GLOBALS['wgUseWikiHiero']){
+                               $hiero_content[$marker] = WikiHiero( $content, WH_MODE_HTML);
+                       } else {
+                               $hiero_content[$marker] = "<hiero>$content</hiero>";
                        }
                }
 
-               if( $this->mOptions->getUseTeX() ){
-                       $text = Parser::extractTags("math", $text, $math_content, $uniq_prefix);
-                       foreach( $math_content as $marker => $content ){
-                               if( $render ){
+               $text = Parser::extractTags("math", $text, $math_content, $uniq_prefix);
+               foreach( $math_content as $marker => $content ){
+                       if( $render ) {
+                               if( $this->mOptions->getUseTeX() ) {
                                        $math_content[$marker] = renderMath( $content );
                                } else {
-                                       $math_content[$marker] = "<math>$content</math>";
+                                       $math_content[$marker] = "&lt;math&gt;$content&lt;math&gt;";
                                }
+                       } else {
+                               $math_content[$marker] = "<math>$content</math>";
                        }
                }
 
@@ -204,6 +239,12 @@ class Parser
                                $pre_content[$marker] = "<pre>$content</pre>";
                        }
                }
+               if($stripcomments) {
+                       $text = Parser::extractTags(STRIP_COMMENTS, $text, $comment_content, $uniq_prefix);
+                       foreach( $comment_content as $marker => $content ){
+                               $comment_content[$marker] = "<!--$content-->";
+                       }
+               }
 
                # Merge state with the pre-existing state, if there is one
                if ( $state ) {
@@ -211,13 +252,14 @@ class Parser
                        $state['hiero'] = $state['hiero'] + $hiero_content;
                        $state['math'] = $state['math'] + $math_content;
                        $state['pre'] = $state['pre'] + $pre_content;
+                       $state['comment'] = $state['comment'] + $comment_content;
                } else {
                        $state = array(
                          'nowiki' => $nowiki_content,
                          'hiero' => $hiero_content,
                          'math' => $math_content,
                          'pre' => $pre_content,
-                         'item' => $item_content
+                         'comment' => $comment_content
                        );
                }
                return $text;
@@ -248,60 +290,61 @@ class Parser
                          'nowiki' => array(),
                          'hiero' => array(),
                          'math' => array(),
-                         'pre' => array(),
-                         'item' => array()
+                         'pre' => array()
                        );
                }
                $state['item'][$rnd] = $text;
                return $rnd;
        }
 
+       # This method generates the list of subcategories and pages for a category
        function categoryMagic ()
        {
                global $wgLang , $wgUser ;
-               if ( !$this->mOptions->getUseCategoryMagic() ) return ;
-               $id = $this->mTitle->getArticleID() ;
-               $cat = $wgLang->ucfirst ( wfMsg ( "category" ) ) ;
-               $ti = $this->mTitle->getText() ;
-               $ti = explode ( ":" , $ti , 2 ) ;
-               if ( $cat != $ti[0] ) return "" ;
-               $r = "<br style=\"clear:both;\"/>\n";
+               if ( !$this->mOptions->getUseCategoryMagic() ) return ; # Doesn't use categories at all
 
-               $articles = array() ;
-               $parents = array () ;
-               $children = array() ;
+               $cns = Namespace::getCategory() ;
+               if ( $this->mTitle->getNamespace() != $cns ) return "" ; # This ain't a category page
+
+               $r = "<br style=\"clear:both;\"/>\n";
 
 
-#              $sk =& $this->mGetSkin();
                $sk =& $wgUser->getSkin() ;
 
+               $articles = array() ;
+               $children = array() ;
                $data = array () ;
-               $sql1 = "SELECT DISTINCT cur_title,cur_namespace FROM cur,links WHERE l_to={$id} AND l_from=cur_id";
-               $sql2 = "SELECT DISTINCT cur_title,cur_namespace FROM cur,brokenlinks WHERE bl_to={$id} AND bl_from=cur_id" ;
-
-               $res = wfQuery ( $sql1, DB_READ ) ;
-               while ( $x = wfFetchObject ( $res ) ) $data[] = $x ;
-
-               $res = wfQuery ( $sql2, DB_READ ) ;
-               while ( $x = wfFetchObject ( $res ) ) $data[] = $x ;
+               $id = $this->mTitle->getArticleID() ;
 
+               # For existing categories
+               if( $id ) {
+                       $sql = "SELECT DISTINCT cur_title,cur_namespace FROM cur,links WHERE l_to={$id} AND l_from=cur_id";
+                       $res = wfQuery ( $sql, DB_READ ) ;
+                       while ( $x = wfFetchObject ( $res ) ) $data[] = $x ;
+               } else {
+                       # For non-existing categories
+                       $t = wfStrencode( $this->mTitle->getPrefixedDBKey() );
+                       $sql = "SELECT DISTINCT cur_title,cur_namespace FROM cur,brokenlinks WHERE bl_to='$t' AND bl_from=cur_id" ;
+                       $res = wfQuery ( $sql, DB_READ ) ;
+                       while ( $x = wfFetchObject ( $res ) ) $data[] = $x ;
+               }
 
+               # For all pages that link to this category
                foreach ( $data AS $x )
                {
                        $t = $wgLang->getNsText ( $x->cur_namespace ) ;
                        if ( $t != "" ) $t .= ":" ;
                        $t .= $x->cur_title ;
 
-                       $y = explode ( ":" , $t , 2 ) ;
-                       if ( count ( $y ) == 2 && $y[0] == $cat ) {
-                               array_push ( $children , $sk->makeLink ( $t , $y[1] ) ) ;
+                       if ( $x->cur_namespace == $cns ) {
+                               array_push ( $children , $sk->makeLink ( $t ) ) ; # Subcategory
                        } else {
-                               array_push ( $articles , $sk->makeLink ( $t ) ) ;
+                               array_push ( $articles , $sk->makeLink ( $t ) ) ; # Page in this category
                        }
                }
                wfFreeResult ( $res ) ;
 
-               # Children
+               # Showing subcategories
                if ( count ( $children ) > 0 )
                {
                        asort ( $children ) ;
@@ -309,11 +352,12 @@ class Parser
                        $r .= implode ( ", " , $children ) ;
                }
 
-               # Articles
+               # Showing pages in this category
                if ( count ( $articles ) > 0 )
                {
+                       $ti = $this->mTitle->getText() ;
                        asort ( $articles ) ;
-                       $h =  wfMsg( "category_header", $ti[1] );
+                       $h =  wfMsg( "category_header", $ti );
                        $r .= "<h2>{$h}</h2>\n" ;
                        $r .= implode ( ", " , $articles ) ;
                }
@@ -362,6 +406,47 @@ class Parser
                return trim ( $t ) ;
        }
 
+       /* interface with html tidy, used if $wgUseTidy = true */
+       function tidy ( $text ) {
+               global $wgTidyConf, $wgTidyBin, $wgTidyOpts;
+               global $wgInputEncoding, $wgOutputEncoding;
+               $cleansource = '';
+               switch(strtoupper($wgOutputEncoding)) {
+                       case 'ISO-8859-1':
+                               $wgTidyOpts .= ($wgInputEncoding == $wgOutputEncoding)? ' -latin1':' -raw';
+                               break;
+                       case 'UTF-8':
+                               $wgTidyOpts .= ($wgInputEncoding == $wgOutputEncoding)? ' -utf8':' -raw';
+                               break;
+                       default:
+                               $wgTidyOpts .= ' -raw';
+                       }
+
+               $text = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"'.
+' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html>'.
+'<head><title>test</title></head><body>'.$text.'</body></html>';
+               $descriptorspec = array(
+                       0 => array("pipe", "r"),
+                       1 => array("pipe", "w"),
+                       2 => array("file", "/dev/null", "a")
+               );
+               $process = proc_open("$wgTidyBin -config $wgTidyConf $wgTidyOpts", $descriptorspec, $pipes);
+               if (is_resource($process)) {
+                       fwrite($pipes[0], $text);
+                       fclose($pipes[0]);
+                       while (!feof($pipes[1])) {
+                               $cleansource .= fgets($pipes[1], 1024);
+                       }
+                       fclose($pipes[1]);
+                       $return_value = proc_close($process);
+               }
+               if( $cleansource == '' && $text != '') {
+                       return '<h2>'.wfMsg('seriousxhtmlerrors').'</h2><pre>'.htmlspecialchars($text).'</pre>';
+               } else {
+                       return $cleansource;
+               }
+       }
+
        function doTableStuff ( $t )
        {
                $t = explode ( "\n" , $t ) ;
@@ -484,7 +569,10 @@ class Parser
                $sk =& $this->mOptions->getSkin();
                $text = $sk->transformContent( $text );
 
-               $text .= $this->categoryMagic () ;
+               if ( !isset ( $this->categoryMagicDone ) ) {
+                  $text .= $this->categoryMagic () ;
+                  $this->categoryMagicDone = true ;
+                  }
 
                wfProfileOut( $fname );
                return $text;
@@ -652,6 +740,7 @@ class Parser
        /* private */ function doTokenizedParser( $str )
        {
                global $wgLang; # for language specific parser hook
+               global $wgUploadDirectory, $wgUseTimeline;
 
                $tokenizer=Tokenizer::newFromString( $str );
                $tokenStack = array();
@@ -672,6 +761,13 @@ class Parser
                                        # simple text with no further markup
                                        $txt = $token["text"];
                                        break;
+                               case "blank":
+                                       # Text that contains blanks that have to be converted to
+                                       # non-breakable spaces for French.
+                                       # U+202F NARROW NO-BREAK SPACE might be a better choice, but
+                                       # browser support for Unicode spacing is poor.
+                                       $txt = str_replace( " ", "&nbsp;", $token["text"] );
+                                       break;
                                case "[[[":
                                        # remember the tag opened with 3 [
                                        $threeopen = true;
@@ -761,6 +857,15 @@ class Parser
                                                $txt = $this->doMagicISBN( $tokenizer );
                                        }
                                        break;
+                               case "<timeline>":
+                                       if ( $wgUseTimeline && 
+                                            "" != ( $timelinesrc = $tokenizer->readAllUntil("&lt;/timeline&gt;") ) )
+                                       {
+                                               $txt = renderTimeline( $timelinesrc );
+                                       } else {
+                                               $txt=$token["text"];
+                                       }
+                                       break;
                                default:
                                        # Call language specific Hook.
                                        $txt = $wgLang->processToken( $token, $tokenStack );
@@ -829,7 +934,7 @@ class Parser
                if ( !$image ) { $image = Namespace::getImage(); }
                if ( !$special ) { $special = Namespace::getSpecial(); }
                if ( !$media ) { $media = Namespace::getMedia(); }
-               if ( !$category ) { $category = wfMsg ( "category" ) ; }
+               if ( !$category ) { $category = Namespace::getCategory(); }
 
                $nottalk = !Namespace::isTalk( $this->mTitle->getNamespace() );
 
@@ -885,6 +990,7 @@ class Parser
                if( $noforce ) {
                        if( $iw && $this->mOptions->getInterwikiMagic() && $nottalk && $wgLang->getLanguageName( $iw ) ) {
                                array_push( $this->mOutput->mLanguageLinks, $nt->getPrefixedText() );
+                               $s .= $prefix . $trail ;
                                return (trim($s) == '')? '': $s;
                        }
                        if( $ns == $image ) {
@@ -892,30 +998,22 @@ class Parser
                                $wgLinkCache->addImageLinkObj( $nt );
                                return $s;
                        }
+                       if ( $ns == $category ) {
+                               $t = $nt->getText() ;
+                               $nnt = Title::newFromText ( Namespace::getCanonicalName($category).":".$t ) ;
+                               $t = $sk->makeLinkObj( $nnt, $t, "", "" , $prefix );
+                               $this->mOutput->mCategoryLinks[] = $t ;
+                               $s .= $prefix . $trail ;
+                               return $s ;
+                       }
                }
                if( ( $nt->getPrefixedText() == $this->mTitle->getPrefixedText() ) &&
                    ( strpos( $link, "#" ) == FALSE ) ) {
-                       $s .= $prefix . "<strong>" . $text . "</strong>" . $trail;
+                       # Self-links are handled specially; generally de-link and change to bold.
+                       $s .= $prefix . $sk->makeSelfLinkObj( $nt, $text, "", $trail );
                        return $s;
                }
 
-               # Category feature
-               $catns = strtoupper ( $nt->getDBkey () ) ;
-               $catns = explode ( ":" , $catns ) ;
-               if ( count ( $catns ) > 1 ) $catns = array_shift ( $catns ) ;
-               else $catns = "" ;
-               if ( $catns == strtoupper($category) && $this->mOptions->getUseCategoryMagic() ) {
-                       $t = explode ( ":" , $nt->getText() ) ;
-                       array_shift ( $t ) ;
-                       $t = implode ( ":" , $t ) ;
-                       $t = $wgLang->ucFirst ( $t ) ;
-                       $nnt = Title::newFromText ( $category.":".$t ) ;
-                       $t = $sk->makeLinkObj( $nnt, $t, "", $trail , $prefix );
-                       $this->mOutput->mCategoryLinks[] = $t ;
-                       $s .= $prefix . $trail ;
-                       return $s ;
-               }
-
                if( $ns == $media ) {
                        $s .= $prefix . $sk->makeMediaLinkObj( $nt, $text ) . $trail;
                        $wgLinkCache->addImageLinkObj( $nt );
@@ -1008,81 +1106,94 @@ class Parser
                return $text."\n";
        }
 
-       /* private */ function doBlockLevels( $text, $linestart )
-       {
+       /* private */ function doBlockLevels( $text, $linestart ) {
                $fname = "Parser::doBlockLevels";
                wfProfileIn( $fname );
+               
                # Parsing through the text line by line.  The main thing
                # happening here is handling of block-level elements p, pre,
                # and making lists from lines starting with * # : etc.
                #
-               $a = explode( "\n", $text );
+               $textLines = explode( "\n", $text );
 
-               $lastPref = $text = $lastLine = '';
+               $lastPrefix = $output = $lastLine = '';
                $this->mDTopen = $inBlockElem = false;
-               $npl = 0;
-               $pstack = false;
-
-               if ( ! $linestart ) { $text .= array_shift( $a ); }
-               foreach ( $a as $t ) {
-                       $oLine = $t;
-                       $opl = strlen( $lastPref );
-                       $preCloseMatch = preg_match("/<\\/pre/i", $t );
-                       $preOpenMatch = preg_match("/<pre/i", $t );
+               $prefixLength = 0;
+               $paragraphStack = false;
+
+               if ( !$linestart ) {
+                       $output .= array_shift( $textLines );
+               }
+               foreach ( $textLines as $oLine ) {
+                       $lastPrefixLength = strlen( $lastPrefix );
+                       $preCloseMatch = preg_match("/<\\/pre/i", $oLine );
+                       $preOpenMatch = preg_match("/<pre/i", $oLine );
                        if (!$this->mInPre) {
                                $this->mInPre = !empty($preOpenMatch);
                        }
                        if ( !$this->mInPre ) {
-                               $npl = strspn( $t, "*#:;" );
-                               $pref = substr( $t, 0, $npl );
+                               # Multiple prefixes may abut each other for nested lists.
+                               $prefixLength = strspn( $oLine, "*#:;" );
+                               $pref = substr( $oLine, 0, $prefixLength );
+                               
+                               # eh?
                                $pref2 = str_replace( ";", ":", $pref );
-                               $t = substr( $t, $npl );
+                               $t = substr( $oLine, $prefixLength );
                        } else {
-                               $npl = 0;
+                               # Don't interpret any other prefixes in preformatted text
+                               $prefixLength = 0;
                                $pref = $pref2 = '';
+                               $t = $oLine;
                        }
 
-                       // list generation
-                       if ( 0 != $npl && 0 == strcmp( $lastPref, $pref2 ) ) {
-                               $text .= $this->nextItem( substr( $pref, -1 ) );
-                               if ( $pstack ) { $pstack = false; }
+                       # List generation
+                       if( $prefixLength && 0 == strcmp( $lastPrefix, $pref2 ) ) {
+                               # Same as the last item, so no need to deal with nesting or opening stuff
+                               $output .= $this->nextItem( substr( $pref, -1 ) );
+                               $paragraphStack = false;
 
                                if ( ";" == substr( $pref, -1 ) ) {
-                                       $cpos = strpos( $t, ":" );
-                                       if ( false !== $cpos ) {
-                                               $term = substr( $t, 0, $cpos );
-                                               $text .= $term . $this->nextItem( ":" );
-                                               $t = substr( $t, $cpos + 1 );
+                                       # The one nasty exception: definition lists work like this:
+                                       # ; title : definition text
+                                       # So we check for : in the remainder text to split up the
+                                       # title and definition, without b0rking links.
+                                       # FIXME: This is not foolproof. Something better in Tokenizer might help.
+                                       if( preg_match( '/^(.*?(?:\s|&nbsp;)):(.*)$/', $t, $match ) ) {
+                                               $term = $match[1];
+                                               $output .= $term . $this->nextItem( ":" );
+                                               $t = $match[2];
                                        }
                                }
-                       } else if (0 != $npl || 0 != $opl) {
-                               $cpl = $this->getCommon( $pref, $lastPref );
-                               if ( $pstack ) { $pstack = false; }
-
-                               while ( $cpl < $opl ) {
-                                       $text .= $this->closeList( $lastPref{$opl-1} );
-                                       --$opl;
+                       } elseif( $prefixLength || $lastPrefixLength ) {
+                               # Either open or close a level...
+                               $commonPrefixLength = $this->getCommon( $pref, $lastPrefix );
+                               $paragraphStack = false;
+
+                               while( $commonPrefixLength < $lastPrefixLength ) {
+                                       $output .= $this->closeList( $lastPrefix{$lastPrefixLength-1} );
+                                       --$lastPrefixLength;
                                }
-                               if ( $npl <= $cpl && $cpl > 0 ) {
-                                       $text .= $this->nextItem( $pref{$cpl-1} );
+                               if ( $prefixLength <= $commonPrefixLength && $commonPrefixLength > 0 ) {
+                                       $output .= $this->nextItem( $pref{$commonPrefixLength-1} );
                                }
-                               while ( $npl > $cpl ) {
-                                       $char = substr( $pref, $cpl, 1 );
-                                       $text .= $this->openList( $char );
+                               while ( $prefixLength > $commonPrefixLength ) {
+                                       $char = substr( $pref, $commonPrefixLength, 1 );
+                                       $output .= $this->openList( $char );
 
                                        if ( ";" == $char ) {
-                                               $cpos = strpos( $t, ":" );
-                                               if ( ! ( false === $cpos ) ) {
-                                                       $term = substr( $t, 0, $cpos );
-                                                       $text .= $term . $this->nextItem( ":" );
-                                                       $t = substr( $t, $cpos + 1 );
+                                               # FIXME: This is dupe of code above
+                                               if( preg_match( '/^(.*?(?:\s|&nbsp;)):(.*)$/', $t, $match ) ) {
+                                                       $term = $match[1];
+                                                       $output .= $term . $this->nextItem( ":" );
+                                                       $t = $match[2];
                                                }
                                        }
-                                       ++$cpl;
+                                       ++$commonPrefixLength;
                                }
-                               $lastPref = $pref2;
+                               $lastPrefix = $pref2;
                        }
-                       if ( 0 == $npl ) { # No prefix (not in list)--go to paragraph mode
+                       if( 0 == $prefixLength ) {
+                               # No prefix (not in list)--go to paragraph mode
                                $uniq_prefix = UNIQ_PREFIX;
                                // XXX: use a stack for nestable elements like span, table and div
                                $openmatch = preg_match("/(<table|<blockquote|<h1|<h2|<h3|<h4|<h5|<h6|<div|<pre|<tr|<td|<p|<ul|<li)/i", $t );
@@ -1090,8 +1201,8 @@ class Parser
                                        "/(<\\/table|<\\/blockquote|<\\/h1|<\\/h2|<\\/h3|<\\/h4|<\\/h5|<\\/h6|".
                                        "<\\/div|<hr|<\\/td|<\\/pre|<\\/p|".$uniq_prefix."-pre|<\\/li|<\\/ul)/i", $t );
                                if ( $openmatch or $closematch ) {
-                                       if ( $pstack ) { $pstack = false; }
-                                       $text .= $this->closeParagraph();
+                                       $paragraphStack = false;
+                                       $output .= $this->closeParagraph();
                                        if($preOpenMatch and !$preCloseMatch) {
                                                $this->mInPre = true;   
                                        }
@@ -1104,54 +1215,54 @@ class Parser
                                        if ( " " == $t{0} ) {
                                                // pre
                                                if ($this->mLastSection != 'pre') {
-                                                       $pstack = false;
-                                                       $text .= $this->closeParagraph().'<pre>';
+                                                       $paragraphStack = false;
+                                                       $output .= $this->closeParagraph().'<pre>';
                                                        $this->mLastSection = 'pre';
                                                }
                                        } else {
                                                // paragraph
                                                if ( '' == trim($t) ) {
-                                                       if ( $pstack ) {
-                                                               $text .= $pstack.'<br/>';
-                                                               $pstack = false;
+                                                       if ( $paragraphStack ) {
+                                                               $output .= $paragraphStack.'<br/>';
+                                                               $paragraphStack = false;
                                                                $this->mLastSection = 'p';
                                                        } else {
                                                                if ($this->mLastSection != 'p' ) {
-                                                                       $text .= $this->closeParagraph();
+                                                                       $output .= $this->closeParagraph();
                                                                        $this->mLastSection = '';
-                                                                       $pstack = "<p>";
+                                                                       $paragraphStack = "<p>";
                                                                } else {
-                                                                       $pstack = '</p><p>';
+                                                                       $paragraphStack = '</p><p>';
                                                                }
                                                        }
                                                } else {
-                                                       if ( $pstack ) {
-                                                               $text .= $pstack;
-                                                               $pstack = false;
+                                                       if ( $paragraphStack ) {
+                                                               $output .= $paragraphStack;
+                                                               $paragraphStack = false;
                                                                $this->mLastSection = 'p';
                                                        } else if ($this->mLastSection != 'p') {
-                                                               $text .= $this->closeParagraph().'<p>';
+                                                               $output .= $this->closeParagraph().'<p>';
                                                                $this->mLastSection = 'p';
                                                        }
                                                }
                                        }
                                }
                        }
-                       if ($pstack === false) {
-                               $text .= $t."\n";
+                       if ($paragraphStack === false) {
+                               $output .= $t."\n";
                        }
                }
-               while ( $npl ) {
-                       $text .= $this->closeList( $pref2{$npl-1} );
-                       --$npl;
+               while ( $prefixLength ) {
+                       $output .= $this->closeList( $pref2{$prefixLength-1} );
+                       --$prefixLength;
                }
                if ( "" != $this->mLastSection ) {
-                       $text .= "</" . $this->mLastSection . ">";
+                       $output .= "</" . $this->mLastSection . ">";
                        $this->mLastSection = "";
                }
 
                wfProfileOut( $fname );
-               return $text;
+               return $output;
        }
 
        function getVariableValue( $index ) {
@@ -1166,6 +1277,11 @@ class Parser
                                return $wgLang->getMonthNameGen( date("n") );
                        case MAG_CURRENTDAY:
                                return date("j");
+                       case MAG_PAGENAME:
+                               return $this->mTitle->getText();
+                       case MAG_NAMESPACE:
+                               # return Namespace::getCanonicalName($this->mTitle->getNamespace());
+                               return $wgLang->getNsText($this->mTitle->getNamespace()); // Patch  by Dori
                        case MAG_CURRENTDAYNAME:
                                return $wgLang->getWeekdayName( date("w")+1 );
                        case MAG_CURRENTYEAR:
@@ -1407,7 +1523,7 @@ class Parser
                if ( !$found ) {
                        return $matches[0];
                } else {
-                       return $newline . $text;
+                       return $text;
                }
        }
 
@@ -1428,25 +1544,34 @@ class Parser
        # Cleans up HTML, removes dangerous tags and attributes
        /* private */ function removeHTMLtags( $text )
        {
+               global $wgUseTidy, $wgUserHtml;
                $fname = "Parser::removeHTMLtags";
                wfProfileIn( $fname );
-               $htmlpairs = array( # Tags that must be closed
-                       "b", "i", "u", "font", "big", "small", "sub", "sup", "h1",
-                       "h2", "h3", "h4", "h5", "h6", "cite", "code", "em", "s",
-                       "strike", "strong", "tt", "var", "div", "center",
-                       "blockquote", "ol", "ul", "dl", "table", "caption", "pre",
-                       "ruby", "rt" , "rb" , "rp", "p"
-               );
-               $htmlsingle = array(
-                       "br", "hr", "li", "dt", "dd"
-               );
-               $htmlnest = array( # Tags that can be nested--??
-                       "table", "tr", "td", "th", "div", "blockquote", "ol", "ul",
-                       "dl", "font", "big", "small", "sub", "sup"
-               );
-               $tabletags = array( # Can only appear inside table
-                       "td", "th", "tr"
-               );
+               
+               if( $wgUserHtml ) {
+                       $htmlpairs = array( # Tags that must be closed
+                               "b", "del", "i", "ins", "u", "font", "big", "small", "sub", "sup", "h1",
+                               "h2", "h3", "h4", "h5", "h6", "cite", "code", "em", "s",
+                               "strike", "strong", "tt", "var", "div", "center",
+                               "blockquote", "ol", "ul", "dl", "table", "caption", "pre",
+                               "ruby", "rt" , "rb" , "rp", "p"
+                       );
+                       $htmlsingle = array(
+                               "br", "hr", "li", "dt", "dd"
+                       );
+                       $htmlnest = array( # Tags that can be nested--??
+                               "table", "tr", "td", "th", "div", "blockquote", "ol", "ul",
+                               "dl", "font", "big", "small", "sub", "sup"
+                       );
+                       $tabletags = array( # Can only appear inside table
+                               "td", "th", "tr"
+                       );
+               } else {
+                       $htmlpairs = array();
+                       $htmlsingle = array();
+                       $htmlnest = array();
+                       $tabletags = array();
+               }
 
                $htmlsingle = array_merge( $tabletags, $htmlsingle );
                $htmlelements = array_merge( $htmlsingle, $htmlpairs );
@@ -1454,70 +1579,86 @@ class Parser
                $htmlattrs = $this->getHTMLattrs () ;
 
                # Remove HTML comments
-               $text = preg_replace( "/<!--.*-->/sU", "", $text );
+               $text = preg_replace( "/(\\n *<!--.*--> *(?=\\n)|<!--.*-->)/sU", "$2", $text );
 
                $bits = explode( "<", $text );
                $text = array_shift( $bits );
-               $tagstack = array(); $tablestack = array();
-
-               foreach ( $bits as $x ) {
-                       $prev = error_reporting( E_ALL & ~( E_NOTICE | E_WARNING ) );
-                       preg_match( "/^(\\/?)(\\w+)([^>]*)(\\/{0,1}>)([^<]*)$/",
-                         $x, $regs );
-                       list( $qbar, $slash, $t, $params, $brace, $rest ) = $regs;
-                       error_reporting( $prev );
-
-                       $badtag = 0 ;
-                       if ( in_array( $t = strtolower( $t ), $htmlelements ) ) {
-                               # Check our stack
-                               if ( $slash ) {
-                                       # Closing a tag...
-                                       if ( ! in_array( $t, $htmlsingle ) &&
-                                         ( $ot = array_pop( $tagstack ) ) != $t ) {
-                                               array_push( $tagstack, $ot );
-                                               $badtag = 1;
+               if(!$wgUseTidy) {
+                       $tagstack = array(); $tablestack = array();
+                       foreach ( $bits as $x ) {
+                               $prev = error_reporting( E_ALL & ~( E_NOTICE | E_WARNING ) );
+                               preg_match( "/^(\\/?)(\\w+)([^>]*)(\\/{0,1}>)([^<]*)$/",
+                               $x, $regs );
+                               list( $qbar, $slash, $t, $params, $brace, $rest ) = $regs;
+                               error_reporting( $prev );
+
+                               $badtag = 0 ;
+                               if ( in_array( $t = strtolower( $t ), $htmlelements ) ) {
+                                       # Check our stack
+                                       if ( $slash ) {
+                                               # Closing a tag...
+                                               if ( ! in_array( $t, $htmlsingle ) &&
+                                               ( count($tagstack) && $ot = array_pop( $tagstack ) ) != $t ) {
+                                                       if(!empty($ot)) array_push( $tagstack, $ot );
+                                                       $badtag = 1;
+                                               } else {
+                                                       if ( $t == "table" ) {
+                                                               $tagstack = array_pop( $tablestack );
+                                                       }
+                                                       $newparams = "";
+                                               }
                                        } else {
-                                               if ( $t == "table" ) {
-                                                       $tagstack = array_pop( $tablestack );
+                                               # Keep track for later
+                                               if ( in_array( $t, $tabletags ) &&
+                                               ! in_array( "table", $tagstack ) ) {
+                                                       $badtag = 1;
+                                               } else if ( in_array( $t, $tagstack ) &&
+                                               ! in_array ( $t , $htmlnest ) ) {
+                                                       $badtag = 1 ;
+                                               } else if ( ! in_array( $t, $htmlsingle ) ) {
+                                                       if ( $t == "table" ) {
+                                                               array_push( $tablestack, $tagstack );
+                                                               $tagstack = array();
+                                                       }
+                                                       array_push( $tagstack, $t );
                                                }
-                                               $newparams = "";
+                                               # Strip non-approved attributes from the tag
+                                               $newparams = $this->fixTagAttributes($params);
+
                                        }
-                               } else {
-                                       # Keep track for later
-                                       if ( in_array( $t, $tabletags ) &&
-                                         ! in_array( "table", $tagstack ) ) {
-                                               $badtag = 1;
-                                       } else if ( in_array( $t, $tagstack ) &&
-                                         ! in_array ( $t , $htmlnest ) ) {
-                                               $badtag = 1 ;
-                                       } else if ( ! in_array( $t, $htmlsingle ) ) {
-                                               if ( $t == "table" ) {
-                                                       array_push( $tablestack, $tagstack );
-                                                       $tagstack = array();
-                                               }
-                                               array_push( $tagstack, $t );
+                                       if ( ! $badtag ) {
+                                               $rest = str_replace( ">", "&gt;", $rest );
+                                               $text .= "<$slash$t $newparams$brace$rest";
+                                               continue;
                                        }
-                                       # Strip non-approved attributes from the tag
-                                       $newparams = $this->fixTagAttributes($params);
-
                                }
-                               if ( ! $badtag ) {
+                               $text .= "&lt;" . str_replace( ">", "&gt;", $x);
+                       }
+                       # Close off any remaining tags
+                       while ( $t = array_pop( $tagstack ) ) {
+                               $text .= "</$t>\n";
+                               if ( $t == "table" ) { $tagstack = array_pop( $tablestack ); }
+                       }
+               } else {
+                       # this might be possible using tidy itself
+                       foreach ( $bits as $x ) {
+                               preg_match( "/^(\\/?)(\\w+)([^>]*)(\\/{0,1}>)([^<]*)$/",
+                               $x, $regs );
+                               @list( $qbar, $slash, $t, $params, $brace, $rest ) = $regs;
+                               if ( in_array( $t = strtolower( $t ), $htmlelements ) ) {
+                                       $newparams = $this->fixTagAttributes($params);
                                        $rest = str_replace( ">", "&gt;", $rest );
                                        $text .= "<$slash$t $newparams$brace$rest";
-                                       continue;
+                               } else {
+                                       $text .= "&lt;" . str_replace( ">", "&gt;", $x);
                                }
-                       }
-                       $text .= "&lt;" . str_replace( ">", "&gt;", $x);
-               }
-               # Close off any remaining tags
-               while ( $t = array_pop( $tagstack ) ) {
-                       $text .= "</$t>\n";
-                       if ( $t == "table" ) { $tagstack = array_pop( $tablestack ); }
+                       }       
                }
                wfProfileOut( $fname );
                return $text;
        }
 
+
 /*
  *
  * This function accomplishes several tasks:
@@ -1533,6 +1674,8 @@ class Parser
 
        /* private */ function formatHeadings( $text )
        {
+               global $wgInputEncoding;
+               
                $doNumberHeadings = $this->mOptions->getNumberHeadings();
                $doShowToc = $this->mOptions->getShowToc();
                if( !$this->mTitle->userCanEdit() ) {
@@ -1633,7 +1776,7 @@ class Parser
                        # strip out HTML
                        $canonized_headline = preg_replace( "/<.*?" . ">/","",$canonized_headline );
                        $tocline = trim( $canonized_headline );
-                       $canonized_headline = preg_replace("/[ \\?&\\/<>\\(\\)\\[\\]=,+']+/", '_', html_entity_decode( $tocline));
+                       $canonized_headline = preg_replace("/[ \\?&\\/<>\\(\\)\\[\\]=,+']+/", '_', urlencode( do_html_entity_decode( $tocline, ENT_COMPAT, $wgInputEncoding ) ) );
                        $refer[$headlineCount] = $canonized_headline;
 
                        # count how many in assoc. array so we can track dupes in anchors