includes/Sanitizer.php

   1 <?php
   2 /**
   3  * XHTML sanitizer for MediaWiki
   4  *
   5  * Copyright © 2002-2005 Brion Vibber <brion@pobox.com> et al
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Parser
  25  */
  26
  27 /**
  28  * XHTML sanitizer for MediaWiki
  29  * @ingroup Parser
  30  */
  31 class Sanitizer {
  32         /**
  33          * Regular expression to match various types of character references in
  34          * Sanitizer::normalizeCharReferences and Sanitizer::decodeCharReferences
  35          */
  36         const CHAR_REFS_REGEX =
  37                 '/&([A-Za-z0-9\x80-\xff]+);
  38                  |&\#([0-9]+);
  39                  |&\#[xX]([0-9A-Fa-f]+);
  40                  |(&)/x';
  41
  42         const EVIL_URI_PATTERN = '!(^|\s|\*/\s*)(javascript|vbscript)([^\w]|$)!i';
  43         const XMLNS_ATTRIBUTE_PATTERN = "/^xmlns:[:A-Z_a-z-.0-9]+$/";
  44
  45         /**
  46          * List of all named character entities defined in HTML 4.01
  47          * http://www.w3.org/TR/html4/sgml/entities.html
  48          * @private
  49          */
  50         static $htmlEntities = array(
  51                 'Aacute'   => 193,
  52                 'aacute'   => 225,
  53                 'Acirc'    => 194,
  54                 'acirc'    => 226,
  55                 'acute'    => 180,
  56                 'AElig'    => 198,
  57                 'aelig'    => 230,
  58                 'Agrave'   => 192,
  59                 'agrave'   => 224,
  60                 'alefsym'  => 8501,
  61                 'Alpha'    => 913,
  62                 'alpha'    => 945,
  63                 'amp'      => 38,
  64                 'and'      => 8743,
  65                 'ang'      => 8736,
  66                 'Aring'    => 197,
  67                 'aring'    => 229,
  68                 'asymp'    => 8776,
  69                 'Atilde'   => 195,
  70                 'atilde'   => 227,
  71                 'Auml'     => 196,
  72                 'auml'     => 228,
  73                 'bdquo'    => 8222,
  74                 'Beta'     => 914,
  75                 'beta'     => 946,
  76                 'brvbar'   => 166,
  77                 'bull'     => 8226,
  78                 'cap'      => 8745,
  79                 'Ccedil'   => 199,
  80                 'ccedil'   => 231,
  81                 'cedil'    => 184,
  82                 'cent'     => 162,
  83                 'Chi'      => 935,
  84                 'chi'      => 967,
  85                 'circ'     => 710,
  86                 'clubs'    => 9827,
  87                 'cong'     => 8773,
  88                 'copy'     => 169,
  89                 'crarr'    => 8629,
  90                 'cup'      => 8746,
  91                 'curren'   => 164,
  92                 'dagger'   => 8224,
  93                 'Dagger'   => 8225,
  94                 'darr'     => 8595,
  95                 'dArr'     => 8659,
  96                 'deg'      => 176,
  97                 'Delta'    => 916,
  98                 'delta'    => 948,
  99                 'diams'    => 9830,
 100                 'divide'   => 247,
 101                 'Eacute'   => 201,
 102                 'eacute'   => 233,
 103                 'Ecirc'    => 202,
 104                 'ecirc'    => 234,
 105                 'Egrave'   => 200,
 106                 'egrave'   => 232,
 107                 'empty'    => 8709,
 108                 'emsp'     => 8195,
 109                 'ensp'     => 8194,
 110                 'Epsilon'  => 917,
 111                 'epsilon'  => 949,
 112                 'equiv'    => 8801,
 113                 'Eta'      => 919,
 114                 'eta'      => 951,
 115                 'ETH'      => 208,
 116                 'eth'      => 240,
 117                 'Euml'     => 203,
 118                 'euml'     => 235,
 119                 'euro'     => 8364,
 120                 'exist'    => 8707,
 121                 'fnof'     => 402,
 122                 'forall'   => 8704,
 123                 'frac12'   => 189,
 124                 'frac14'   => 188,
 125                 'frac34'   => 190,
 126                 'frasl'    => 8260,
 127                 'Gamma'    => 915,
 128                 'gamma'    => 947,
 129                 'ge'       => 8805,
 130                 'gt'       => 62,
 131                 'harr'     => 8596,
 132                 'hArr'     => 8660,
 133                 'hearts'   => 9829,
 134                 'hellip'   => 8230,
 135                 'Iacute'   => 205,
 136                 'iacute'   => 237,
 137                 'Icirc'    => 206,
 138                 'icirc'    => 238,
 139                 'iexcl'    => 161,
 140                 'Igrave'   => 204,
 141                 'igrave'   => 236,
 142                 'image'    => 8465,
 143                 'infin'    => 8734,
 144                 'int'      => 8747,
 145                 'Iota'     => 921,
 146                 'iota'     => 953,
 147                 'iquest'   => 191,
 148                 'isin'     => 8712,
 149                 'Iuml'     => 207,
 150                 'iuml'     => 239,
 151                 'Kappa'    => 922,
 152                 'kappa'    => 954,
 153                 'Lambda'   => 923,
 154                 'lambda'   => 955,
 155                 'lang'     => 9001,
 156                 'laquo'    => 171,
 157                 'larr'     => 8592,
 158                 'lArr'     => 8656,
 159                 'lceil'    => 8968,
 160                 'ldquo'    => 8220,
 161                 'le'       => 8804,
 162                 'lfloor'   => 8970,
 163                 'lowast'   => 8727,
 164                 'loz'      => 9674,
 165                 'lrm'      => 8206,
 166                 'lsaquo'   => 8249,
 167                 'lsquo'    => 8216,
 168                 'lt'       => 60,
 169                 'macr'     => 175,
 170                 'mdash'    => 8212,
 171                 'micro'    => 181,
 172                 'middot'   => 183,
 173                 'minus'    => 8722,
 174                 'Mu'       => 924,
 175                 'mu'       => 956,
 176                 'nabla'    => 8711,
 177                 'nbsp'     => 160,
 178                 'ndash'    => 8211,
 179                 'ne'       => 8800,
 180                 'ni'       => 8715,
 181                 'not'      => 172,
 182                 'notin'    => 8713,
 183                 'nsub'     => 8836,
 184                 'Ntilde'   => 209,
 185                 'ntilde'   => 241,
 186                 'Nu'       => 925,
 187                 'nu'       => 957,
 188                 'Oacute'   => 211,
 189                 'oacute'   => 243,
 190                 'Ocirc'    => 212,
 191                 'ocirc'    => 244,
 192                 'OElig'    => 338,
 193                 'oelig'    => 339,
 194                 'Ograve'   => 210,
 195                 'ograve'   => 242,
 196                 'oline'    => 8254,
 197                 'Omega'    => 937,
 198                 'omega'    => 969,
 199                 'Omicron'  => 927,
 200                 'omicron'  => 959,
 201                 'oplus'    => 8853,
 202                 'or'       => 8744,
 203                 'ordf'     => 170,
 204                 'ordm'     => 186,
 205                 'Oslash'   => 216,
 206                 'oslash'   => 248,
 207                 'Otilde'   => 213,
 208                 'otilde'   => 245,
 209                 'otimes'   => 8855,
 210                 'Ouml'     => 214,
 211                 'ouml'     => 246,
 212                 'para'     => 182,
 213                 'part'     => 8706,
 214                 'permil'   => 8240,
 215                 'perp'     => 8869,
 216                 'Phi'      => 934,
 217                 'phi'      => 966,
 218                 'Pi'       => 928,
 219                 'pi'       => 960,
 220                 'piv'      => 982,
 221                 'plusmn'   => 177,
 222                 'pound'    => 163,
 223                 'prime'    => 8242,
 224                 'Prime'    => 8243,
 225                 'prod'     => 8719,
 226                 'prop'     => 8733,
 227                 'Psi'      => 936,
 228                 'psi'      => 968,
 229                 'quot'     => 34,
 230                 'radic'    => 8730,
 231                 'rang'     => 9002,
 232                 'raquo'    => 187,
 233                 'rarr'     => 8594,
 234                 'rArr'     => 8658,
 235                 'rceil'    => 8969,
 236                 'rdquo'    => 8221,
 237                 'real'     => 8476,
 238                 'reg'      => 174,
 239                 'rfloor'   => 8971,
 240                 'Rho'      => 929,
 241                 'rho'      => 961,
 242                 'rlm'      => 8207,
 243                 'rsaquo'   => 8250,
 244                 'rsquo'    => 8217,
 245                 'sbquo'    => 8218,
 246                 'Scaron'   => 352,
 247                 'scaron'   => 353,
 248                 'sdot'     => 8901,
 249                 'sect'     => 167,
 250                 'shy'      => 173,
 251                 'Sigma'    => 931,
 252                 'sigma'    => 963,
 253                 'sigmaf'   => 962,
 254                 'sim'      => 8764,
 255                 'spades'   => 9824,
 256                 'sub'      => 8834,
 257                 'sube'     => 8838,
 258                 'sum'      => 8721,
 259                 'sup'      => 8835,
 260                 'sup1'     => 185,
 261                 'sup2'     => 178,
 262                 'sup3'     => 179,
 263                 'supe'     => 8839,
 264                 'szlig'    => 223,
 265                 'Tau'      => 932,
 266                 'tau'      => 964,
 267                 'there4'   => 8756,
 268                 'Theta'    => 920,
 269                 'theta'    => 952,
 270                 'thetasym' => 977,
 271                 'thinsp'   => 8201,
 272                 'THORN'    => 222,
 273                 'thorn'    => 254,
 274                 'tilde'    => 732,
 275                 'times'    => 215,
 276                 'trade'    => 8482,
 277                 'Uacute'   => 218,
 278                 'uacute'   => 250,
 279                 'uarr'     => 8593,
 280                 'uArr'     => 8657,
 281                 'Ucirc'    => 219,
 282                 'ucirc'    => 251,
 283                 'Ugrave'   => 217,
 284                 'ugrave'   => 249,
 285                 'uml'      => 168,
 286                 'upsih'    => 978,
 287                 'Upsilon'  => 933,
 288                 'upsilon'  => 965,
 289                 'Uuml'     => 220,
 290                 'uuml'     => 252,
 291                 'weierp'   => 8472,
 292                 'Xi'       => 926,
 293                 'xi'       => 958,
 294                 'Yacute'   => 221,
 295                 'yacute'   => 253,
 296                 'yen'      => 165,
 297                 'Yuml'     => 376,
 298                 'yuml'     => 255,
 299                 'Zeta'     => 918,
 300                 'zeta'     => 950,
 301                 'zwj'      => 8205,
 302                 'zwnj'     => 8204
 303         );
 304
 305         /**
 306          * Character entity aliases accepted by MediaWiki
 307          */
 308         static $htmlEntityAliases = array(
 309                 'רלמ' => 'rlm',
 310                 'رلم' => 'rlm',
 311         );
 312
 313         /**
 314          * Lazy-initialised attributes regex, see getAttribsRegex()
 315          */
 316         static $attribsRegex;
 317
 318         /**
 319          * Regular expression to match HTML/XML attribute pairs within a tag.
 320          * Allows some... latitude.
 321          * Used in Sanitizer::fixTagAttributes and Sanitizer::decodeTagAttributes
 322          */
 323         static function getAttribsRegex() {
 324                 if ( self::$attribsRegex === null ) {
 325                         $attribFirst = '[:A-Z_a-z0-9]';
 326                         $attrib = '[:A-Z_a-z-.0-9]';
 327                         $space = '[\x09\x0a\x0d\x20]';
 328                         self::$attribsRegex =
 329                                 "/(?:^|$space)({$attribFirst}{$attrib}*)
 330                                   ($space*=$space*
 331                                         (?:
 332                                          # The attribute value: quoted or alone
 333                                           \"([^<\"]*)\"
 334                                          | '([^<']*)'
 335                                          |  ([a-zA-Z0-9!#$%&()*,\\-.\\/:;<>?@[\\]^_`{|}~]+)
 336                                          |  (\#[0-9a-fA-F]+) # Technically wrong, but lots of
 337                                                                                  # colors are specified like this.
 338                                                                                  # We'll be normalizing it.
 339                                         )
 340                                 )?(?=$space|\$)/sx";
 341                 }
 342                 return self::$attribsRegex;
 343         }
 344
 345         /**
 346          * Cleans up HTML, removes dangerous tags and attributes, and
 347          * removes HTML comments
 348          * @private
 349          * @param $text String
 350          * @param $processCallback Callback to do any variable or parameter replacements in HTML attribute values
 351          * @param $args Array for the processing callback
 352          * @param $extratags Array for any extra tags to include
 353          * @param $removetags Array for any tags (default or extra) to exclude
 354          * @return string
 355          */
 356         static function removeHTMLtags( $text, $processCallback = null, $args = array(), $extratags = array(), $removetags = array() ) {
 357                 global $wgUseTidy;
 358
 359                 static $htmlpairsStatic, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
 360                         $htmllist, $listtags, $htmlsingleallowed, $htmlelementsStatic, $staticInitialised;
 361
 362                 wfProfileIn( __METHOD__ );
 363
 364                 if ( !$staticInitialised ) {
 365
 366                         $htmlpairsStatic = array( # Tags that must be closed
 367                                 'b', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
 368                                 'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
 369                                 'strike', 'strong', 'tt', 'var', 'div', 'center',
 370                                 'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
 371                                 'ruby', 'rt' , 'rb' , 'rp', 'p', 'span', 'abbr', 'dfn',
 372                                 'kbd', 'samp'
 373                         );
 374                         $htmlsingle = array(
 375                                 'br', 'hr', 'li', 'dt', 'dd'
 376                         );
 377                         $htmlsingleonly = array( # Elements that cannot have close tags
 378                                 'br', 'hr'
 379                         );
 380                         $htmlnest = array( # Tags that can be nested--??
 381                                 'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
 382                                 'dl', 'font', 'big', 'small', 'sub', 'sup', 'span'
 383                         );
 384                         $tabletags = array( # Can only appear inside table, we will close them
 385                                 'td', 'th', 'tr',
 386                         );
 387                         $htmllist = array( # Tags used by list
 388                                 'ul','ol',
 389                         );
 390                         $listtags = array( # Tags that can appear in a list
 391                                 'li',
 392                         );
 393
 394                         global $wgAllowImageTag;
 395                         if ( $wgAllowImageTag ) {
 396                                 $htmlsingle[] = 'img';
 397                                 $htmlsingleonly[] = 'img';
 398                         }
 399
 400                         $htmlsingleallowed = array_unique( array_merge( $htmlsingle, $tabletags ) );
 401                         $htmlelementsStatic = array_unique( array_merge( $htmlsingle, $htmlpairsStatic, $htmlnest ) );
 402
 403                         # Convert them all to hashtables for faster lookup
 404                         $vars = array( 'htmlpairsStatic', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
 405                                 'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelementsStatic' );
 406                         foreach ( $vars as $var ) {
 407                                 $$var = array_flip( $$var );
 408                         }
 409                         $staticInitialised = true;
 410                 }
 411                 # Populate $htmlpairs and $htmlelements with the $extratags and $removetags arrays
 412                 $extratags = array_flip( $extratags );
 413                 $removetags = array_flip( $removetags );
 414                 $htmlpairs = array_merge( $extratags, $htmlpairsStatic );
 415                 $htmlelements = array_diff_key( array_merge( $extratags, $htmlelementsStatic ) , $removetags );
 416
 417                 # Remove HTML comments
 418                 $text = Sanitizer::removeHTMLcomments( $text );
 419                 $bits = explode( '<', $text );
 420                 $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 421                 if ( !$wgUseTidy ) {
 422                         $tagstack = $tablestack = array();
 423                         foreach ( $bits as $x ) {
 424                                 $regs = array();
 425                                 # $slash: Does the current element start with a '/'?
 426                                 # $t: Current element name
 427                                 # $params: String between element name and >
 428                                 # $brace: Ending '>' or '/>'
 429                                 # $rest: Everything until the next element of $bits
 430                                 if( preg_match( '!^(/?)(\\w+)([^>]*?)(/{0,1}>)([^<]*)$!', $x, $regs ) ) {
 431                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 432                                 } else {
 433                                         $slash = $t = $params = $brace = $rest = null;
 434                                 }
 435
 436                                 $badtag = false;
 437                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 438                                         # Check our stack
 439                                         if ( $slash && isset( $htmlsingleonly[$t] ) ) {
 440                                                 $badtag = true;
 441                                         } elseif ( $slash ) {
 442                                                 # Closing a tag... is it the one we just opened?
 443                                                 $ot = @array_pop( $tagstack );
 444                                                 if ( $ot != $t ) {
 445                                                         if ( isset( $htmlsingleallowed[$ot] ) ) {
 446                                                                 # Pop all elements with an optional close tag
 447                                                                 # and see if we find a match below them
 448                                                                 $optstack = array();
 449                                                                 array_push( $optstack, $ot );
 450                                                                 $ot = @array_pop( $tagstack );
 451                                                                 while ( $ot != $t && isset( $htmlsingleallowed[$ot] ) ) {
 452                                                                         array_push( $optstack, $ot );
 453                                                                         $ot = @array_pop( $tagstack );
 454                                                                 }
 455                                                                 if ( $t != $ot ) {
 456                                                                         # No match. Push the optional elements back again
 457                                                                         $badtag = true;
 458                                                                         while ( $ot = @array_pop( $optstack ) ) {
 459                                                                                 array_push( $tagstack, $ot );
 460                                                                         }
 461                                                                 }
 462                                                         } else {
 463                                                                 @array_push( $tagstack, $ot );
 464                                                                 # <li> can be nested in <ul> or <ol>, skip those cases:
 465                                                                 if ( !isset( $htmllist[$ot] ) || !isset( $listtags[$t] ) ) {
 466                                                                         $badtag = true;
 467                                                                 }
 468                                                         }
 469                                                 } else {
 470                                                         if ( $t == 'table' ) {
 471                                                                 $tagstack = array_pop( $tablestack );
 472                                                         }
 473                                                 }
 474                                                 $newparams = '';
 475                                         } else {
 476                                                 # Keep track for later
 477                                                 if ( isset( $tabletags[$t] ) &&
 478                                                 !in_array( 'table', $tagstack ) ) {
 479                                                         $badtag = true;
 480                                                 } elseif ( in_array( $t, $tagstack ) &&
 481                                                 !isset( $htmlnest [$t ] ) ) {
 482                                                         $badtag = true;
 483                                                 # Is it a self closed htmlpair ? (bug 5487)
 484                                                 } elseif ( $brace == '/>' &&
 485                                                 isset( $htmlpairs[$t] ) ) {
 486                                                         $badtag = true;
 487                                                 } elseif ( isset( $htmlsingleonly[$t] ) ) {
 488                                                         # Hack to force empty tag for uncloseable elements
 489                                                         $brace = '/>';
 490                                                 } elseif ( isset( $htmlsingle[$t] ) ) {
 491                                                         # Hack to not close $htmlsingle tags
 492                                                         $brace = null;
 493                                                 } elseif ( isset( $tabletags[$t] )
 494                                                 && in_array( $t, $tagstack ) ) {
 495                                                         // New table tag but forgot to close the previous one
 496                                                         $text .= "</$t>";
 497                                                 } else {
 498                                                         if ( $t == 'table' ) {
 499                                                                 array_push( $tablestack, $tagstack );
 500                                                                 $tagstack = array();
 501                                                         }
 502                                                         array_push( $tagstack, $t );
 503                                                 }
 504
 505                                                 # Replace any variables or template parameters with
 506                                                 # plaintext results.
 507                                                 if( is_callable( $processCallback ) ) {
 508                                                         call_user_func_array( $processCallback, array( &$params, $args ) );
 509                                                 }
 510
 511                                                 # Strip non-approved attributes from the tag
 512                                                 $newparams = Sanitizer::fixTagAttributes( $params, $t );
 513                                         }
 514                                         if ( !$badtag ) {
 515                                                 $rest = str_replace( '>', '&gt;', $rest );
 516                                                 $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
 517                                                 $text .= "<$slash$t$newparams$close>$rest";
 518                                                 continue;
 519                                         }
 520                                 }
 521                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 522                         }
 523                         # Close off any remaining tags
 524                         while ( is_array( $tagstack ) && ($t = array_pop( $tagstack )) ) {
 525                                 $text .= "</$t>\n";
 526                                 if ( $t == 'table' ) { $tagstack = array_pop( $tablestack ); }
 527                         }
 528                 } else {
 529                         # this might be possible using tidy itself
 530                         foreach ( $bits as $x ) {
 531                                 preg_match( '/^(\\/?)(\\w+)([^>]*?)(\\/{0,1}>)([^<]*)$/',
 532                                 $x, $regs );
 533                                 @list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 534                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 535                                         if( is_callable( $processCallback ) ) {
 536                                                 call_user_func_array( $processCallback, array( &$params, $args ) );
 537                                         }
 538                                         $newparams = Sanitizer::fixTagAttributes( $params, $t );
 539                                         $rest = str_replace( '>', '&gt;', $rest );
 540                                         $text .= "<$slash$t$newparams$brace$rest";
 541                                 } else {
 542                                         $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 543                                 }
 544                         }
 545                 }
 546                 wfProfileOut( __METHOD__ );
 547                 return $text;
 548         }
 549
 550         /**
 551          * Remove '<!--', '-->', and everything between.
 552          * To avoid leaving blank lines, when a comment is both preceded
 553          * and followed by a newline (ignoring spaces), trim leading and
 554          * trailing spaces and one of the newlines.
 555          *
 556          * @private
 557          * @param $text String
 558          * @return string
 559          */
 560         static function removeHTMLcomments( $text ) {
 561                 wfProfileIn( __METHOD__ );
 562                 while (($start = strpos($text, '<!--')) !== false) {
 563                         $end = strpos($text, '-->', $start + 4);
 564                         if ($end === false) {
 565                                 # Unterminated comment; bail out
 566                                 break;
 567                         }
 568
 569                         $end += 3;
 570
 571                         # Trim space and newline if the comment is both
 572                         # preceded and followed by a newline
 573                         $spaceStart = max($start - 1, 0);
 574                         $spaceLen = $end - $spaceStart;
 575                         while (substr($text, $spaceStart, 1) === ' ' && $spaceStart > 0) {
 576                                 $spaceStart--;
 577                                 $spaceLen++;
 578                         }
 579                         while (substr($text, $spaceStart + $spaceLen, 1) === ' ')
 580                                 $spaceLen++;
 581                         if (substr($text, $spaceStart, 1) === "\n" and substr($text, $spaceStart + $spaceLen, 1) === "\n") {
 582                                 # Remove the comment, leading and trailing
 583                                 # spaces, and leave only one newline.
 584                                 $text = substr_replace($text, "\n", $spaceStart, $spaceLen + 1);
 585                         }
 586                         else {
 587                                 # Remove just the comment.
 588                                 $text = substr_replace($text, '', $start, $end - $start);
 589                         }
 590                 }
 591                 wfProfileOut( __METHOD__ );
 592                 return $text;
 593         }
 594
 595         /**
 596          * Take an array of attribute names and values and normalize or discard
 597          * illegal values for the given element type.
 598          *
 599          * - Discards attributes not on a whitelist for the given element
 600          * - Unsafe style attributes are discarded
 601          * - Invalid id attributes are reencoded
 602          *
 603          * @param $attribs Array
 604          * @param $element String
 605          * @return Array
 606          *
 607          * @todo Check for legal values where the DTD limits things.
 608          * @todo Check for unique id attribute :P
 609          */
 610         static function validateTagAttributes( $attribs, $element ) {
 611                 return Sanitizer::validateAttributes( $attribs,
 612                         Sanitizer::attributeWhitelist( $element ) );
 613         }
 614
 615         /**
 616          * Take an array of attribute names and values and normalize or discard
 617          * illegal values for the given whitelist.
 618          *
 619          * - Discards attributes not the given whitelist
 620          * - Unsafe style attributes are discarded
 621          * - Invalid id attributes are reencoded
 622          *
 623          * @param $attribs Array
 624          * @param $whitelist Array: list of allowed attribute names
 625          * @return Array
 626          *
 627          * @todo Check for legal values where the DTD limits things.
 628          * @todo Check for unique id attribute :P
 629          */
 630         static function validateAttributes( $attribs, $whitelist ) {
 631                 global $wgAllowRdfaAttributes, $wgAllowMicrodataAttributes, $wgHtml5;
 632
 633                 $whitelist = array_flip( $whitelist );
 634                 $hrefExp = '/^(' . wfUrlProtocols() . ')[^\s]+$/';
 635
 636                 $out = array();
 637                 foreach( $attribs as $attribute => $value ) {
 638                         #allow XML namespace declaration if RDFa is enabled
 639                         if ( $wgAllowRdfaAttributes && preg_match( self::XMLNS_ATTRIBUTE_PATTERN, $attribute ) ) {
 640                                 if ( !preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 641                                         $out[$attribute] = $value;
 642                                 }
 643
 644                                 continue;
 645                         }
 646
 647                         # Allow any attribute beginning with "data-", if in HTML5 mode
 648                         if ( !($wgHtml5 && preg_match( '/^data-/i', $attribute )) && !isset( $whitelist[$attribute] ) ) {
 649                                 continue;
 650                         }
 651
 652                         # Strip javascript "expression" from stylesheets.
 653                         # http://msdn.microsoft.com/workshop/author/dhtml/overview/recalc.asp
 654                         if( $attribute == 'style' ) {
 655                                 $value = Sanitizer::checkCss( $value );
 656                         }
 657
 658                         if ( $attribute === 'id' ) {
 659                                 $value = Sanitizer::escapeId( $value, 'noninitial' );
 660                         }
 661
 662                         //RDFa and microdata properties allow URLs, URIs and/or CURIs. check them for sanity
 663                         if ( $attribute === 'rel' || $attribute === 'rev' ||
 664                                 $attribute === 'about' || $attribute === 'property' || $attribute === 'resource' || #RDFa
 665                                 $attribute === 'datatype' || $attribute === 'typeof' ||                             #RDFa
 666                                 $attribute === 'itemid' || $attribute === 'itemprop' || $attribute === 'itemref' || #HTML5 microdata
 667                                 $attribute === 'itemscope' || $attribute === 'itemtype' ) {                         #HTML5 microdata
 668
 669                                 //Paranoia. Allow "simple" values but suppress javascript
 670                                 if ( preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 671                                         continue;
 672                                 }
 673                         }
 674
 675                         # NOTE: even though elements using href/src are not allowed directly, supply
 676                         #       validation code that can be used by tag hook handlers, etc
 677                         if ( $attribute === 'href' || $attribute === 'src' ) {
 678                                 if ( !preg_match( $hrefExp, $value ) ) {
 679                                         continue; //drop any href or src attributes not using an allowed protocol.
 680                                                   //NOTE: this also drops all relative URLs
 681                                 }
 682                         }
 683
 684                         // If this attribute was previously set, override it.
 685                         // Output should only have one attribute of each name.
 686                         $out[$attribute] = $value;
 687                 }
 688
 689                 if ( $wgAllowMicrodataAttributes ) {
 690                         # itemtype, itemid, itemref don't make sense without itemscope
 691                         if ( !array_key_exists( 'itemscope', $out ) ) {
 692                                 unset( $out['itemtype'] );
 693                                 unset( $out['itemid'] );
 694                                 unset( $out['itemref'] );
 695                         }
 696                         # TODO: Strip itemprop if we aren't descendants of an itemscope.
 697                 }
 698                 return $out;
 699         }
 700
 701         /**
 702          * Merge two sets of HTML attributes.  Conflicting items in the second set
 703          * will override those in the first, except for 'class' attributes which
 704          * will be combined (if they're both strings).
 705          *
 706          * @todo implement merging for other attributes such as style
 707          * @param $a Array
 708          * @param $b Array
 709          * @return array
 710          */
 711         static function mergeAttributes( $a, $b ) {
 712                 $out = array_merge( $a, $b );
 713                 if( isset( $a['class'] ) && isset( $b['class'] )
 714                 && is_string( $a['class'] ) && is_string( $b['class'] )
 715                 && $a['class'] !== $b['class'] ) {
 716                         $classes = preg_split( '/\s+/', "{$a['class']} {$b['class']}",
 717                                 -1, PREG_SPLIT_NO_EMPTY );
 718                         $out['class'] = implode( ' ', array_unique( $classes ) );
 719                 }
 720                 return $out;
 721         }
 722
 723         /**
 724          * Pick apart some CSS and check it for forbidden or unsafe structures.
 725          * Returns a sanitized string, or false if it was just too evil.
 726          *
 727          * Currently URL references, 'expression', 'tps' are forbidden.
 728          *
 729          * @param $value String
 730          * @return Mixed
 731          */
 732         static function checkCss( $value ) {
 733                 $value = Sanitizer::decodeCharReferences( $value );
 734
 735                 // Remove any comments; IE gets token splitting wrong
 736                 $value = StringUtils::delimiterReplace( '/*', '*/', ' ', $value );
 737
 738                 // Remove anything after a comment-start token, to guard against
 739                 // incorrect client implementations.
 740                 $commentPos = strpos( $value, '/*' );
 741                 if ( $commentPos !== false ) {
 742                         $value = substr( $value, 0, $commentPos );
 743                 }
 744
 745                 // Decode escape sequences and line continuation
 746                 // See the grammar in the CSS 2 spec, appendix D.
 747                 static $decodeRegex;
 748                 if ( !$decodeRegex ) {
 749                         $space = '[\\x20\\t\\r\\n\\f]';
 750                         $nl = '(?:\\n|\\r\\n|\\r|\\f)';
 751                         $backslash = '\\\\';
 752                         $decodeRegex = "/ $backslash
 753                                 (?:
 754                                         ($nl) |  # 1. Line continuation
 755                                         ([0-9A-Fa-f]{1,6})$space? |  # 2. character number
 756                                         (.) | # 3. backslash cancelling special meaning
 757                                         () | # 4. backslash at end of string
 758                                 )/xu";
 759                 }
 760                 $value = preg_replace_callback( $decodeRegex,
 761                         array( __CLASS__, 'cssDecodeCallback' ), $value );
 762
 763                 // Reject problematic keywords and control characters
 764                 if ( preg_match( '/[\000-\010\016-\037\177]/', $value ) ) {
 765                         return '/* invalid control char */';
 766                 } elseif ( preg_match( '! expression | filter\s*: | accelerator\s*: | url\s*\( !ix', $value ) ) {
 767                         return '/* insecure input */';
 768                 }
 769                 return $value;
 770         }
 771
 772         static function cssDecodeCallback( $matches ) {
 773                 if ( $matches[1] !== '' ) {
 774                         // Line continuation
 775                         return '';
 776                 } elseif ( $matches[2] !== '' ) {
 777                         $char = codepointToUtf8( hexdec( $matches[2] ) );
 778                 } elseif ( $matches[3] !== '' ) {
 779                         $char = $matches[3];
 780                 } else {
 781                         $char = '\\';
 782                 }
 783                 if ( $char == "\n" || $char == '"' || $char == "'" || $char == '\\' ) {
 784                         // These characters need to be escaped in strings
 785                         // Clean up the escape sequence to avoid parsing errors by clients
 786                         return '\\' . dechex( ord( $char ) ) . ' ';
 787                 } else {
 788                         // Decode unnecessary escape
 789                         return $char;
 790                 }
 791         }
 792
 793         /**
 794          * Take a tag soup fragment listing an HTML element's attributes
 795          * and normalize it to well-formed XML, discarding unwanted attributes.
 796          * Output is safe for further wikitext processing, with escaping of
 797          * values that could trigger problems.
 798          *
 799          * - Normalizes attribute names to lowercase
 800          * - Discards attributes not on a whitelist for the given element
 801          * - Turns broken or invalid entities into plaintext
 802          * - Double-quotes all attribute values
 803          * - Attributes without values are given the name as attribute
 804          * - Double attributes are discarded
 805          * - Unsafe style attributes are discarded
 806          * - Prepends space if there are attributes.
 807          *
 808          * @param $text String
 809          * @param $element String
 810          * @return String
 811          */
 812         static function fixTagAttributes( $text, $element ) {
 813                 if( trim( $text ) == '' ) {
 814                         return '';
 815                 }
 816
 817                 $stripped = Sanitizer::validateTagAttributes(
 818                         Sanitizer::decodeTagAttributes( $text ), $element );
 819
 820                 $attribs = array();
 821                 foreach( $stripped as $attribute => $value ) {
 822                         $encAttribute = htmlspecialchars( $attribute );
 823                         $encValue = Sanitizer::safeEncodeAttribute( $value );
 824
 825                         $attribs[] = "$encAttribute=\"$encValue\"";
 826                 }
 827                 return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
 828         }
 829
 830         /**
 831          * Encode an attribute value for HTML output.
 832          * @param $text String
 833          * @return HTML-encoded text fragment
 834          */
 835         static function encodeAttribute( $text ) {
 836                 $encValue = htmlspecialchars( $text, ENT_QUOTES );
 837
 838                 // Whitespace is normalized during attribute decoding,
 839                 // so if we've been passed non-spaces we must encode them
 840                 // ahead of time or they won't be preserved.
 841                 $encValue = strtr( $encValue, array(
 842                         "\n" => '&#10;',
 843                         "\r" => '&#13;',
 844                         "\t" => '&#9;',
 845                 ) );
 846
 847                 return $encValue;
 848         }
 849
 850         /**
 851          * Encode an attribute value for HTML tags, with extra armoring
 852          * against further wiki processing.
 853          * @param $text String
 854          * @return HTML-encoded text fragment
 855          */
 856         static function safeEncodeAttribute( $text ) {
 857                 $encValue = Sanitizer::encodeAttribute( $text );
 858
 859                 # Templates and links may be expanded in later parsing,
 860                 # creating invalid or dangerous output. Suppress this.
 861                 $encValue = strtr( $encValue, array(
 862                         '<'    => '&lt;',   // This should never happen,
 863                         '>'    => '&gt;',   // we've received invalid input
 864                         '"'    => '&quot;', // which should have been escaped.
 865                         '{'    => '&#123;',
 866                         '['    => '&#91;',
 867                         "''"   => '&#39;&#39;',
 868                         'ISBN' => '&#73;SBN',
 869                         'RFC'  => '&#82;FC',
 870                         'PMID' => '&#80;MID',
 871                         '|'    => '&#124;',
 872                         '__'   => '&#95;_',
 873                 ) );
 874
 875                 # Stupid hack
 876                 $encValue = preg_replace_callback(
 877                         '/(' . wfUrlProtocols() . ')/',
 878                         array( 'Sanitizer', 'armorLinksCallback' ),
 879                         $encValue );
 880                 return $encValue;
 881         }
 882
 883         /**
 884          * Given a value, escape it so that it can be used in an id attribute and
 885          * return it.  This will use HTML5 validation if $wgExperimentalHtmlIds is
 886          * true, allowing anything but ASCII whitespace.  Otherwise it will use
 887          * HTML 4 rules, which means a narrow subset of ASCII, with bad characters
 888          * escaped with lots of dots.
 889          *
 890          * To ensure we don't have to bother escaping anything, we also strip ', ",
 891          * & even if $wgExperimentalIds is true.  TODO: Is this the best tactic?
 892          * We also strip # because it upsets IE, and % because it could be
 893          * ambiguous if it's part of something that looks like a percent escape
 894          * (which don't work reliably in fragments cross-browser).
 895          *
 896          * @see http://www.w3.org/TR/html401/types.html#type-name Valid characters
 897          *                                                          in the id and
 898          *                                                          name attributes
 899          * @see http://www.w3.org/TR/html401/struct/links.html#h-12.2.3 Anchors with the id attribute
 900          * @see http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#the-id-attribute
 901          *   HTML5 definition of id attribute
 902          *
 903          * @param $id String: id to escape
 904          * @param $options Mixed: string or array of strings (default is array()):
 905          *   'noninitial': This is a non-initial fragment of an id, not a full id,
 906          *       so don't pay attention if the first character isn't valid at the
 907          *       beginning of an id.  Only matters if $wgExperimentalHtmlIds is
 908          *       false.
 909          *   'legacy': Behave the way the old HTML 4-based ID escaping worked even
 910          *       if $wgExperimentalHtmlIds is used, so we can generate extra
 911          *       anchors and links won't break.
 912          * @return String
 913          */
 914         static function escapeId( $id, $options = array() ) {
 915                 global $wgHtml5, $wgExperimentalHtmlIds;
 916                 $options = (array)$options;
 917
 918                 if ( $wgHtml5 && $wgExperimentalHtmlIds && !in_array( 'legacy', $options ) ) {
 919                         $id = Sanitizer::decodeCharReferences( $id );
 920                         $id = preg_replace( '/[ \t\n\r\f_\'"&#%]+/', '_', $id );
 921                         $id = trim( $id, '_' );
 922                         if ( $id === '' ) {
 923                                 # Must have been all whitespace to start with.
 924                                 return '_';
 925                         } else {
 926                                 return $id;
 927                         }
 928                 }
 929
 930                 # HTML4-style escaping
 931                 static $replace = array(
 932                         '%3A' => ':',
 933                         '%' => '.'
 934                 );
 935
 936                 $id = urlencode( Sanitizer::decodeCharReferences( strtr( $id, ' ', '_' ) ) );
 937                 $id = str_replace( array_keys( $replace ), array_values( $replace ), $id );
 938
 939                 if ( !preg_match( '/^[a-zA-Z]/', $id )
 940                 && !in_array( 'noninitial', $options ) )  {
 941                         // Initial character must be a letter!
 942                         $id = "x$id";
 943                 }
 944                 return $id;
 945         }
 946
 947         /**
 948          * Given a value, escape it so that it can be used as a CSS class and
 949          * return it.
 950          *
 951          * @todo For extra validity, input should be validated UTF-8.
 952          *
 953          * @see http://www.w3.org/TR/CSS21/syndata.html Valid characters/format
 954          *
 955          * @param $class String
 956          * @return String
 957          */
 958         static function escapeClass( $class ) {
 959                 // Convert ugly stuff to underscores and kill underscores in ugly places
 960                 return rtrim(preg_replace(
 961                         array('/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/','/_+/'),
 962                         '_',
 963                         $class ), '_');
 964         }
 965
 966         /**
 967          * Given HTML input, escape with htmlspecialchars but un-escape entites.
 968          * This allows (generally harmless) entities like &#160; to survive.
 969          *
 970          * @param $html String to escape
 971          * @return String: escaped input
 972          */
 973         static function escapeHtmlAllowEntities( $html ) {
 974                 $html = Sanitizer::decodeCharReferences( $html );
 975                 # It seems wise to escape ' as well as ", as a matter of course.  Can't
 976                 # hurt.
 977                 $html = htmlspecialchars( $html, ENT_QUOTES );
 978                 return $html;
 979         }
 980
 981         /**
 982          * Regex replace callback for armoring links against further processing.
 983          * @param $matches Array
 984          * @return string
 985          */
 986         private static function armorLinksCallback( $matches ) {
 987                 return str_replace( ':', '&#58;', $matches[1] );
 988         }
 989
 990         /**
 991          * Return an associative array of attribute names and values from
 992          * a partial tag string. Attribute names are forces to lowercase,
 993          * character references are decoded to UTF-8 text.
 994          *
 995          * @param $text String
 996          * @return Array
 997          */
 998         public static function decodeTagAttributes( $text ) {
 999                 if( trim( $text ) == '' ) {
1000                         return array();
1001                 }
1002
1003                 $attribs = array();
1004                 $pairs = array();
1005                 if( !preg_match_all(
1006                         self::getAttribsRegex(),
1007                         $text,
1008                         $pairs,
1009                         PREG_SET_ORDER ) ) {
1010                         return $attribs;
1011                 }
1012
1013                 foreach( $pairs as $set ) {
1014                         $attribute = strtolower( $set[1] );
1015                         $value = Sanitizer::getTagAttributeCallback( $set );
1016
1017                         // Normalize whitespace
1018                         $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
1019                         $value = trim( $value );
1020
1021                         // Decode character references
1022                         $attribs[$attribute] = Sanitizer::decodeCharReferences( $value );
1023                 }
1024                 return $attribs;
1025         }
1026
1027         /**
1028          * Pick the appropriate attribute value from a match set from the
1029          * attribs regex matches.
1030          *
1031          * @param $set Array
1032          * @return String
1033          */
1034         private static function getTagAttributeCallback( $set ) {
1035                 if( isset( $set[6] ) ) {
1036                         # Illegal #XXXXXX color with no quotes.
1037                         return $set[6];
1038                 } elseif( isset( $set[5] ) ) {
1039                         # No quotes.
1040                         return $set[5];
1041                 } elseif( isset( $set[4] ) ) {
1042                         # Single-quoted
1043                         return $set[4];
1044                 } elseif( isset( $set[3] ) ) {
1045                         # Double-quoted
1046                         return $set[3];
1047                 } elseif( !isset( $set[2] ) ) {
1048                         # In XHTML, attributes must have a value.
1049                         # For 'reduced' form, return explicitly the attribute name here.
1050                         return $set[1];
1051                 } else {
1052                         throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
1053                 }
1054         }
1055
1056         /**
1057          * Normalize whitespace and character references in an XML source-
1058          * encoded text for an attribute value.
1059          *
1060          * See http://www.w3.org/TR/REC-xml/#AVNormalize for background,
1061          * but note that we're not returning the value, but are returning
1062          * XML source fragments that will be slapped into output.
1063          *
1064          * @param $text String
1065          * @return String
1066          */
1067         private static function normalizeAttributeValue( $text ) {
1068                 return str_replace( '"', '&quot;',
1069                         self::normalizeWhitespace(
1070                                 Sanitizer::normalizeCharReferences( $text ) ) );
1071         }
1072
1073         private static function normalizeWhitespace( $text ) {
1074                 return preg_replace(
1075                         '/\r\n|[\x20\x0d\x0a\x09]/',
1076                         ' ',
1077                         $text );
1078         }
1079
1080         /**
1081          * Normalizes whitespace in a section name, such as might be returned
1082          * by Parser::stripSectionName(), for use in the id's that are used for
1083          * section links.
1084          *
1085          * @param $section String
1086          * @return String
1087          */
1088         static function normalizeSectionNameWhitespace( $section ) {
1089                 return trim( preg_replace( '/[ _]+/', ' ', $section ) );
1090         }
1091
1092         /**
1093          * Ensure that any entities and character references are legal
1094          * for XML and XHTML specifically. Any stray bits will be
1095          * &amp;-escaped to result in a valid text fragment.
1096          *
1097          * a. named char refs can only be &lt; &gt; &amp; &quot;, others are
1098          *   numericized (this way we're well-formed even without a DTD)
1099          * b. any numeric char refs must be legal chars, not invalid or forbidden
1100          * c. use &#x, not &#X
1101          * d. fix or reject non-valid attributes
1102          *
1103          * @param $text String
1104          * @return String
1105          * @private
1106          */
1107         static function normalizeCharReferences( $text ) {
1108                 return preg_replace_callback(
1109                         self::CHAR_REFS_REGEX,
1110                         array( 'Sanitizer', 'normalizeCharReferencesCallback' ),
1111                         $text );
1112         }
1113         /**
1114          * @param $matches String
1115          * @return String
1116          */
1117         static function normalizeCharReferencesCallback( $matches ) {
1118                 $ret = null;
1119                 if( $matches[1] != '' ) {
1120                         $ret = Sanitizer::normalizeEntity( $matches[1] );
1121                 } elseif( $matches[2] != '' ) {
1122                         $ret = Sanitizer::decCharReference( $matches[2] );
1123                 } elseif( $matches[3] != ''  ) {
1124                         $ret = Sanitizer::hexCharReference( $matches[3] );
1125                 }
1126                 if( is_null( $ret ) ) {
1127                         return htmlspecialchars( $matches[0] );
1128                 } else {
1129                         return $ret;
1130                 }
1131         }
1132
1133         /**
1134          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1135          * return the equivalent numeric entity reference (except for the core &lt;
1136          * &gt; &amp; &quot;). If the entity is a MediaWiki-specific alias, returns
1137          * the HTML equivalent. Otherwise, returns HTML-escaped text of
1138          * pseudo-entity source (eg &amp;foo;)
1139          *
1140          * @param $name String
1141          * @return String
1142          */
1143         static function normalizeEntity( $name ) {
1144                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1145                         return '&' . self::$htmlEntityAliases[$name] . ';';
1146                 } elseif ( in_array( $name,
1147                 array( 'lt', 'gt', 'amp', 'quot' ) ) ) {
1148                         return "&$name;";
1149                 } elseif ( isset( self::$htmlEntities[$name] ) ) {
1150                         return '&#' . self::$htmlEntities[$name] . ';';
1151                 } else {
1152                         return "&amp;$name;";
1153                 }
1154         }
1155
1156         static function decCharReference( $codepoint ) {
1157                 $point = intval( $codepoint );
1158                 if( Sanitizer::validateCodepoint( $point ) ) {
1159                         return sprintf( '&#%d;', $point );
1160                 } else {
1161                         return null;
1162                 }
1163         }
1164
1165         static function hexCharReference( $codepoint ) {
1166                 $point = hexdec( $codepoint );
1167                 if( Sanitizer::validateCodepoint( $point ) ) {
1168                         return sprintf( '&#x%x;', $point );
1169                 } else {
1170                         return null;
1171                 }
1172         }
1173
1174         /**
1175          * Returns true if a given Unicode codepoint is a valid character in XML.
1176          * @param $codepoint Integer
1177          * @return Boolean
1178          */
1179         private static function validateCodepoint( $codepoint ) {
1180                 return ($codepoint ==    0x09)
1181                         || ($codepoint ==    0x0a)
1182                         || ($codepoint ==    0x0d)
1183                         || ($codepoint >=    0x20 && $codepoint <=   0xd7ff)
1184                         || ($codepoint >=  0xe000 && $codepoint <=   0xfffd)
1185                         || ($codepoint >= 0x10000 && $codepoint <= 0x10ffff);
1186         }
1187
1188         /**
1189          * Decode any character references, numeric or named entities,
1190          * in the text and return a UTF-8 string.
1191          *
1192          * @param $text String
1193          * @return String
1194          */
1195         public static function decodeCharReferences( $text ) {
1196                 return preg_replace_callback(
1197                         self::CHAR_REFS_REGEX,
1198                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1199                         $text );
1200         }
1201
1202         /**
1203          * Decode any character references, numeric or named entities,
1204          * in the next and normalize the resulting string. (bug 14952)
1205          *
1206          * This is useful for page titles, not for text to be displayed,
1207          * MediaWiki allows HTML entities to escape normalization as a feature.
1208          *
1209          * @param $text String (already normalized, containing entities)
1210          * @return String (still normalized, without entities)
1211          */
1212         public static function decodeCharReferencesAndNormalize( $text ) {
1213                 global $wgContLang;
1214                 $text = preg_replace_callback(
1215                         self::CHAR_REFS_REGEX,
1216                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1217                         $text, /* limit */ -1, $count );
1218
1219                 if ( $count ) {
1220                         return $wgContLang->normalize( $text );
1221                 } else {
1222                         return $text;
1223                 }
1224         }
1225
1226         /**
1227          * @param $matches String
1228          * @return String
1229          */
1230         static function decodeCharReferencesCallback( $matches ) {
1231                 if( $matches[1] != '' ) {
1232                         return Sanitizer::decodeEntity( $matches[1] );
1233                 } elseif( $matches[2] != '' ) {
1234                         return  Sanitizer::decodeChar( intval( $matches[2] ) );
1235                 } elseif( $matches[3] != ''  ) {
1236                         return  Sanitizer::decodeChar( hexdec( $matches[3] ) );
1237                 }
1238                 # Last case should be an ampersand by itself
1239                 return $matches[0];
1240         }
1241
1242         /**
1243          * Return UTF-8 string for a codepoint if that is a valid
1244          * character reference, otherwise U+FFFD REPLACEMENT CHARACTER.
1245          * @param $codepoint Integer
1246          * @return String
1247          * @private
1248          */
1249         static function decodeChar( $codepoint ) {
1250                 if( Sanitizer::validateCodepoint( $codepoint ) ) {
1251                         return codepointToUtf8( $codepoint );
1252                 } else {
1253                         return UTF8_REPLACEMENT;
1254                 }
1255         }
1256
1257         /**
1258          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1259          * return the UTF-8 encoding of that character. Otherwise, returns
1260          * pseudo-entity source (eg &foo;)
1261          *
1262          * @param $name Strings
1263          * @return String
1264          */
1265         static function decodeEntity( $name ) {
1266                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1267                         $name = self::$htmlEntityAliases[$name];
1268                 }
1269                 if( isset( self::$htmlEntities[$name] ) ) {
1270                         return codepointToUtf8( self::$htmlEntities[$name] );
1271                 } else {
1272                         return "&$name;";
1273                 }
1274         }
1275
1276         /**
1277          * Fetch the whitelist of acceptable attributes for a given element name.
1278          *
1279          * @param $element String
1280          * @return Array
1281          */
1282         static function attributeWhitelist( $element ) {
1283                 static $list;
1284                 if( !isset( $list ) ) {
1285                         $list = Sanitizer::setupAttributeWhitelist();
1286                 }
1287                 return isset( $list[$element] )
1288                         ? $list[$element]
1289                         : array();
1290         }
1291
1292         /**
1293          * Foreach array key (an allowed HTML element), return an array
1294          * of allowed attributes
1295          * @return Array
1296          */
1297         static function setupAttributeWhitelist() {
1298                 global $wgAllowRdfaAttributes, $wgHtml5, $wgAllowMicrodataAttributes;
1299
1300                 $common = array( 'id', 'class', 'lang', 'dir', 'title', 'style' );
1301
1302                 if ( $wgAllowRdfaAttributes ) {
1303                         #RDFa attributes as specified in section 9 of http://www.w3.org/TR/2008/REC-rdfa-syntax-20081014
1304                         $common = array_merge( $common, array(
1305                             'about', 'property', 'resource', 'datatype', 'typeof',
1306                         ) );
1307                 }
1308
1309                 if ( $wgHtml5 && $wgAllowMicrodataAttributes ) {
1310                         # add HTML5 microdata tages as pecified by http://www.whatwg.org/specs/web-apps/current-work/multipage/microdata.html#the-microdata-model
1311                         $common = array_merge( $common, array(
1312                             'itemid', 'itemprop', 'itemref', 'itemscope', 'itemtype'
1313                         ) );
1314                 }
1315
1316                 $block = array_merge( $common, array( 'align' ) );
1317                 $tablealign = array( 'align', 'char', 'charoff', 'valign' );
1318                 $tablecell = array( 'abbr',
1319                                     'axis',
1320                                     'headers',
1321                                     'scope',
1322                                     'rowspan',
1323                                     'colspan',
1324                                     'nowrap', # deprecated
1325                                     'width',  # deprecated
1326                                     'height', # deprecated
1327                                     'bgcolor' # deprecated
1328                                     );
1329
1330                 # Numbers refer to sections in HTML 4.01 standard describing the element.
1331                 # See: http://www.w3.org/TR/html4/
1332                 $whitelist = array (
1333                         # 7.5.4
1334                         'div'        => $block,
1335                         'center'     => $common, # deprecated
1336                         'span'       => $block, # ??
1337
1338                         # 7.5.5
1339                         'h1'         => $block,
1340                         'h2'         => $block,
1341                         'h3'         => $block,
1342                         'h4'         => $block,
1343                         'h5'         => $block,
1344                         'h6'         => $block,
1345
1346                         # 7.5.6
1347                         # address
1348
1349                         # 8.2.4
1350                         # bdo
1351
1352                         # 9.2.1
1353                         'em'         => $common,
1354                         'strong'     => $common,
1355                         'cite'       => $common,
1356                         'dfn'        => $common,
1357                         'code'       => $common,
1358                         'samp'       => $common,
1359                         'kbd'        => $common,
1360                         'var'        => $common,
1361                         'abbr'       => $common,
1362                         # acronym
1363
1364                         # 9.2.2
1365                         'blockquote' => array_merge( $common, array( 'cite' ) ),
1366                         # q
1367
1368                         # 9.2.3
1369                         'sub'        => $common,
1370                         'sup'        => $common,
1371
1372                         # 9.3.1
1373                         'p'          => $block,
1374
1375                         # 9.3.2
1376                         'br'         => array( 'id', 'class', 'title', 'style', 'clear' ),
1377
1378                         # 9.3.4
1379                         'pre'        => array_merge( $common, array( 'width' ) ),
1380
1381                         # 9.4
1382                         'ins'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1383                         'del'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1384
1385                         # 10.2
1386                         'ul'         => array_merge( $common, array( 'type' ) ),
1387                         'ol'         => array_merge( $common, array( 'type', 'start' ) ),
1388                         'li'         => array_merge( $common, array( 'type', 'value' ) ),
1389
1390                         # 10.3
1391                         'dl'         => $common,
1392                         'dd'         => $common,
1393                         'dt'         => $common,
1394
1395                         # 11.2.1
1396                         'table'      => array_merge( $common,
1397                                                                 array( 'summary', 'width', 'border', 'frame',
1398                                                                                 'rules', 'cellspacing', 'cellpadding',
1399                                                                                 'align', 'bgcolor',
1400                                                                 ) ),
1401
1402                         # 11.2.2
1403                         'caption'    => array_merge( $common, array( 'align' ) ),
1404
1405                         # 11.2.3
1406                         'thead'      => array_merge( $common, $tablealign ),
1407                         'tfoot'      => array_merge( $common, $tablealign ),
1408                         'tbody'      => array_merge( $common, $tablealign ),
1409
1410                         # 11.2.4
1411                         'colgroup'   => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1412                         'col'        => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1413
1414                         # 11.2.5
1415                         'tr'         => array_merge( $common, array( 'bgcolor' ), $tablealign ),
1416
1417                         # 11.2.6
1418                         'td'         => array_merge( $common, $tablecell, $tablealign ),
1419                         'th'         => array_merge( $common, $tablecell, $tablealign ),
1420
1421                         # 12.2 # NOTE: <a> is not allowed directly, but the attrib whitelist is used from the Parser object
1422                         'a'          => array_merge( $common, array( 'href', 'rel', 'rev' ) ), # rel/rev esp. for RDFa
1423
1424                         # 13.2
1425                         # Not usually allowed, but may be used for extension-style hooks
1426                         # such as <math> when it is rasterized, or if $wgAllowImageTag is
1427                         # true
1428                         'img'        => array_merge( $common, array( 'alt', 'src', 'width', 'height' ) ),
1429
1430                         # 15.2.1
1431                         'tt'         => $common,
1432                         'b'          => $common,
1433                         'i'          => $common,
1434                         'big'        => $common,
1435                         'small'      => $common,
1436                         'strike'     => $common,
1437                         's'          => $common,
1438                         'u'          => $common,
1439
1440                         # 15.2.2
1441                         'font'       => array_merge( $common, array( 'size', 'color', 'face' ) ),
1442                         # basefont
1443
1444                         # 15.3
1445                         'hr'         => array_merge( $common, array( 'noshade', 'size', 'width' ) ),
1446
1447                         # XHTML Ruby annotation text module, simple ruby only.
1448                         # http://www.w3c.org/TR/ruby/
1449                         'ruby'       => $common,
1450                         # rbc
1451                         # rtc
1452                         'rb'         => $common,
1453                         'rt'         => $common, #array_merge( $common, array( 'rbspan' ) ),
1454                         'rp'         => $common,
1455
1456                         # MathML root element, where used for extensions
1457                         # 'title' may not be 100% valid here; it's XHTML
1458                         # http://www.w3.org/TR/REC-MathML/
1459                         'math'       => array( 'class', 'style', 'id', 'title' ),
1460                         );
1461                 return $whitelist;
1462         }
1463
1464         /**
1465          * Take a fragment of (potentially invalid) HTML and return
1466          * a version with any tags removed, encoded as plain text.
1467          *
1468          * Warning: this return value must be further escaped for literal
1469          * inclusion in HTML output as of 1.10!
1470          *
1471          * @param $text String: HTML fragment
1472          * @return String
1473          */
1474         static function stripAllTags( $text ) {
1475                 # Actual <tags>
1476                 $text = StringUtils::delimiterReplace( '<', '>', '', $text );
1477
1478                 # Normalize &entities and whitespace
1479                 $text = self::decodeCharReferences( $text );
1480                 $text = self::normalizeWhitespace( $text );
1481
1482                 return $text;
1483         }
1484
1485         /**
1486          * Hack up a private DOCTYPE with HTML's standard entity declarations.
1487          * PHP 4 seemed to know these if you gave it an HTML doctype, but
1488          * PHP 5.1 doesn't.
1489          *
1490          * Use for passing XHTML fragments to PHP's XML parsing functions
1491          *
1492          * @return String
1493          */
1494         static function hackDocType() {
1495                 $out = "<!DOCTYPE html [\n";
1496                 foreach( self::$htmlEntities as $entity => $codepoint ) {
1497                         $out .= "<!ENTITY $entity \"&#$codepoint;\">";
1498                 }
1499                 $out .= "]>\n";
1500                 return $out;
1501         }
1502
1503         static function cleanUrl( $url ) {
1504                 # Normalize any HTML entities in input. They will be
1505                 # re-escaped by makeExternalLink().
1506                 $url = Sanitizer::decodeCharReferences( $url );
1507
1508                 # Escape any control characters introduced by the above step
1509                 $url = preg_replace( '/[\][<>"\\x00-\\x20\\x7F\|]/e', "urlencode('\\0')", $url );
1510
1511                 # Validate hostname portion
1512                 $matches = array();
1513                 if( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
1514                         list( /* $whole */, $protocol, $host, $rest ) = $matches;
1515
1516                         // Characters that will be ignored in IDNs.
1517                         // http://tools.ietf.org/html/3454#section-3.1
1518                         // Strip them before further processing so blacklists and such work.
1519                         $strip = "/
1520                                 \\s|          # general whitespace
1521                                 \xc2\xad|     # 00ad SOFT HYPHEN
1522                                 \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
1523                                 \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
1524                                 \xe2\x81\xa0| # 2060 WORD JOINER
1525                                 \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
1526                                 \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
1527                                 \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
1528                                 \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
1529                                 \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
1530                                 \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
1531                                 \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
1532                                 [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe00f VARIATION SELECTOR-1-16
1533                                 /xuD";
1534
1535                         $host = preg_replace( $strip, '', $host );
1536
1537                         // @todo Fixme: validate hostnames here
1538
1539                         return $protocol . $host . $rest;
1540                 } else {
1541                         return $url;
1542                 }
1543         }
1544
1545 }