includes/Sanitizer.php

   1 <?php
   2 /**
   3  * XHTML sanitizer for MediaWiki
   4  *
   5  * Copyright © 2002-2005 Brion Vibber <brion@pobox.com> et al
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Parser
  25  */
  26
  27 /**
  28  * XHTML sanitizer for MediaWiki
  29  * @ingroup Parser
  30  */
  31 class Sanitizer {
  32         /**
  33          * Regular expression to match various types of character references in
  34          * Sanitizer::normalizeCharReferences and Sanitizer::decodeCharReferences
  35          */
  36         const CHAR_REFS_REGEX =
  37                 '/&([A-Za-z0-9\x80-\xff]+);
  38                  |&\#([0-9]+);
  39                  |&\#[xX]([0-9A-Fa-f]+);
  40                  |(&)/x';
  41
  42         const EVIL_URI_PATTERN = '!(^|\s|\*/\s*)(javascript|vbscript)([^\w]|$)!i';
  43         const XMLNS_ATTRIBUTE_PATTERN = "/^xmlns:[:A-Z_a-z-.0-9]+$/";
  44
  45         /**
  46          * List of all named character entities defined in HTML 4.01
  47          * http://www.w3.org/TR/html4/sgml/entities.html
  48          * @private
  49          */
  50         static $htmlEntities = array(
  51                 'Aacute'   => 193,
  52                 'aacute'   => 225,
  53                 'Acirc'    => 194,
  54                 'acirc'    => 226,
  55                 'acute'    => 180,
  56                 'AElig'    => 198,
  57                 'aelig'    => 230,
  58                 'Agrave'   => 192,
  59                 'agrave'   => 224,
  60                 'alefsym'  => 8501,
  61                 'Alpha'    => 913,
  62                 'alpha'    => 945,
  63                 'amp'      => 38,
  64                 'and'      => 8743,
  65                 'ang'      => 8736,
  66                 'Aring'    => 197,
  67                 'aring'    => 229,
  68                 'asymp'    => 8776,
  69                 'Atilde'   => 195,
  70                 'atilde'   => 227,
  71                 'Auml'     => 196,
  72                 'auml'     => 228,
  73                 'bdquo'    => 8222,
  74                 'Beta'     => 914,
  75                 'beta'     => 946,
  76                 'brvbar'   => 166,
  77                 'bull'     => 8226,
  78                 'cap'      => 8745,
  79                 'Ccedil'   => 199,
  80                 'ccedil'   => 231,
  81                 'cedil'    => 184,
  82                 'cent'     => 162,
  83                 'Chi'      => 935,
  84                 'chi'      => 967,
  85                 'circ'     => 710,
  86                 'clubs'    => 9827,
  87                 'cong'     => 8773,
  88                 'copy'     => 169,
  89                 'crarr'    => 8629,
  90                 'cup'      => 8746,
  91                 'curren'   => 164,
  92                 'dagger'   => 8224,
  93                 'Dagger'   => 8225,
  94                 'darr'     => 8595,
  95                 'dArr'     => 8659,
  96                 'deg'      => 176,
  97                 'Delta'    => 916,
  98                 'delta'    => 948,
  99                 'diams'    => 9830,
 100                 'divide'   => 247,
 101                 'Eacute'   => 201,
 102                 'eacute'   => 233,
 103                 'Ecirc'    => 202,
 104                 'ecirc'    => 234,
 105                 'Egrave'   => 200,
 106                 'egrave'   => 232,
 107                 'empty'    => 8709,
 108                 'emsp'     => 8195,
 109                 'ensp'     => 8194,
 110                 'Epsilon'  => 917,
 111                 'epsilon'  => 949,
 112                 'equiv'    => 8801,
 113                 'Eta'      => 919,
 114                 'eta'      => 951,
 115                 'ETH'      => 208,
 116                 'eth'      => 240,
 117                 'Euml'     => 203,
 118                 'euml'     => 235,
 119                 'euro'     => 8364,
 120                 'exist'    => 8707,
 121                 'fnof'     => 402,
 122                 'forall'   => 8704,
 123                 'frac12'   => 189,
 124                 'frac14'   => 188,
 125                 'frac34'   => 190,
 126                 'frasl'    => 8260,
 127                 'Gamma'    => 915,
 128                 'gamma'    => 947,
 129                 'ge'       => 8805,
 130                 'gt'       => 62,
 131                 'harr'     => 8596,
 132                 'hArr'     => 8660,
 133                 'hearts'   => 9829,
 134                 'hellip'   => 8230,
 135                 'Iacute'   => 205,
 136                 'iacute'   => 237,
 137                 'Icirc'    => 206,
 138                 'icirc'    => 238,
 139                 'iexcl'    => 161,
 140                 'Igrave'   => 204,
 141                 'igrave'   => 236,
 142                 'image'    => 8465,
 143                 'infin'    => 8734,
 144                 'int'      => 8747,
 145                 'Iota'     => 921,
 146                 'iota'     => 953,
 147                 'iquest'   => 191,
 148                 'isin'     => 8712,
 149                 'Iuml'     => 207,
 150                 'iuml'     => 239,
 151                 'Kappa'    => 922,
 152                 'kappa'    => 954,
 153                 'Lambda'   => 923,
 154                 'lambda'   => 955,
 155                 'lang'     => 9001,
 156                 'laquo'    => 171,
 157                 'larr'     => 8592,
 158                 'lArr'     => 8656,
 159                 'lceil'    => 8968,
 160                 'ldquo'    => 8220,
 161                 'le'       => 8804,
 162                 'lfloor'   => 8970,
 163                 'lowast'   => 8727,
 164                 'loz'      => 9674,
 165                 'lrm'      => 8206,
 166                 'lsaquo'   => 8249,
 167                 'lsquo'    => 8216,
 168                 'lt'       => 60,
 169                 'macr'     => 175,
 170                 'mdash'    => 8212,
 171                 'micro'    => 181,
 172                 'middot'   => 183,
 173                 'minus'    => 8722,
 174                 'Mu'       => 924,
 175                 'mu'       => 956,
 176                 'nabla'    => 8711,
 177                 'nbsp'     => 160,
 178                 'ndash'    => 8211,
 179                 'ne'       => 8800,
 180                 'ni'       => 8715,
 181                 'not'      => 172,
 182                 'notin'    => 8713,
 183                 'nsub'     => 8836,
 184                 'Ntilde'   => 209,
 185                 'ntilde'   => 241,
 186                 'Nu'       => 925,
 187                 'nu'       => 957,
 188                 'Oacute'   => 211,
 189                 'oacute'   => 243,
 190                 'Ocirc'    => 212,
 191                 'ocirc'    => 244,
 192                 'OElig'    => 338,
 193                 'oelig'    => 339,
 194                 'Ograve'   => 210,
 195                 'ograve'   => 242,
 196                 'oline'    => 8254,
 197                 'Omega'    => 937,
 198                 'omega'    => 969,
 199                 'Omicron'  => 927,
 200                 'omicron'  => 959,
 201                 'oplus'    => 8853,
 202                 'or'       => 8744,
 203                 'ordf'     => 170,
 204                 'ordm'     => 186,
 205                 'Oslash'   => 216,
 206                 'oslash'   => 248,
 207                 'Otilde'   => 213,
 208                 'otilde'   => 245,
 209                 'otimes'   => 8855,
 210                 'Ouml'     => 214,
 211                 'ouml'     => 246,
 212                 'para'     => 182,
 213                 'part'     => 8706,
 214                 'permil'   => 8240,
 215                 'perp'     => 8869,
 216                 'Phi'      => 934,
 217                 'phi'      => 966,
 218                 'Pi'       => 928,
 219                 'pi'       => 960,
 220                 'piv'      => 982,
 221                 'plusmn'   => 177,
 222                 'pound'    => 163,
 223                 'prime'    => 8242,
 224                 'Prime'    => 8243,
 225                 'prod'     => 8719,
 226                 'prop'     => 8733,
 227                 'Psi'      => 936,
 228                 'psi'      => 968,
 229                 'quot'     => 34,
 230                 'radic'    => 8730,
 231                 'rang'     => 9002,
 232                 'raquo'    => 187,
 233                 'rarr'     => 8594,
 234                 'rArr'     => 8658,
 235                 'rceil'    => 8969,
 236                 'rdquo'    => 8221,
 237                 'real'     => 8476,
 238                 'reg'      => 174,
 239                 'rfloor'   => 8971,
 240                 'Rho'      => 929,
 241                 'rho'      => 961,
 242                 'rlm'      => 8207,
 243                 'rsaquo'   => 8250,
 244                 'rsquo'    => 8217,
 245                 'sbquo'    => 8218,
 246                 'Scaron'   => 352,
 247                 'scaron'   => 353,
 248                 'sdot'     => 8901,
 249                 'sect'     => 167,
 250                 'shy'      => 173,
 251                 'Sigma'    => 931,
 252                 'sigma'    => 963,
 253                 'sigmaf'   => 962,
 254                 'sim'      => 8764,
 255                 'spades'   => 9824,
 256                 'sub'      => 8834,
 257                 'sube'     => 8838,
 258                 'sum'      => 8721,
 259                 'sup'      => 8835,
 260                 'sup1'     => 185,
 261                 'sup2'     => 178,
 262                 'sup3'     => 179,
 263                 'supe'     => 8839,
 264                 'szlig'    => 223,
 265                 'Tau'      => 932,
 266                 'tau'      => 964,
 267                 'there4'   => 8756,
 268                 'Theta'    => 920,
 269                 'theta'    => 952,
 270                 'thetasym' => 977,
 271                 'thinsp'   => 8201,
 272                 'THORN'    => 222,
 273                 'thorn'    => 254,
 274                 'tilde'    => 732,
 275                 'times'    => 215,
 276                 'trade'    => 8482,
 277                 'Uacute'   => 218,
 278                 'uacute'   => 250,
 279                 'uarr'     => 8593,
 280                 'uArr'     => 8657,
 281                 'Ucirc'    => 219,
 282                 'ucirc'    => 251,
 283                 'Ugrave'   => 217,
 284                 'ugrave'   => 249,
 285                 'uml'      => 168,
 286                 'upsih'    => 978,
 287                 'Upsilon'  => 933,
 288                 'upsilon'  => 965,
 289                 'Uuml'     => 220,
 290                 'uuml'     => 252,
 291                 'weierp'   => 8472,
 292                 'Xi'       => 926,
 293                 'xi'       => 958,
 294                 'Yacute'   => 221,
 295                 'yacute'   => 253,
 296                 'yen'      => 165,
 297                 'Yuml'     => 376,
 298                 'yuml'     => 255,
 299                 'Zeta'     => 918,
 300                 'zeta'     => 950,
 301                 'zwj'      => 8205,
 302                 'zwnj'     => 8204
 303         );
 304
 305         /**
 306          * Character entity aliases accepted by MediaWiki
 307          */
 308         static $htmlEntityAliases = array(
 309                 'רלמ' => 'rlm',
 310                 'رلم' => 'rlm',
 311         );
 312
 313         /**
 314          * Lazy-initialised attributes regex, see getAttribsRegex()
 315          */
 316         static $attribsRegex;
 317
 318         /**
 319          * Regular expression to match HTML/XML attribute pairs within a tag.
 320          * Allows some... latitude.
 321          * Used in Sanitizer::fixTagAttributes and Sanitizer::decodeTagAttributes
 322          */
 323         static function getAttribsRegex() {
 324                 if ( self::$attribsRegex === null ) {
 325                         $attribFirst = '[:A-Z_a-z0-9]';
 326                         $attrib = '[:A-Z_a-z-.0-9]';
 327                         $space = '[\x09\x0a\x0d\x20]';
 328                         self::$attribsRegex =
 329                                 "/(?:^|$space)({$attribFirst}{$attrib}*)
 330                                   ($space*=$space*
 331                                         (?:
 332                                          # The attribute value: quoted or alone
 333                                           \"([^<\"]*)\"
 334                                          | '([^<']*)'
 335                                          |  ([a-zA-Z0-9!#$%&()*,\\-.\\/:;<>?@[\\]^_`{|}~]+)
 336                                          |  (\#[0-9a-fA-F]+) # Technically wrong, but lots of
 337                                                                                  # colors are specified like this.
 338                                                                                  # We'll be normalizing it.
 339                                         )
 340                                 )?(?=$space|\$)/sx";
 341                 }
 342                 return self::$attribsRegex;
 343         }
 344
 345         /**
 346          * Cleans up HTML, removes dangerous tags and attributes, and
 347          * removes HTML comments
 348          * @private
 349          * @param $text String
 350          * @param $processCallback Callback to do any variable or parameter replacements in HTML attribute values
 351          * @param $args Array for the processing callback
 352          * @param $extratags Array for any extra tags to include
 353          * @param $removetags Array for any tags (default or extra) to exclude
 354          * @return string
 355          */
 356         static function removeHTMLtags( $text, $processCallback = null, $args = array(), $extratags = array(), $removetags = array() ) {
 357                 global $wgUseTidy;
 358
 359                 static $htmlpairsStatic, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
 360                         $htmllist, $listtags, $htmlsingleallowed, $htmlelementsStatic, $staticInitialised;
 361
 362                 wfProfileIn( __METHOD__ );
 363
 364                 if ( !$staticInitialised ) {
 365
 366                         $htmlpairsStatic = array( # Tags that must be closed
 367                                 'b', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
 368                                 'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
 369                                 'strike', 'strong', 'tt', 'var', 'div', 'center',
 370                                 'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
 371                                 'ruby', 'rt' , 'rb' , 'rp', 'p', 'span', 'abbr', 'dfn',
 372                                 'kbd', 'samp'
 373                         );
 374                         $htmlsingle = array(
 375                                 'br', 'hr', 'li', 'dt', 'dd'
 376                         );
 377                         $htmlsingleonly = array( # Elements that cannot have close tags
 378                                 'br', 'hr'
 379                         );
 380                         $htmlnest = array( # Tags that can be nested--??
 381                                 'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
 382                                 'dl', 'font', 'big', 'small', 'sub', 'sup', 'span'
 383                         );
 384                         $tabletags = array( # Can only appear inside table, we will close them
 385                                 'td', 'th', 'tr',
 386                         );
 387                         $htmllist = array( # Tags used by list
 388                                 'ul','ol',
 389                         );
 390                         $listtags = array( # Tags that can appear in a list
 391                                 'li',
 392                         );
 393
 394                         global $wgAllowImageTag;
 395                         if ( $wgAllowImageTag ) {
 396                                 $htmlsingle[] = 'img';
 397                                 $htmlsingleonly[] = 'img';
 398                         }
 399
 400                         $htmlsingleallowed = array_unique( array_merge( $htmlsingle, $tabletags ) );
 401                         $htmlelementsStatic = array_unique( array_merge( $htmlsingle, $htmlpairsStatic, $htmlnest ) );
 402
 403                         # Convert them all to hashtables for faster lookup
 404                         $vars = array( 'htmlpairsStatic', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
 405                                 'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelementsStatic' );
 406                         foreach ( $vars as $var ) {
 407                                 $$var = array_flip( $$var );
 408                         }
 409                         $staticInitialised = true;
 410                 }
 411                 # Populate $htmlpairs and $htmlelements with the $extratags and $removetags arrays
 412                 $extratags = array_flip( $extratags );
 413                 $removetags = array_flip( $removetags );
 414                 $htmlpairs = array_merge( $extratags, $htmlpairsStatic );
 415                 $htmlelements = array_diff_key( array_merge( $extratags, $htmlelementsStatic ) , $removetags );
 416
 417                 # Remove HTML comments
 418                 $text = Sanitizer::removeHTMLcomments( $text );
 419                 $bits = explode( '<', $text );
 420                 $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 421                 if ( !$wgUseTidy ) {
 422                         $tagstack = $tablestack = array();
 423                         foreach ( $bits as $x ) {
 424                                 $regs = array();
 425                                 # $slash: Does the current element start with a '/'?
 426                                 # $t: Current element name
 427                                 # $params: String between element name and >
 428                                 # $brace: Ending '>' or '/>'
 429                                 # $rest: Everything until the next element of $bits
 430                                 if( preg_match( '!^(/?)(\\w+)([^>]*?)(/{0,1}>)([^<]*)$!', $x, $regs ) ) {
 431                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 432                                 } else {
 433                                         $slash = $t = $params = $brace = $rest = null;
 434                                 }
 435
 436                                 $badtag = false;
 437                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 438                                         # Check our stack
 439                                         if ( $slash && isset( $htmlsingleonly[$t] ) ) {
 440                                                 $badtag = true;
 441                                         } elseif ( $slash ) {
 442                                                 # Closing a tag... is it the one we just opened?
 443                                                 $ot = @array_pop( $tagstack );
 444                                                 if ( $ot != $t ) {
 445                                                         if ( isset( $htmlsingleallowed[$ot] ) ) {
 446                                                                 # Pop all elements with an optional close tag
 447                                                                 # and see if we find a match below them
 448                                                                 $optstack = array();
 449                                                                 array_push( $optstack, $ot );
 450                                                                 $ot = @array_pop( $tagstack );
 451                                                                 while ( $ot != $t && isset( $htmlsingleallowed[$ot] ) ) {
 452                                                                         array_push( $optstack, $ot );
 453                                                                         $ot = @array_pop( $tagstack );
 454                                                                 }
 455                                                                 if ( $t != $ot ) {
 456                                                                         # No match. Push the optional elements back again
 457                                                                         $badtag = true;
 458                                                                         while ( $ot = @array_pop( $optstack ) ) {
 459                                                                                 array_push( $tagstack, $ot );
 460                                                                         }
 461                                                                 }
 462                                                         } else {
 463                                                                 @array_push( $tagstack, $ot );
 464                                                                 # <li> can be nested in <ul> or <ol>, skip those cases:
 465                                                                 if ( !isset( $htmllist[$ot] ) || !isset( $listtags[$t] ) ) {
 466                                                                         $badtag = true;
 467                                                                 }
 468                                                         }
 469                                                 } else {
 470                                                         if ( $t == 'table' ) {
 471                                                                 $tagstack = array_pop( $tablestack );
 472                                                         }
 473                                                 }
 474                                                 $newparams = '';
 475                                         } else {
 476                                                 # Keep track for later
 477                                                 if ( isset( $tabletags[$t] ) &&
 478                                                 !in_array( 'table', $tagstack ) ) {
 479                                                         $badtag = true;
 480                                                 } elseif ( in_array( $t, $tagstack ) &&
 481                                                 !isset( $htmlnest [$t ] ) ) {
 482                                                         $badtag = true;
 483                                                 # Is it a self closed htmlpair ? (bug 5487)
 484                                                 } elseif ( $brace == '/>' &&
 485                                                 isset( $htmlpairs[$t] ) ) {
 486                                                         $badtag = true;
 487                                                 } elseif ( isset( $htmlsingleonly[$t] ) ) {
 488                                                         # Hack to force empty tag for uncloseable elements
 489                                                         $brace = '/>';
 490                                                 } elseif ( isset( $htmlsingle[$t] ) ) {
 491                                                         # Hack to not close $htmlsingle tags
 492                                                         $brace = null;
 493                                                 } elseif ( isset( $tabletags[$t] )
 494                                                 && in_array( $t, $tagstack ) ) {
 495                                                         // New table tag but forgot to close the previous one
 496                                                         $text .= "</$t>";
 497                                                 } else {
 498                                                         if ( $t == 'table' ) {
 499                                                                 array_push( $tablestack, $tagstack );
 500                                                                 $tagstack = array();
 501                                                         }
 502                                                         array_push( $tagstack, $t );
 503                                                 }
 504
 505                                                 # Replace any variables or template parameters with
 506                                                 # plaintext results.
 507                                                 if( is_callable( $processCallback ) ) {
 508                                                         call_user_func_array( $processCallback, array( &$params, $args ) );
 509                                                 }
 510
 511                                                 # Strip non-approved attributes from the tag
 512                                                 $newparams = Sanitizer::fixTagAttributes( $params, $t );
 513                                         }
 514                                         if ( !$badtag ) {
 515                                                 $rest = str_replace( '>', '&gt;', $rest );
 516                                                 $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
 517                                                 $text .= "<$slash$t$newparams$close>$rest";
 518                                                 continue;
 519                                         }
 520                                 }
 521                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 522                         }
 523                         # Close off any remaining tags
 524                         while ( is_array( $tagstack ) && ($t = array_pop( $tagstack )) ) {
 525                                 $text .= "</$t>\n";
 526                                 if ( $t == 'table' ) { $tagstack = array_pop( $tablestack ); }
 527                         }
 528                 } else {
 529                         # this might be possible using tidy itself
 530                         foreach ( $bits as $x ) {
 531                                 preg_match( '/^(\\/?)(\\w+)([^>]*?)(\\/{0,1}>)([^<]*)$/',
 532                                 $x, $regs );
 533                                 @list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 534                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 535                                         if( is_callable( $processCallback ) ) {
 536                                                 call_user_func_array( $processCallback, array( &$params, $args ) );
 537                                         }
 538                                         $newparams = Sanitizer::fixTagAttributes( $params, $t );
 539                                         $rest = str_replace( '>', '&gt;', $rest );
 540                                         $text .= "<$slash$t$newparams$brace$rest";
 541                                 } else {
 542                                         $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 543                                 }
 544                         }
 545                 }
 546                 wfProfileOut( __METHOD__ );
 547                 return $text;
 548         }
 549
 550         /**
 551          * Remove '<!--', '-->', and everything between.
 552          * To avoid leaving blank lines, when a comment is both preceded
 553          * and followed by a newline (ignoring spaces), trim leading and
 554          * trailing spaces and one of the newlines.
 555          *
 556          * @private
 557          * @param $text String
 558          * @return string
 559          */
 560         static function removeHTMLcomments( $text ) {
 561                 wfProfileIn( __METHOD__ );
 562                 while (($start = strpos($text, '<!--')) !== false) {
 563                         $end = strpos($text, '-->', $start + 4);
 564                         if ($end === false) {
 565                                 # Unterminated comment; bail out
 566                                 break;
 567                         }
 568
 569                         $end += 3;
 570
 571                         # Trim space and newline if the comment is both
 572                         # preceded and followed by a newline
 573                         $spaceStart = max($start - 1, 0);
 574                         $spaceLen = $end - $spaceStart;
 575                         while (substr($text, $spaceStart, 1) === ' ' && $spaceStart > 0) {
 576                                 $spaceStart--;
 577                                 $spaceLen++;
 578                         }
 579                         while (substr($text, $spaceStart + $spaceLen, 1) === ' ')
 580                                 $spaceLen++;
 581                         if (substr($text, $spaceStart, 1) === "\n" and substr($text, $spaceStart + $spaceLen, 1) === "\n") {
 582                                 # Remove the comment, leading and trailing
 583                                 # spaces, and leave only one newline.
 584                                 $text = substr_replace($text, "\n", $spaceStart, $spaceLen + 1);
 585                         }
 586                         else {
 587                                 # Remove just the comment.
 588                                 $text = substr_replace($text, '', $start, $end - $start);
 589                         }
 590                 }
 591                 wfProfileOut( __METHOD__ );
 592                 return $text;
 593         }
 594
 595         /**
 596          * Take an array of attribute names and values and normalize or discard
 597          * illegal values for the given element type.
 598          *
 599          * - Discards attributes not on a whitelist for the given element
 600          * - Unsafe style attributes are discarded
 601          * - Invalid id attributes are reencoded
 602          *
 603          * @param $attribs Array
 604          * @param $element String
 605          * @return Array
 606          *
 607          * @todo Check for legal values where the DTD limits things.
 608          * @todo Check for unique id attribute :P
 609          */
 610         static function validateTagAttributes( $attribs, $element ) {
 611                 return Sanitizer::validateAttributes( $attribs,
 612                         Sanitizer::attributeWhitelist( $element ) );
 613         }
 614
 615         /**
 616          * Take an array of attribute names and values and normalize or discard
 617          * illegal values for the given whitelist.
 618          *
 619          * - Discards attributes not the given whitelist
 620          * - Unsafe style attributes are discarded
 621          * - Invalid id attributes are reencoded
 622          *
 623          * @param $attribs Array
 624          * @param $whitelist Array: list of allowed attribute names
 625          * @return Array
 626          *
 627          * @todo Check for legal values where the DTD limits things.
 628          * @todo Check for unique id attribute :P
 629          */
 630         static function validateAttributes( $attribs, $whitelist ) {
 631                 global $wgAllowRdfaAttributes, $wgAllowMicrodataAttributes, $wgHtml5;
 632
 633                 $whitelist = array_flip( $whitelist );
 634                 $hrefExp = '/^(' . wfUrlProtocols() . ')[^\s]+$/';
 635
 636                 $out = array();
 637                 foreach( $attribs as $attribute => $value ) {
 638                         #allow XML namespace declaration if RDFa is enabled
 639                         if ( $wgAllowRdfaAttributes && preg_match( self::XMLNS_ATTRIBUTE_PATTERN, $attribute ) ) {
 640                                 if ( !preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 641                                         $out[$attribute] = $value;
 642                                 }
 643
 644                                 continue;
 645                         }
 646
 647                         # Allow any attribute beginning with "data-", if in HTML5 mode
 648                         if ( !($wgHtml5 && preg_match( '/^data-/i', $attribute )) && !isset( $whitelist[$attribute] ) ) {
 649                                 continue;
 650                         }
 651
 652                         # Strip javascript "expression" from stylesheets.
 653                         # http://msdn.microsoft.com/workshop/author/dhtml/overview/recalc.asp
 654                         if( $attribute == 'style' ) {
 655                                 $value = Sanitizer::checkCss( $value );
 656                         }
 657
 658                         if ( $attribute === 'id' ) {
 659                                 $value = Sanitizer::escapeId( $value, 'noninitial' );
 660                         }
 661
 662                         //RDFa and microdata properties allow URLs, URIs and/or CURIs. check them for sanity
 663                         if ( $attribute === 'rel' || $attribute === 'rev' ||
 664                                 $attribute === 'about' || $attribute === 'property' || $attribute === 'resource' || #RDFa
 665                                 $attribute === 'datatype' || $attribute === 'typeof' ||                             #RDFa
 666                                 $attribute === 'itemid' || $attribute === 'itemprop' || $attribute === 'itemref' || #HTML5 microdata
 667                                 $attribute === 'itemscope' || $attribute === 'itemtype' ) {                         #HTML5 microdata
 668
 669                                 //Paranoia. Allow "simple" values but suppress javascript
 670                                 if ( preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 671                                         continue;
 672                                 }
 673                         }
 674
 675                         # NOTE: even though elements using href/src are not allowed directly, supply
 676                         #       validation code that can be used by tag hook handlers, etc
 677                         if ( $attribute === 'href' || $attribute === 'src' ) {
 678                                 if ( !preg_match( $hrefExp, $value ) ) {
 679                                         continue; //drop any href or src attributes not using an allowed protocol.
 680                                                   //NOTE: this also drops all relative URLs
 681                                 }
 682                         }
 683
 684                         // If this attribute was previously set, override it.
 685                         // Output should only have one attribute of each name.
 686                         $out[$attribute] = $value;
 687                 }
 688
 689                 if ( $wgAllowMicrodataAttributes ) {
 690                         # itemtype, itemid, itemref don't make sense without itemscope
 691                         if ( !array_key_exists( 'itemscope', $out ) ) {
 692                                 unset( $out['itemtype'] );
 693                                 unset( $out['itemid'] );
 694                                 unset( $out['itemref'] );
 695                         }
 696                         # TODO: Strip itemprop if we aren't descendants of an itemscope.
 697                 }
 698                 return $out;
 699         }
 700
 701         /**
 702          * Merge two sets of HTML attributes.  Conflicting items in the second set
 703          * will override those in the first, except for 'class' attributes which
 704          * will be combined (if they're both strings).
 705          *
 706          * @todo implement merging for other attributes such as style
 707          * @param $a Array
 708          * @param $b Array
 709          * @return array
 710          */
 711         static function mergeAttributes( $a, $b ) {
 712                 $out = array_merge( $a, $b );
 713                 if( isset( $a['class'] ) && isset( $b['class'] )
 714                 && is_string( $a['class'] ) && is_string( $b['class'] )
 715                 && $a['class'] !== $b['class'] ) {
 716                         $classes = preg_split( '/\s+/', "{$a['class']} {$b['class']}",
 717                                 -1, PREG_SPLIT_NO_EMPTY );
 718                         $out['class'] = implode( ' ', array_unique( $classes ) );
 719                 }
 720                 return $out;
 721         }
 722
 723         /**
 724          * Pick apart some CSS and check it for forbidden or unsafe structures.
 725          * Returns a sanitized string. This sanitized string will have
 726          * character references and escape sequences decoded, and comments
 727          * stripped. If the input is just too evil, only a comment complaining
 728          * about evilness will be returned.
 729          *
 730          * Currently URL references, 'expression', 'tps' are forbidden.
 731          *
 732          * NOTE: Despite the fact that character references are decoded, the
 733          * returned string may contain character references given certain
 734          * clever input strings. These character references must
 735          * be escaped before the return value is embedded in HTML.
 736          *
 737          * @param $value String
 738          * @return String
 739          */
 740         static function checkCss( $value ) {
 741                 // Decode character references like &#123;
 742                 $value = Sanitizer::decodeCharReferences( $value );
 743
 744                 // Decode escape sequences and line continuation
 745                 // See the grammar in the CSS 2 spec, appendix D.
 746                 // This has to be done AFTER decoding character references.
 747                 // This means it isn't possible for this function to return
 748                 // unsanitized escape sequences. It is possible to manufacture
 749                 // input that contains character references that decode to
 750                 // escape sequences that decode to character references, but
 751                 // it's OK for the return value to contain character references
 752                 // because the caller is supposed to escape those anyway.
 753                 static $decodeRegex;
 754                 if ( !$decodeRegex ) {
 755                         $space = '[\\x20\\t\\r\\n\\f]';
 756                         $nl = '(?:\\n|\\r\\n|\\r|\\f)';
 757                         $backslash = '\\\\';
 758                         $decodeRegex = "/ $backslash
 759                                 (?:
 760                                         ($nl) |  # 1. Line continuation
 761                                         ([0-9A-Fa-f]{1,6})$space? |  # 2. character number
 762                                         (.) | # 3. backslash cancelling special meaning
 763                                         () | # 4. backslash at end of string
 764                                 )/xu";
 765                 }
 766                 $value = preg_replace_callback( $decodeRegex,
 767                         array( __CLASS__, 'cssDecodeCallback' ), $value );
 768
 769                 // Remove any comments; IE gets token splitting wrong
 770                 // This must be done AFTER decoding character references and
 771                 // escape sequences, because those steps can introduce comments
 772                 // This step cannot introduce character references or escape
 773                 // sequences, because it replaces comments with spaces rather
 774                 // than removing them completely.
 775                 $value = StringUtils::delimiterReplace( '/*', '*/', ' ', $value );
 776
 777                 // Remove anything after a comment-start token, to guard against
 778                 // incorrect client implementations.
 779                 $commentPos = strpos( $value, '/*' );
 780                 if ( $commentPos !== false ) {
 781                         $value = substr( $value, 0, $commentPos );
 782                 }
 783
 784                 // Reject problematic keywords and control characters
 785                 if ( preg_match( '/[\000-\010\016-\037\177]/', $value ) ) {
 786                         return '/* invalid control char */';
 787                 } elseif ( preg_match( '! expression | filter\s*: | accelerator\s*: | url\s*\( !ix', $value ) ) {
 788                         return '/* insecure input */';
 789                 }
 790                 return $value;
 791         }
 792
 793         static function cssDecodeCallback( $matches ) {
 794                 if ( $matches[1] !== '' ) {
 795                         // Line continuation
 796                         return '';
 797                 } elseif ( $matches[2] !== '' ) {
 798                         $char = codepointToUtf8( hexdec( $matches[2] ) );
 799                 } elseif ( $matches[3] !== '' ) {
 800                         $char = $matches[3];
 801                 } else {
 802                         $char = '\\';
 803                 }
 804                 if ( $char == "\n" || $char == '"' || $char == "'" || $char == '\\' ) {
 805                         // These characters need to be escaped in strings
 806                         // Clean up the escape sequence to avoid parsing errors by clients
 807                         return '\\' . dechex( ord( $char ) ) . ' ';
 808                 } else {
 809                         // Decode unnecessary escape
 810                         return $char;
 811                 }
 812         }
 813
 814         /**
 815          * Take a tag soup fragment listing an HTML element's attributes
 816          * and normalize it to well-formed XML, discarding unwanted attributes.
 817          * Output is safe for further wikitext processing, with escaping of
 818          * values that could trigger problems.
 819          *
 820          * - Normalizes attribute names to lowercase
 821          * - Discards attributes not on a whitelist for the given element
 822          * - Turns broken or invalid entities into plaintext
 823          * - Double-quotes all attribute values
 824          * - Attributes without values are given the name as attribute
 825          * - Double attributes are discarded
 826          * - Unsafe style attributes are discarded
 827          * - Prepends space if there are attributes.
 828          *
 829          * @param $text String
 830          * @param $element String
 831          * @return String
 832          */
 833         static function fixTagAttributes( $text, $element ) {
 834                 if( trim( $text ) == '' ) {
 835                         return '';
 836                 }
 837
 838                 $stripped = Sanitizer::validateTagAttributes(
 839                         Sanitizer::decodeTagAttributes( $text ), $element );
 840
 841                 $attribs = array();
 842                 foreach( $stripped as $attribute => $value ) {
 843                         $encAttribute = htmlspecialchars( $attribute );
 844                         $encValue = Sanitizer::safeEncodeAttribute( $value );
 845
 846                         $attribs[] = "$encAttribute=\"$encValue\"";
 847                 }
 848                 return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
 849         }
 850
 851         /**
 852          * Encode an attribute value for HTML output.
 853          * @param $text String
 854          * @return HTML-encoded text fragment
 855          */
 856         static function encodeAttribute( $text ) {
 857                 $encValue = htmlspecialchars( $text, ENT_QUOTES );
 858
 859                 // Whitespace is normalized during attribute decoding,
 860                 // so if we've been passed non-spaces we must encode them
 861                 // ahead of time or they won't be preserved.
 862                 $encValue = strtr( $encValue, array(
 863                         "\n" => '&#10;',
 864                         "\r" => '&#13;',
 865                         "\t" => '&#9;',
 866                 ) );
 867
 868                 return $encValue;
 869         }
 870
 871         /**
 872          * Encode an attribute value for HTML tags, with extra armoring
 873          * against further wiki processing.
 874          * @param $text String
 875          * @return HTML-encoded text fragment
 876          */
 877         static function safeEncodeAttribute( $text ) {
 878                 $encValue = Sanitizer::encodeAttribute( $text );
 879
 880                 # Templates and links may be expanded in later parsing,
 881                 # creating invalid or dangerous output. Suppress this.
 882                 $encValue = strtr( $encValue, array(
 883                         '<'    => '&lt;',   // This should never happen,
 884                         '>'    => '&gt;',   // we've received invalid input
 885                         '"'    => '&quot;', // which should have been escaped.
 886                         '{'    => '&#123;',
 887                         '['    => '&#91;',
 888                         "''"   => '&#39;&#39;',
 889                         'ISBN' => '&#73;SBN',
 890                         'RFC'  => '&#82;FC',
 891                         'PMID' => '&#80;MID',
 892                         '|'    => '&#124;',
 893                         '__'   => '&#95;_',
 894                 ) );
 895
 896                 # Stupid hack
 897                 $encValue = preg_replace_callback(
 898                         '/(' . wfUrlProtocols() . ')/',
 899                         array( 'Sanitizer', 'armorLinksCallback' ),
 900                         $encValue );
 901                 return $encValue;
 902         }
 903
 904         /**
 905          * Given a value, escape it so that it can be used in an id attribute and
 906          * return it.  This will use HTML5 validation if $wgExperimentalHtmlIds is
 907          * true, allowing anything but ASCII whitespace.  Otherwise it will use
 908          * HTML 4 rules, which means a narrow subset of ASCII, with bad characters
 909          * escaped with lots of dots.
 910          *
 911          * To ensure we don't have to bother escaping anything, we also strip ', ",
 912          * & even if $wgExperimentalIds is true.  TODO: Is this the best tactic?
 913          * We also strip # because it upsets IE, and % because it could be
 914          * ambiguous if it's part of something that looks like a percent escape
 915          * (which don't work reliably in fragments cross-browser).
 916          *
 917          * @see http://www.w3.org/TR/html401/types.html#type-name Valid characters
 918          *                                                          in the id and
 919          *                                                          name attributes
 920          * @see http://www.w3.org/TR/html401/struct/links.html#h-12.2.3 Anchors with the id attribute
 921          * @see http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#the-id-attribute
 922          *   HTML5 definition of id attribute
 923          *
 924          * @param $id String: id to escape
 925          * @param $options Mixed: string or array of strings (default is array()):
 926          *   'noninitial': This is a non-initial fragment of an id, not a full id,
 927          *       so don't pay attention if the first character isn't valid at the
 928          *       beginning of an id.  Only matters if $wgExperimentalHtmlIds is
 929          *       false.
 930          *   'legacy': Behave the way the old HTML 4-based ID escaping worked even
 931          *       if $wgExperimentalHtmlIds is used, so we can generate extra
 932          *       anchors and links won't break.
 933          * @return String
 934          */
 935         static function escapeId( $id, $options = array() ) {
 936                 global $wgHtml5, $wgExperimentalHtmlIds;
 937                 $options = (array)$options;
 938
 939                 if ( $wgHtml5 && $wgExperimentalHtmlIds && !in_array( 'legacy', $options ) ) {
 940                         $id = Sanitizer::decodeCharReferences( $id );
 941                         $id = preg_replace( '/[ \t\n\r\f_\'"&#%]+/', '_', $id );
 942                         $id = trim( $id, '_' );
 943                         if ( $id === '' ) {
 944                                 # Must have been all whitespace to start with.
 945                                 return '_';
 946                         } else {
 947                                 return $id;
 948                         }
 949                 }
 950
 951                 # HTML4-style escaping
 952                 static $replace = array(
 953                         '%3A' => ':',
 954                         '%' => '.'
 955                 );
 956
 957                 $id = urlencode( Sanitizer::decodeCharReferences( strtr( $id, ' ', '_' ) ) );
 958                 $id = str_replace( array_keys( $replace ), array_values( $replace ), $id );
 959
 960                 if ( !preg_match( '/^[a-zA-Z]/', $id )
 961                 && !in_array( 'noninitial', $options ) )  {
 962                         // Initial character must be a letter!
 963                         $id = "x$id";
 964                 }
 965                 return $id;
 966         }
 967
 968         /**
 969          * Given a value, escape it so that it can be used as a CSS class and
 970          * return it.
 971          *
 972          * @todo For extra validity, input should be validated UTF-8.
 973          *
 974          * @see http://www.w3.org/TR/CSS21/syndata.html Valid characters/format
 975          *
 976          * @param $class String
 977          * @return String
 978          */
 979         static function escapeClass( $class ) {
 980                 // Convert ugly stuff to underscores and kill underscores in ugly places
 981                 return rtrim(preg_replace(
 982                         array('/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/','/_+/'),
 983                         '_',
 984                         $class ), '_');
 985         }
 986
 987         /**
 988          * Given HTML input, escape with htmlspecialchars but un-escape entites.
 989          * This allows (generally harmless) entities like &#160; to survive.
 990          *
 991          * @param $html String to escape
 992          * @return String: escaped input
 993          */
 994         static function escapeHtmlAllowEntities( $html ) {
 995                 $html = Sanitizer::decodeCharReferences( $html );
 996                 # It seems wise to escape ' as well as ", as a matter of course.  Can't
 997                 # hurt.
 998                 $html = htmlspecialchars( $html, ENT_QUOTES );
 999                 return $html;
1000         }
1001
1002         /**
1003          * Regex replace callback for armoring links against further processing.
1004          * @param $matches Array
1005          * @return string
1006          */
1007         private static function armorLinksCallback( $matches ) {
1008                 return str_replace( ':', '&#58;', $matches[1] );
1009         }
1010
1011         /**
1012          * Return an associative array of attribute names and values from
1013          * a partial tag string. Attribute names are forces to lowercase,
1014          * character references are decoded to UTF-8 text.
1015          *
1016          * @param $text String
1017          * @return Array
1018          */
1019         public static function decodeTagAttributes( $text ) {
1020                 if( trim( $text ) == '' ) {
1021                         return array();
1022                 }
1023
1024                 $attribs = array();
1025                 $pairs = array();
1026                 if( !preg_match_all(
1027                         self::getAttribsRegex(),
1028                         $text,
1029                         $pairs,
1030                         PREG_SET_ORDER ) ) {
1031                         return $attribs;
1032                 }
1033
1034                 foreach( $pairs as $set ) {
1035                         $attribute = strtolower( $set[1] );
1036                         $value = Sanitizer::getTagAttributeCallback( $set );
1037
1038                         // Normalize whitespace
1039                         $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
1040                         $value = trim( $value );
1041
1042                         // Decode character references
1043                         $attribs[$attribute] = Sanitizer::decodeCharReferences( $value );
1044                 }
1045                 return $attribs;
1046         }
1047
1048         /**
1049          * Pick the appropriate attribute value from a match set from the
1050          * attribs regex matches.
1051          *
1052          * @param $set Array
1053          * @return String
1054          */
1055         private static function getTagAttributeCallback( $set ) {
1056                 if( isset( $set[6] ) ) {
1057                         # Illegal #XXXXXX color with no quotes.
1058                         return $set[6];
1059                 } elseif( isset( $set[5] ) ) {
1060                         # No quotes.
1061                         return $set[5];
1062                 } elseif( isset( $set[4] ) ) {
1063                         # Single-quoted
1064                         return $set[4];
1065                 } elseif( isset( $set[3] ) ) {
1066                         # Double-quoted
1067                         return $set[3];
1068                 } elseif( !isset( $set[2] ) ) {
1069                         # In XHTML, attributes must have a value.
1070                         # For 'reduced' form, return explicitly the attribute name here.
1071                         return $set[1];
1072                 } else {
1073                         throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
1074                 }
1075         }
1076
1077         /**
1078          * Normalize whitespace and character references in an XML source-
1079          * encoded text for an attribute value.
1080          *
1081          * See http://www.w3.org/TR/REC-xml/#AVNormalize for background,
1082          * but note that we're not returning the value, but are returning
1083          * XML source fragments that will be slapped into output.
1084          *
1085          * @param $text String
1086          * @return String
1087          */
1088         private static function normalizeAttributeValue( $text ) {
1089                 return str_replace( '"', '&quot;',
1090                         self::normalizeWhitespace(
1091                                 Sanitizer::normalizeCharReferences( $text ) ) );
1092         }
1093
1094         private static function normalizeWhitespace( $text ) {
1095                 return preg_replace(
1096                         '/\r\n|[\x20\x0d\x0a\x09]/',
1097                         ' ',
1098                         $text );
1099         }
1100
1101         /**
1102          * Normalizes whitespace in a section name, such as might be returned
1103          * by Parser::stripSectionName(), for use in the id's that are used for
1104          * section links.
1105          *
1106          * @param $section String
1107          * @return String
1108          */
1109         static function normalizeSectionNameWhitespace( $section ) {
1110                 return trim( preg_replace( '/[ _]+/', ' ', $section ) );
1111         }
1112
1113         /**
1114          * Ensure that any entities and character references are legal
1115          * for XML and XHTML specifically. Any stray bits will be
1116          * &amp;-escaped to result in a valid text fragment.
1117          *
1118          * a. named char refs can only be &lt; &gt; &amp; &quot;, others are
1119          *   numericized (this way we're well-formed even without a DTD)
1120          * b. any numeric char refs must be legal chars, not invalid or forbidden
1121          * c. use &#x, not &#X
1122          * d. fix or reject non-valid attributes
1123          *
1124          * @param $text String
1125          * @return String
1126          * @private
1127          */
1128         static function normalizeCharReferences( $text ) {
1129                 return preg_replace_callback(
1130                         self::CHAR_REFS_REGEX,
1131                         array( 'Sanitizer', 'normalizeCharReferencesCallback' ),
1132                         $text );
1133         }
1134         /**
1135          * @param $matches String
1136          * @return String
1137          */
1138         static function normalizeCharReferencesCallback( $matches ) {
1139                 $ret = null;
1140                 if( $matches[1] != '' ) {
1141                         $ret = Sanitizer::normalizeEntity( $matches[1] );
1142                 } elseif( $matches[2] != '' ) {
1143                         $ret = Sanitizer::decCharReference( $matches[2] );
1144                 } elseif( $matches[3] != ''  ) {
1145                         $ret = Sanitizer::hexCharReference( $matches[3] );
1146                 }
1147                 if( is_null( $ret ) ) {
1148                         return htmlspecialchars( $matches[0] );
1149                 } else {
1150                         return $ret;
1151                 }
1152         }
1153
1154         /**
1155          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1156          * return the equivalent numeric entity reference (except for the core &lt;
1157          * &gt; &amp; &quot;). If the entity is a MediaWiki-specific alias, returns
1158          * the HTML equivalent. Otherwise, returns HTML-escaped text of
1159          * pseudo-entity source (eg &amp;foo;)
1160          *
1161          * @param $name String
1162          * @return String
1163          */
1164         static function normalizeEntity( $name ) {
1165                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1166                         return '&' . self::$htmlEntityAliases[$name] . ';';
1167                 } elseif ( in_array( $name,
1168                 array( 'lt', 'gt', 'amp', 'quot' ) ) ) {
1169                         return "&$name;";
1170                 } elseif ( isset( self::$htmlEntities[$name] ) ) {
1171                         return '&#' . self::$htmlEntities[$name] . ';';
1172                 } else {
1173                         return "&amp;$name;";
1174                 }
1175         }
1176
1177         static function decCharReference( $codepoint ) {
1178                 $point = intval( $codepoint );
1179                 if( Sanitizer::validateCodepoint( $point ) ) {
1180                         return sprintf( '&#%d;', $point );
1181                 } else {
1182                         return null;
1183                 }
1184         }
1185
1186         static function hexCharReference( $codepoint ) {
1187                 $point = hexdec( $codepoint );
1188                 if( Sanitizer::validateCodepoint( $point ) ) {
1189                         return sprintf( '&#x%x;', $point );
1190                 } else {
1191                         return null;
1192                 }
1193         }
1194
1195         /**
1196          * Returns true if a given Unicode codepoint is a valid character in XML.
1197          * @param $codepoint Integer
1198          * @return Boolean
1199          */
1200         private static function validateCodepoint( $codepoint ) {
1201                 return ($codepoint ==    0x09)
1202                         || ($codepoint ==    0x0a)
1203                         || ($codepoint ==    0x0d)
1204                         || ($codepoint >=    0x20 && $codepoint <=   0xd7ff)
1205                         || ($codepoint >=  0xe000 && $codepoint <=   0xfffd)
1206                         || ($codepoint >= 0x10000 && $codepoint <= 0x10ffff);
1207         }
1208
1209         /**
1210          * Decode any character references, numeric or named entities,
1211          * in the text and return a UTF-8 string.
1212          *
1213          * @param $text String
1214          * @return String
1215          */
1216         public static function decodeCharReferences( $text ) {
1217                 return preg_replace_callback(
1218                         self::CHAR_REFS_REGEX,
1219                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1220                         $text );
1221         }
1222
1223         /**
1224          * Decode any character references, numeric or named entities,
1225          * in the next and normalize the resulting string. (bug 14952)
1226          *
1227          * This is useful for page titles, not for text to be displayed,
1228          * MediaWiki allows HTML entities to escape normalization as a feature.
1229          *
1230          * @param $text String (already normalized, containing entities)
1231          * @return String (still normalized, without entities)
1232          */
1233         public static function decodeCharReferencesAndNormalize( $text ) {
1234                 global $wgContLang;
1235                 $text = preg_replace_callback(
1236                         self::CHAR_REFS_REGEX,
1237                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1238                         $text, /* limit */ -1, $count );
1239
1240                 if ( $count ) {
1241                         return $wgContLang->normalize( $text );
1242                 } else {
1243                         return $text;
1244                 }
1245         }
1246
1247         /**
1248          * @param $matches String
1249          * @return String
1250          */
1251         static function decodeCharReferencesCallback( $matches ) {
1252                 if( $matches[1] != '' ) {
1253                         return Sanitizer::decodeEntity( $matches[1] );
1254                 } elseif( $matches[2] != '' ) {
1255                         return  Sanitizer::decodeChar( intval( $matches[2] ) );
1256                 } elseif( $matches[3] != ''  ) {
1257                         return  Sanitizer::decodeChar( hexdec( $matches[3] ) );
1258                 }
1259                 # Last case should be an ampersand by itself
1260                 return $matches[0];
1261         }
1262
1263         /**
1264          * Return UTF-8 string for a codepoint if that is a valid
1265          * character reference, otherwise U+FFFD REPLACEMENT CHARACTER.
1266          * @param $codepoint Integer
1267          * @return String
1268          * @private
1269          */
1270         static function decodeChar( $codepoint ) {
1271                 if( Sanitizer::validateCodepoint( $codepoint ) ) {
1272                         return codepointToUtf8( $codepoint );
1273                 } else {
1274                         return UTF8_REPLACEMENT;
1275                 }
1276         }
1277
1278         /**
1279          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1280          * return the UTF-8 encoding of that character. Otherwise, returns
1281          * pseudo-entity source (eg &foo;)
1282          *
1283          * @param $name Strings
1284          * @return String
1285          */
1286         static function decodeEntity( $name ) {
1287                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1288                         $name = self::$htmlEntityAliases[$name];
1289                 }
1290                 if( isset( self::$htmlEntities[$name] ) ) {
1291                         return codepointToUtf8( self::$htmlEntities[$name] );
1292                 } else {
1293                         return "&$name;";
1294                 }
1295         }
1296
1297         /**
1298          * Fetch the whitelist of acceptable attributes for a given element name.
1299          *
1300          * @param $element String
1301          * @return Array
1302          */
1303         static function attributeWhitelist( $element ) {
1304                 static $list;
1305                 if( !isset( $list ) ) {
1306                         $list = Sanitizer::setupAttributeWhitelist();
1307                 }
1308                 return isset( $list[$element] )
1309                         ? $list[$element]
1310                         : array();
1311         }
1312
1313         /**
1314          * Foreach array key (an allowed HTML element), return an array
1315          * of allowed attributes
1316          * @return Array
1317          */
1318         static function setupAttributeWhitelist() {
1319                 global $wgAllowRdfaAttributes, $wgHtml5, $wgAllowMicrodataAttributes;
1320
1321                 $common = array( 'id', 'class', 'lang', 'dir', 'title', 'style' );
1322
1323                 if ( $wgAllowRdfaAttributes ) {
1324                         #RDFa attributes as specified in section 9 of http://www.w3.org/TR/2008/REC-rdfa-syntax-20081014
1325                         $common = array_merge( $common, array(
1326                             'about', 'property', 'resource', 'datatype', 'typeof',
1327                         ) );
1328                 }
1329
1330                 if ( $wgHtml5 && $wgAllowMicrodataAttributes ) {
1331                         # add HTML5 microdata tages as pecified by http://www.whatwg.org/specs/web-apps/current-work/multipage/microdata.html#the-microdata-model
1332                         $common = array_merge( $common, array(
1333                             'itemid', 'itemprop', 'itemref', 'itemscope', 'itemtype'
1334                         ) );
1335                 }
1336
1337                 $block = array_merge( $common, array( 'align' ) );
1338                 $tablealign = array( 'align', 'char', 'charoff', 'valign' );
1339                 $tablecell = array( 'abbr',
1340                                     'axis',
1341                                     'headers',
1342                                     'scope',
1343                                     'rowspan',
1344                                     'colspan',
1345                                     'nowrap', # deprecated
1346                                     'width',  # deprecated
1347                                     'height', # deprecated
1348                                     'bgcolor' # deprecated
1349                                     );
1350
1351                 # Numbers refer to sections in HTML 4.01 standard describing the element.
1352                 # See: http://www.w3.org/TR/html4/
1353                 $whitelist = array (
1354                         # 7.5.4
1355                         'div'        => $block,
1356                         'center'     => $common, # deprecated
1357                         'span'       => $block, # ??
1358
1359                         # 7.5.5
1360                         'h1'         => $block,
1361                         'h2'         => $block,
1362                         'h3'         => $block,
1363                         'h4'         => $block,
1364                         'h5'         => $block,
1365                         'h6'         => $block,
1366
1367                         # 7.5.6
1368                         # address
1369
1370                         # 8.2.4
1371                         # bdo
1372
1373                         # 9.2.1
1374                         'em'         => $common,
1375                         'strong'     => $common,
1376                         'cite'       => $common,
1377                         'dfn'        => $common,
1378                         'code'       => $common,
1379                         'samp'       => $common,
1380                         'kbd'        => $common,
1381                         'var'        => $common,
1382                         'abbr'       => $common,
1383                         # acronym
1384
1385                         # 9.2.2
1386                         'blockquote' => array_merge( $common, array( 'cite' ) ),
1387                         # q
1388
1389                         # 9.2.3
1390                         'sub'        => $common,
1391                         'sup'        => $common,
1392
1393                         # 9.3.1
1394                         'p'          => $block,
1395
1396                         # 9.3.2
1397                         'br'         => array( 'id', 'class', 'title', 'style', 'clear' ),
1398
1399                         # 9.3.4
1400                         'pre'        => array_merge( $common, array( 'width' ) ),
1401
1402                         # 9.4
1403                         'ins'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1404                         'del'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1405
1406                         # 10.2
1407                         'ul'         => array_merge( $common, array( 'type' ) ),
1408                         'ol'         => array_merge( $common, array( 'type', 'start' ) ),
1409                         'li'         => array_merge( $common, array( 'type', 'value' ) ),
1410
1411                         # 10.3
1412                         'dl'         => $common,
1413                         'dd'         => $common,
1414                         'dt'         => $common,
1415
1416                         # 11.2.1
1417                         'table'      => array_merge( $common,
1418                                                                 array( 'summary', 'width', 'border', 'frame',
1419                                                                                 'rules', 'cellspacing', 'cellpadding',
1420                                                                                 'align', 'bgcolor',
1421                                                                 ) ),
1422
1423                         # 11.2.2
1424                         'caption'    => array_merge( $common, array( 'align' ) ),
1425
1426                         # 11.2.3
1427                         'thead'      => array_merge( $common, $tablealign ),
1428                         'tfoot'      => array_merge( $common, $tablealign ),
1429                         'tbody'      => array_merge( $common, $tablealign ),
1430
1431                         # 11.2.4
1432                         'colgroup'   => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1433                         'col'        => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1434
1435                         # 11.2.5
1436                         'tr'         => array_merge( $common, array( 'bgcolor' ), $tablealign ),
1437
1438                         # 11.2.6
1439                         'td'         => array_merge( $common, $tablecell, $tablealign ),
1440                         'th'         => array_merge( $common, $tablecell, $tablealign ),
1441
1442                         # 12.2 # NOTE: <a> is not allowed directly, but the attrib whitelist is used from the Parser object
1443                         'a'          => array_merge( $common, array( 'href', 'rel', 'rev' ) ), # rel/rev esp. for RDFa
1444
1445                         # 13.2
1446                         # Not usually allowed, but may be used for extension-style hooks
1447                         # such as <math> when it is rasterized, or if $wgAllowImageTag is
1448                         # true
1449                         'img'        => array_merge( $common, array( 'alt', 'src', 'width', 'height' ) ),
1450
1451                         # 15.2.1
1452                         'tt'         => $common,
1453                         'b'          => $common,
1454                         'i'          => $common,
1455                         'big'        => $common,
1456                         'small'      => $common,
1457                         'strike'     => $common,
1458                         's'          => $common,
1459                         'u'          => $common,
1460
1461                         # 15.2.2
1462                         'font'       => array_merge( $common, array( 'size', 'color', 'face' ) ),
1463                         # basefont
1464
1465                         # 15.3
1466                         'hr'         => array_merge( $common, array( 'noshade', 'size', 'width' ) ),
1467
1468                         # XHTML Ruby annotation text module, simple ruby only.
1469                         # http://www.w3c.org/TR/ruby/
1470                         'ruby'       => $common,
1471                         # rbc
1472                         # rtc
1473                         'rb'         => $common,
1474                         'rt'         => $common, #array_merge( $common, array( 'rbspan' ) ),
1475                         'rp'         => $common,
1476
1477                         # MathML root element, where used for extensions
1478                         # 'title' may not be 100% valid here; it's XHTML
1479                         # http://www.w3.org/TR/REC-MathML/
1480                         'math'       => array( 'class', 'style', 'id', 'title' ),
1481                         );
1482                 return $whitelist;
1483         }
1484
1485         /**
1486          * Take a fragment of (potentially invalid) HTML and return
1487          * a version with any tags removed, encoded as plain text.
1488          *
1489          * Warning: this return value must be further escaped for literal
1490          * inclusion in HTML output as of 1.10!
1491          *
1492          * @param $text String: HTML fragment
1493          * @return String
1494          */
1495         static function stripAllTags( $text ) {
1496                 # Actual <tags>
1497                 $text = StringUtils::delimiterReplace( '<', '>', '', $text );
1498
1499                 # Normalize &entities and whitespace
1500                 $text = self::decodeCharReferences( $text );
1501                 $text = self::normalizeWhitespace( $text );
1502
1503                 return $text;
1504         }
1505
1506         /**
1507          * Hack up a private DOCTYPE with HTML's standard entity declarations.
1508          * PHP 4 seemed to know these if you gave it an HTML doctype, but
1509          * PHP 5.1 doesn't.
1510          *
1511          * Use for passing XHTML fragments to PHP's XML parsing functions
1512          *
1513          * @return String
1514          */
1515         static function hackDocType() {
1516                 $out = "<!DOCTYPE html [\n";
1517                 foreach( self::$htmlEntities as $entity => $codepoint ) {
1518                         $out .= "<!ENTITY $entity \"&#$codepoint;\">";
1519                 }
1520                 $out .= "]>\n";
1521                 return $out;
1522         }
1523
1524         static function cleanUrl( $url ) {
1525                 # Normalize any HTML entities in input. They will be
1526                 # re-escaped by makeExternalLink().
1527                 $url = Sanitizer::decodeCharReferences( $url );
1528
1529                 # Escape any control characters introduced by the above step
1530                 $url = preg_replace_callback( '/[\][<>"\\x00-\\x20\\x7F\|]/',
1531                         array( __CLASS__, 'cleanUrlCallback' ), $url );
1532
1533                 # Validate hostname portion
1534                 $matches = array();
1535                 if( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
1536                         list( /* $whole */, $protocol, $host, $rest ) = $matches;
1537
1538                         // Characters that will be ignored in IDNs.
1539                         // http://tools.ietf.org/html/3454#section-3.1
1540                         // Strip them before further processing so blacklists and such work.
1541                         $strip = "/
1542                                 \\s|          # general whitespace
1543                                 \xc2\xad|     # 00ad SOFT HYPHEN
1544                                 \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
1545                                 \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
1546                                 \xe2\x81\xa0| # 2060 WORD JOINER
1547                                 \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
1548                                 \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
1549                                 \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
1550                                 \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
1551                                 \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
1552                                 \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
1553                                 \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
1554                                 [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe00f VARIATION SELECTOR-1-16
1555                                 /xuD";
1556
1557                         $host = preg_replace( $strip, '', $host );
1558
1559                         // @todo Fixme: validate hostnames here
1560
1561                         return $protocol . $host . $rest;
1562                 } else {
1563                         return $url;
1564                 }
1565         }
1566
1567         static function cleanUrlCallback( $matches ) {
1568                 return urlencode( $matches[0] );
1569         }
1570 }