includes/Sanitizer.php

   1 <?php
   2 /**
   3  * XHTML sanitizer for %MediaWiki.
   4  *
   5  * Copyright © 2002-2005 Brion Vibber <brion@pobox.com> et al
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Parser
  25  */
  26
  27 /**
  28  * XHTML sanitizer for MediaWiki
  29  * @ingroup Parser
  30  */
  31 class Sanitizer {
  32         /**
  33          * Regular expression to match various types of character references in
  34          * Sanitizer::normalizeCharReferences and Sanitizer::decodeCharReferences
  35          */
  36         const CHAR_REFS_REGEX =
  37                 '/&([A-Za-z0-9\x80-\xff]+);
  38                  |&\#([0-9]+);
  39                  |&\#[xX]([0-9A-Fa-f]+);
  40                  |(&)/x';
  41
  42         /**
  43          * Blacklist for evil uris like javascript:
  44          * WARNING: DO NOT use this in any place that actually requires blacklisting
  45          * for security reasons. There are NUMEROUS[1] ways to bypass blacklisting, the
  46          * only way to be secure from javascript: uri based xss vectors is to whitelist
  47          * things that you know are safe and deny everything else.
  48          * [1]: http://ha.ckers.org/xss.html
  49          */
  50         const EVIL_URI_PATTERN = '!(^|\s|\*/\s*)(javascript|vbscript)([^\w]|$)!i';
  51         const XMLNS_ATTRIBUTE_PATTERN = "/^xmlns:[:A-Z_a-z-.0-9]+$/";
  52
  53         /**
  54          * List of all named character entities defined in HTML 4.01
  55          * http://www.w3.org/TR/html4/sgml/entities.html
  56          * As well as &apos; which is only defined starting in XHTML1.
  57          * @private
  58          */
  59         static $htmlEntities = array(
  60                 'Aacute'   => 193,
  61                 'aacute'   => 225,
  62                 'Acirc'    => 194,
  63                 'acirc'    => 226,
  64                 'acute'    => 180,
  65                 'AElig'    => 198,
  66                 'aelig'    => 230,
  67                 'Agrave'   => 192,
  68                 'agrave'   => 224,
  69                 'alefsym'  => 8501,
  70                 'Alpha'    => 913,
  71                 'alpha'    => 945,
  72                 'amp'      => 38,
  73                 'and'      => 8743,
  74                 'ang'      => 8736,
  75                 'apos'     => 39, // New in XHTML & HTML 5; avoid in output for compatibility with IE.
  76                 'Aring'    => 197,
  77                 'aring'    => 229,
  78                 'asymp'    => 8776,
  79                 'Atilde'   => 195,
  80                 'atilde'   => 227,
  81                 'Auml'     => 196,
  82                 'auml'     => 228,
  83                 'bdquo'    => 8222,
  84                 'Beta'     => 914,
  85                 'beta'     => 946,
  86                 'brvbar'   => 166,
  87                 'bull'     => 8226,
  88                 'cap'      => 8745,
  89                 'Ccedil'   => 199,
  90                 'ccedil'   => 231,
  91                 'cedil'    => 184,
  92                 'cent'     => 162,
  93                 'Chi'      => 935,
  94                 'chi'      => 967,
  95                 'circ'     => 710,
  96                 'clubs'    => 9827,
  97                 'cong'     => 8773,
  98                 'copy'     => 169,
  99                 'crarr'    => 8629,
 100                 'cup'      => 8746,
 101                 'curren'   => 164,
 102                 'dagger'   => 8224,
 103                 'Dagger'   => 8225,
 104                 'darr'     => 8595,
 105                 'dArr'     => 8659,
 106                 'deg'      => 176,
 107                 'Delta'    => 916,
 108                 'delta'    => 948,
 109                 'diams'    => 9830,
 110                 'divide'   => 247,
 111                 'Eacute'   => 201,
 112                 'eacute'   => 233,
 113                 'Ecirc'    => 202,
 114                 'ecirc'    => 234,
 115                 'Egrave'   => 200,
 116                 'egrave'   => 232,
 117                 'empty'    => 8709,
 118                 'emsp'     => 8195,
 119                 'ensp'     => 8194,
 120                 'Epsilon'  => 917,
 121                 'epsilon'  => 949,
 122                 'equiv'    => 8801,
 123                 'Eta'      => 919,
 124                 'eta'      => 951,
 125                 'ETH'      => 208,
 126                 'eth'      => 240,
 127                 'Euml'     => 203,
 128                 'euml'     => 235,
 129                 'euro'     => 8364,
 130                 'exist'    => 8707,
 131                 'fnof'     => 402,
 132                 'forall'   => 8704,
 133                 'frac12'   => 189,
 134                 'frac14'   => 188,
 135                 'frac34'   => 190,
 136                 'frasl'    => 8260,
 137                 'Gamma'    => 915,
 138                 'gamma'    => 947,
 139                 'ge'       => 8805,
 140                 'gt'       => 62,
 141                 'harr'     => 8596,
 142                 'hArr'     => 8660,
 143                 'hearts'   => 9829,
 144                 'hellip'   => 8230,
 145                 'Iacute'   => 205,
 146                 'iacute'   => 237,
 147                 'Icirc'    => 206,
 148                 'icirc'    => 238,
 149                 'iexcl'    => 161,
 150                 'Igrave'   => 204,
 151                 'igrave'   => 236,
 152                 'image'    => 8465,
 153                 'infin'    => 8734,
 154                 'int'      => 8747,
 155                 'Iota'     => 921,
 156                 'iota'     => 953,
 157                 'iquest'   => 191,
 158                 'isin'     => 8712,
 159                 'Iuml'     => 207,
 160                 'iuml'     => 239,
 161                 'Kappa'    => 922,
 162                 'kappa'    => 954,
 163                 'Lambda'   => 923,
 164                 'lambda'   => 955,
 165                 'lang'     => 9001,
 166                 'laquo'    => 171,
 167                 'larr'     => 8592,
 168                 'lArr'     => 8656,
 169                 'lceil'    => 8968,
 170                 'ldquo'    => 8220,
 171                 'le'       => 8804,
 172                 'lfloor'   => 8970,
 173                 'lowast'   => 8727,
 174                 'loz'      => 9674,
 175                 'lrm'      => 8206,
 176                 'lsaquo'   => 8249,
 177                 'lsquo'    => 8216,
 178                 'lt'       => 60,
 179                 'macr'     => 175,
 180                 'mdash'    => 8212,
 181                 'micro'    => 181,
 182                 'middot'   => 183,
 183                 'minus'    => 8722,
 184                 'Mu'       => 924,
 185                 'mu'       => 956,
 186                 'nabla'    => 8711,
 187                 'nbsp'     => 160,
 188                 'ndash'    => 8211,
 189                 'ne'       => 8800,
 190                 'ni'       => 8715,
 191                 'not'      => 172,
 192                 'notin'    => 8713,
 193                 'nsub'     => 8836,
 194                 'Ntilde'   => 209,
 195                 'ntilde'   => 241,
 196                 'Nu'       => 925,
 197                 'nu'       => 957,
 198                 'Oacute'   => 211,
 199                 'oacute'   => 243,
 200                 'Ocirc'    => 212,
 201                 'ocirc'    => 244,
 202                 'OElig'    => 338,
 203                 'oelig'    => 339,
 204                 'Ograve'   => 210,
 205                 'ograve'   => 242,
 206                 'oline'    => 8254,
 207                 'Omega'    => 937,
 208                 'omega'    => 969,
 209                 'Omicron'  => 927,
 210                 'omicron'  => 959,
 211                 'oplus'    => 8853,
 212                 'or'       => 8744,
 213                 'ordf'     => 170,
 214                 'ordm'     => 186,
 215                 'Oslash'   => 216,
 216                 'oslash'   => 248,
 217                 'Otilde'   => 213,
 218                 'otilde'   => 245,
 219                 'otimes'   => 8855,
 220                 'Ouml'     => 214,
 221                 'ouml'     => 246,
 222                 'para'     => 182,
 223                 'part'     => 8706,
 224                 'permil'   => 8240,
 225                 'perp'     => 8869,
 226                 'Phi'      => 934,
 227                 'phi'      => 966,
 228                 'Pi'       => 928,
 229                 'pi'       => 960,
 230                 'piv'      => 982,
 231                 'plusmn'   => 177,
 232                 'pound'    => 163,
 233                 'prime'    => 8242,
 234                 'Prime'    => 8243,
 235                 'prod'     => 8719,
 236                 'prop'     => 8733,
 237                 'Psi'      => 936,
 238                 'psi'      => 968,
 239                 'quot'     => 34,
 240                 'radic'    => 8730,
 241                 'rang'     => 9002,
 242                 'raquo'    => 187,
 243                 'rarr'     => 8594,
 244                 'rArr'     => 8658,
 245                 'rceil'    => 8969,
 246                 'rdquo'    => 8221,
 247                 'real'     => 8476,
 248                 'reg'      => 174,
 249                 'rfloor'   => 8971,
 250                 'Rho'      => 929,
 251                 'rho'      => 961,
 252                 'rlm'      => 8207,
 253                 'rsaquo'   => 8250,
 254                 'rsquo'    => 8217,
 255                 'sbquo'    => 8218,
 256                 'Scaron'   => 352,
 257                 'scaron'   => 353,
 258                 'sdot'     => 8901,
 259                 'sect'     => 167,
 260                 'shy'      => 173,
 261                 'Sigma'    => 931,
 262                 'sigma'    => 963,
 263                 'sigmaf'   => 962,
 264                 'sim'      => 8764,
 265                 'spades'   => 9824,
 266                 'sub'      => 8834,
 267                 'sube'     => 8838,
 268                 'sum'      => 8721,
 269                 'sup'      => 8835,
 270                 'sup1'     => 185,
 271                 'sup2'     => 178,
 272                 'sup3'     => 179,
 273                 'supe'     => 8839,
 274                 'szlig'    => 223,
 275                 'Tau'      => 932,
 276                 'tau'      => 964,
 277                 'there4'   => 8756,
 278                 'Theta'    => 920,
 279                 'theta'    => 952,
 280                 'thetasym' => 977,
 281                 'thinsp'   => 8201,
 282                 'THORN'    => 222,
 283                 'thorn'    => 254,
 284                 'tilde'    => 732,
 285                 'times'    => 215,
 286                 'trade'    => 8482,
 287                 'Uacute'   => 218,
 288                 'uacute'   => 250,
 289                 'uarr'     => 8593,
 290                 'uArr'     => 8657,
 291                 'Ucirc'    => 219,
 292                 'ucirc'    => 251,
 293                 'Ugrave'   => 217,
 294                 'ugrave'   => 249,
 295                 'uml'      => 168,
 296                 'upsih'    => 978,
 297                 'Upsilon'  => 933,
 298                 'upsilon'  => 965,
 299                 'Uuml'     => 220,
 300                 'uuml'     => 252,
 301                 'weierp'   => 8472,
 302                 'Xi'       => 926,
 303                 'xi'       => 958,
 304                 'Yacute'   => 221,
 305                 'yacute'   => 253,
 306                 'yen'      => 165,
 307                 'Yuml'     => 376,
 308                 'yuml'     => 255,
 309                 'Zeta'     => 918,
 310                 'zeta'     => 950,
 311                 'zwj'      => 8205,
 312                 'zwnj'     => 8204
 313         );
 314
 315         /**
 316          * Character entity aliases accepted by MediaWiki
 317          */
 318         static $htmlEntityAliases = array(
 319                 'רלמ' => 'rlm',
 320                 'رلم' => 'rlm',
 321         );
 322
 323         /**
 324          * Lazy-initialised attributes regex, see getAttribsRegex()
 325          */
 326         static $attribsRegex;
 327
 328         /**
 329          * Regular expression to match HTML/XML attribute pairs within a tag.
 330          * Allows some... latitude.
 331          * Used in Sanitizer::fixTagAttributes and Sanitizer::decodeTagAttributes
 332          */
 333         static function getAttribsRegex() {
 334                 if ( self::$attribsRegex === null ) {
 335                         $attribFirst = '[:A-Z_a-z0-9]';
 336                         $attrib = '[:A-Z_a-z-.0-9]';
 337                         $space = '[\x09\x0a\x0d\x20]';
 338                         self::$attribsRegex =
 339                                 "/(?:^|$space)({$attribFirst}{$attrib}*)
 340                                   ($space*=$space*
 341                                         (?:
 342                                          # The attribute value: quoted or alone
 343                                           \"([^<\"]*)\"
 344                                          | '([^<']*)'
 345                                          |  ([a-zA-Z0-9!#$%&()*,\\-.\\/:;<>?@[\\]^_`{|}~]+)
 346                                          |  (\#[0-9a-fA-F]+) # Technically wrong, but lots of
 347                                                                                  # colors are specified like this.
 348                                                                                  # We'll be normalizing it.
 349                                         )
 350                                 )?(?=$space|\$)/sx";
 351                 }
 352                 return self::$attribsRegex;
 353         }
 354
 355         /**
 356          * Cleans up HTML, removes dangerous tags and attributes, and
 357          * removes HTML comments
 358          * @private
 359          * @param $text String
 360          * @param $processCallback Callback to do any variable or parameter replacements in HTML attribute values
 361          * @param $args Array for the processing callback
 362          * @param $extratags Array for any extra tags to include
 363          * @param $removetags Array for any tags (default or extra) to exclude
 364          * @return string
 365          */
 366         static function removeHTMLtags( $text, $processCallback = null, $args = array(), $extratags = array(), $removetags = array() ) {
 367                 global $wgUseTidy, $wgHtml5, $wgAllowMicrodataAttributes, $wgAllowImageTag;
 368
 369                 static $htmlpairsStatic, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
 370                         $htmllist, $listtags, $htmlsingleallowed, $htmlelementsStatic, $staticInitialised;
 371
 372                 wfProfileIn( __METHOD__ );
 373
 374                 // Base our staticInitialised variable off of the global config state so that if the globals
 375                 // are changed (like in the secrewed up test system) we will re-initialise the settings.
 376                 $globalContext = implode( '-', compact( 'wgHtml5', 'wgAllowMicrodataAttributes', 'wgAllowImageTag' ) );
 377                 if ( !$staticInitialised || $staticInitialised != $globalContext ) {
 378
 379                         $htmlpairsStatic = array( # Tags that must be closed
 380                                 'b', 'bdi', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
 381                                 'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
 382                                 'strike', 'strong', 'tt', 'var', 'div', 'center',
 383                                 'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
 384                                 'ruby', 'rt', 'rb', 'rp', 'p', 'span', 'abbr', 'dfn',
 385                                 'kbd', 'samp'
 386                         );
 387                         if ( $wgHtml5 ) {
 388                                 $htmlpairsStatic = array_merge( $htmlpairsStatic, array( 'data', 'time', 'mark' ) );
 389                         }
 390                         $htmlsingle = array(
 391                                 'br', 'hr', 'li', 'dt', 'dd'
 392                         );
 393                         $htmlsingleonly = array( # Elements that cannot have close tags
 394                                 'br', 'hr'
 395                         );
 396                         if ( $wgHtml5 && $wgAllowMicrodataAttributes ) {
 397                                 $htmlsingle[] = $htmlsingleonly[] = 'meta';
 398                                 $htmlsingle[] = $htmlsingleonly[] = 'link';
 399                         }
 400                         $htmlnest = array( # Tags that can be nested--??
 401                                 'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
 402                                 'li', 'dl', 'dt', 'dd', 'font', 'big', 'small', 'sub', 'sup', 'span'
 403                         );
 404                         $tabletags = array( # Can only appear inside table, we will close them
 405                                 'td', 'th', 'tr',
 406                         );
 407                         $htmllist = array( # Tags used by list
 408                                 'ul','ol',
 409                         );
 410                         $listtags = array( # Tags that can appear in a list
 411                                 'li',
 412                         );
 413
 414                         if ( $wgAllowImageTag ) {
 415                                 $htmlsingle[] = 'img';
 416                                 $htmlsingleonly[] = 'img';
 417                         }
 418
 419                         $htmlsingleallowed = array_unique( array_merge( $htmlsingle, $tabletags ) );
 420                         $htmlelementsStatic = array_unique( array_merge( $htmlsingle, $htmlpairsStatic, $htmlnest ) );
 421
 422                         # Convert them all to hashtables for faster lookup
 423                         $vars = array( 'htmlpairsStatic', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
 424                                 'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelementsStatic' );
 425                         foreach ( $vars as $var ) {
 426                                 $$var = array_flip( $$var );
 427                         }
 428                         $staticInitialised = $globalContext;
 429                 }
 430                 # Populate $htmlpairs and $htmlelements with the $extratags and $removetags arrays
 431                 $extratags = array_flip( $extratags );
 432                 $removetags = array_flip( $removetags );
 433                 $htmlpairs = array_merge( $extratags, $htmlpairsStatic );
 434                 $htmlelements = array_diff_key( array_merge( $extratags, $htmlelementsStatic ), $removetags );
 435
 436                 # Remove HTML comments
 437                 $text = Sanitizer::removeHTMLcomments( $text );
 438                 $bits = explode( '<', $text );
 439                 $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 440                 if ( !$wgUseTidy ) {
 441                         $tagstack = $tablestack = array();
 442                         foreach ( $bits as $x ) {
 443                                 $regs = array();
 444                                 # $slash: Does the current element start with a '/'?
 445                                 # $t: Current element name
 446                                 # $params: String between element name and >
 447                                 # $brace: Ending '>' or '/>'
 448                                 # $rest: Everything until the next element of $bits
 449                                 if( preg_match( '!^(/?)(\\w+)([^>]*?)(/{0,1}>)([^<]*)$!', $x, $regs ) ) {
 450                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 451                                 } else {
 452                                         $slash = $t = $params = $brace = $rest = null;
 453                                 }
 454
 455                                 $badtag = false;
 456                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 457                                         # Check our stack
 458                                         if ( $slash && isset( $htmlsingleonly[$t] ) ) {
 459                                                 $badtag = true;
 460                                         } elseif ( $slash ) {
 461                                                 # Closing a tag... is it the one we just opened?
 462                                                 $ot = @array_pop( $tagstack );
 463                                                 if ( $ot != $t ) {
 464                                                         if ( isset( $htmlsingleallowed[$ot] ) ) {
 465                                                                 # Pop all elements with an optional close tag
 466                                                                 # and see if we find a match below them
 467                                                                 $optstack = array();
 468                                                                 array_push( $optstack, $ot );
 469                                                                 wfSuppressWarnings();
 470                                                                 $ot = array_pop( $tagstack );
 471                                                                 wfRestoreWarnings();
 472                                                                 while ( $ot != $t && isset( $htmlsingleallowed[$ot] ) ) {
 473                                                                         array_push( $optstack, $ot );
 474                                                                         wfSuppressWarnings();
 475                                                                         $ot = array_pop( $tagstack );
 476                                                                         wfRestoreWarnings();
 477                                                                 }
 478                                                                 if ( $t != $ot ) {
 479                                                                         # No match. Push the optional elements back again
 480                                                                         $badtag = true;
 481                                                                         wfSuppressWarnings();
 482                                                                         $ot = array_pop( $optstack );
 483                                                                         wfRestoreWarnings();
 484                                                                         while ( $ot ) {
 485                                                                                 array_push( $tagstack, $ot );
 486                                                                                 wfSuppressWarnings();
 487                                                                                 $ot = array_pop( $optstack );
 488                                                                                 wfRestoreWarnings();
 489                                                                         }
 490                                                                 }
 491                                                         } else {
 492                                                                 @array_push( $tagstack, $ot );
 493                                                                 # <li> can be nested in <ul> or <ol>, skip those cases:
 494                                                                 if ( !isset( $htmllist[$ot] ) || !isset( $listtags[$t] ) ) {
 495                                                                         $badtag = true;
 496                                                                 }
 497                                                         }
 498                                                 } else {
 499                                                         if ( $t == 'table' ) {
 500                                                                 $tagstack = array_pop( $tablestack );
 501                                                         }
 502                                                 }
 503                                                 $newparams = '';
 504                                         } else {
 505                                                 # Keep track for later
 506                                                 if ( isset( $tabletags[$t] ) &&
 507                                                 !in_array( 'table', $tagstack ) ) {
 508                                                         $badtag = true;
 509                                                 } elseif ( in_array( $t, $tagstack ) &&
 510                                                 !isset( $htmlnest [$t ] ) ) {
 511                                                         $badtag = true;
 512                                                 # Is it a self closed htmlpair ? (bug 5487)
 513                                                 } elseif ( $brace == '/>' &&
 514                                                 isset( $htmlpairs[$t] ) ) {
 515                                                         $badtag = true;
 516                                                 } elseif ( isset( $htmlsingleonly[$t] ) ) {
 517                                                         # Hack to force empty tag for uncloseable elements
 518                                                         $brace = '/>';
 519                                                 } elseif ( isset( $htmlsingle[$t] ) ) {
 520                                                         # Hack to not close $htmlsingle tags
 521                                                         $brace = null;
 522                                                         # Still need to push this optionally-closed tag to
 523                                                         # the tag stack so that we can match end tags
 524                                                         # instead of marking them as bad.
 525                                                         array_push( $tagstack, $t );
 526                                                 } elseif ( isset( $tabletags[$t] )
 527                                                 && in_array( $t, $tagstack ) ) {
 528                                                         // New table tag but forgot to close the previous one
 529                                                         $text .= "</$t>";
 530                                                 } else {
 531                                                         if ( $t == 'table' ) {
 532                                                                 array_push( $tablestack, $tagstack );
 533                                                                 $tagstack = array();
 534                                                         }
 535                                                         array_push( $tagstack, $t );
 536                                                 }
 537
 538                                                 # Replace any variables or template parameters with
 539                                                 # plaintext results.
 540                                                 if( is_callable( $processCallback ) ) {
 541                                                         call_user_func_array( $processCallback, array( &$params, $args ) );
 542                                                 }
 543
 544                                                 if ( !Sanitizer::validateTag( $params, $t ) ) {
 545                                                         $badtag = true;
 546                                                 }
 547
 548                                                 # Strip non-approved attributes from the tag
 549                                                 $newparams = Sanitizer::fixTagAttributes( $params, $t );
 550                                         }
 551                                         if ( !$badtag ) {
 552                                                 $rest = str_replace( '>', '&gt;', $rest );
 553                                                 $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
 554                                                 $text .= "<$slash$t$newparams$close>$rest";
 555                                                 continue;
 556                                         }
 557                                 }
 558                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 559                         }
 560                         # Close off any remaining tags
 561                         while ( is_array( $tagstack ) && ($t = array_pop( $tagstack )) ) {
 562                                 $text .= "</$t>\n";
 563                                 if ( $t == 'table' ) { $tagstack = array_pop( $tablestack ); }
 564                         }
 565                 } else {
 566                         # this might be possible using tidy itself
 567                         foreach ( $bits as $x ) {
 568                                 preg_match( '/^(\\/?)(\\w+)([^>]*?)(\\/{0,1}>)([^<]*)$/',
 569                                 $x, $regs );
 570                                 @list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 571                                 $badtag = false;
 572                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 573                                         if( is_callable( $processCallback ) ) {
 574                                                 call_user_func_array( $processCallback, array( &$params, $args ) );
 575                                         }
 576
 577                                         if ( !Sanitizer::validateTag( $params, $t ) ) {
 578                                                 $badtag = true;
 579                                         }
 580
 581                                         $newparams = Sanitizer::fixTagAttributes( $params, $t );
 582                                         if ( !$badtag ) {
 583                                                 $rest = str_replace( '>', '&gt;', $rest );
 584                                                 $text .= "<$slash$t$newparams$brace$rest";
 585                                                 continue;
 586                                         }
 587                                 }
 588                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 589                         }
 590                 }
 591                 wfProfileOut( __METHOD__ );
 592                 return $text;
 593         }
 594
 595         /**
 596          * Remove '<!--', '-->', and everything between.
 597          * To avoid leaving blank lines, when a comment is both preceded
 598          * and followed by a newline (ignoring spaces), trim leading and
 599          * trailing spaces and one of the newlines.
 600          *
 601          * @private
 602          * @param $text String
 603          * @return string
 604          */
 605         static function removeHTMLcomments( $text ) {
 606                 wfProfileIn( __METHOD__ );
 607                 while ( ($start = strpos( $text, '<!--' ) ) !== false ) {
 608                         $end = strpos( $text, '-->', $start + 4 );
 609                         if ( $end === false ) {
 610                                 # Unterminated comment; bail out
 611                                 break;
 612                         }
 613
 614                         $end += 3;
 615
 616                         # Trim space and newline if the comment is both
 617                         # preceded and followed by a newline
 618                         $spaceStart = max( $start - 1, 0 );
 619                         $spaceLen = $end - $spaceStart;
 620                         while ( substr( $text, $spaceStart, 1 ) === ' ' && $spaceStart > 0 ) {
 621                                 $spaceStart--;
 622                                 $spaceLen++;
 623                         }
 624                         while ( substr( $text, $spaceStart + $spaceLen, 1 ) === ' ' )
 625                                 $spaceLen++;
 626                         if ( substr( $text, $spaceStart, 1 ) === "\n" and substr( $text, $spaceStart + $spaceLen, 1 ) === "\n" ) {
 627                                 # Remove the comment, leading and trailing
 628                                 # spaces, and leave only one newline.
 629                                 $text = substr_replace( $text, "\n", $spaceStart, $spaceLen + 1 );
 630                         }
 631                         else {
 632                                 # Remove just the comment.
 633                                 $text = substr_replace( $text, '', $start, $end - $start );
 634                         }
 635                 }
 636                 wfProfileOut( __METHOD__ );
 637                 return $text;
 638         }
 639
 640         /**
 641          * Takes attribute names and values for a tag and the tag name and
 642          * validates that the tag is allowed to be present.
 643          * This DOES NOT validate the attributes, nor does it validate the
 644          * tags themselves. This method only handles the special circumstances
 645          * where we may want to allow a tag within content but ONLY when it has
 646          * specific attributes set.
 647          *
 648          * @param $params
 649          * @param $element
 650          */
 651         static function validateTag( $params, $element ) {
 652                 $params = Sanitizer::decodeTagAttributes( $params );
 653
 654                 if ( $element == 'meta' || $element == 'link' ) {
 655                         if ( !isset( $params['itemprop'] ) ) {
 656                                 // <meta> and <link> must have an itemprop="" otherwise they are not valid or safe in content
 657                                 return false;
 658                         }
 659                         if ( $element == 'meta' && !isset( $params['content'] ) ) {
 660                                 // <meta> must have a content="" for the itemprop
 661                                 return false;
 662                         }
 663                         if ( $element == 'link' && !isset( $params['href'] ) ) {
 664                                 // <link> must have an associated href=""
 665                                 return false;
 666                         }
 667                 }
 668
 669                 return true;
 670         }
 671
 672         /**
 673          * Take an array of attribute names and values and normalize or discard
 674          * illegal values for the given element type.
 675          *
 676          * - Discards attributes not on a whitelist for the given element
 677          * - Unsafe style attributes are discarded
 678          * - Invalid id attributes are reencoded
 679          *
 680          * @param $attribs Array
 681          * @param $element String
 682          * @return Array
 683          *
 684          * @todo Check for legal values where the DTD limits things.
 685          * @todo Check for unique id attribute :P
 686          */
 687         static function validateTagAttributes( $attribs, $element ) {
 688                 return Sanitizer::validateAttributes( $attribs,
 689                         Sanitizer::attributeWhitelist( $element ) );
 690         }
 691
 692         /**
 693          * Take an array of attribute names and values and normalize or discard
 694          * illegal values for the given whitelist.
 695          *
 696          * - Discards attributes not the given whitelist
 697          * - Unsafe style attributes are discarded
 698          * - Invalid id attributes are reencoded
 699          *
 700          * @param $attribs Array
 701          * @param $whitelist Array: list of allowed attribute names
 702          * @return Array
 703          *
 704          * @todo Check for legal values where the DTD limits things.
 705          * @todo Check for unique id attribute :P
 706          */
 707         static function validateAttributes( $attribs, $whitelist ) {
 708                 global $wgAllowRdfaAttributes, $wgAllowMicrodataAttributes, $wgHtml5;
 709
 710                 $whitelist = array_flip( $whitelist );
 711                 $hrefExp = '/^(' . wfUrlProtocols() . ')[^\s]+$/';
 712
 713                 $out = array();
 714                 foreach( $attribs as $attribute => $value ) {
 715                         #allow XML namespace declaration if RDFa is enabled
 716                         if ( $wgAllowRdfaAttributes && preg_match( self::XMLNS_ATTRIBUTE_PATTERN, $attribute ) ) {
 717                                 if ( !preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 718                                         $out[$attribute] = $value;
 719                                 }
 720
 721                                 continue;
 722                         }
 723
 724                         # Allow any attribute beginning with "data-", if in HTML5 mode
 725                         if ( !($wgHtml5 && preg_match( '/^data-/i', $attribute )) && !isset( $whitelist[$attribute] ) ) {
 726                                 continue;
 727                         }
 728
 729                         # Strip javascript "expression" from stylesheets.
 730                         # http://msdn.microsoft.com/workshop/author/dhtml/overview/recalc.asp
 731                         if( $attribute == 'style' ) {
 732                                 $value = Sanitizer::checkCss( $value );
 733                         }
 734
 735                         if ( $attribute === 'id' ) {
 736                                 $value = Sanitizer::escapeId( $value, 'noninitial' );
 737                         }
 738
 739                         # WAI-ARIA
 740                         # http://www.w3.org/TR/wai-aria/
 741                         # http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#wai-aria
 742                         # For now we only support role="presentation" until we work out what roles should be
 743                         # usable by content and we ensure that our code explicitly rejects patterns that
 744                         # violate HTML5's ARIA restrictions.
 745                         if ( $attribute === 'role' && $value !== 'presentation' ) {
 746                                 continue;
 747                         }
 748
 749                         //RDFa and microdata properties allow URLs, URIs and/or CURIs. check them for sanity
 750                         if ( $attribute === 'rel' || $attribute === 'rev' ||
 751                                 $attribute === 'about' || $attribute === 'property' || $attribute === 'resource' || #RDFa
 752                                 $attribute === 'datatype' || $attribute === 'typeof' ||                             #RDFa
 753                                 $attribute === 'itemid' || $attribute === 'itemprop' || $attribute === 'itemref' || #HTML5 microdata
 754                                 $attribute === 'itemscope' || $attribute === 'itemtype' ) {                         #HTML5 microdata
 755
 756                                 //Paranoia. Allow "simple" values but suppress javascript
 757                                 if ( preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 758                                         continue;
 759                                 }
 760                         }
 761
 762                         # NOTE: even though elements using href/src are not allowed directly, supply
 763                         #       validation code that can be used by tag hook handlers, etc
 764                         if ( $attribute === 'href' || $attribute === 'src' ) {
 765                                 if ( !preg_match( $hrefExp, $value ) ) {
 766                                         continue; //drop any href or src attributes not using an allowed protocol.
 767                                                   //NOTE: this also drops all relative URLs
 768                                 }
 769                         }
 770
 771                         // If this attribute was previously set, override it.
 772                         // Output should only have one attribute of each name.
 773                         $out[$attribute] = $value;
 774                 }
 775
 776                 if ( $wgAllowMicrodataAttributes ) {
 777                         # itemtype, itemid, itemref don't make sense without itemscope
 778                         if ( !array_key_exists( 'itemscope', $out ) ) {
 779                                 unset( $out['itemtype'] );
 780                                 unset( $out['itemid'] );
 781                                 unset( $out['itemref'] );
 782                         }
 783                         # TODO: Strip itemprop if we aren't descendants of an itemscope or pointed to by an itemref.
 784                 }
 785                 return $out;
 786         }
 787
 788         /**
 789          * Merge two sets of HTML attributes.  Conflicting items in the second set
 790          * will override those in the first, except for 'class' attributes which
 791          * will be combined (if they're both strings).
 792          *
 793          * @todo implement merging for other attributes such as style
 794          * @param $a Array
 795          * @param $b Array
 796          * @return array
 797          */
 798         static function mergeAttributes( $a, $b ) {
 799                 $out = array_merge( $a, $b );
 800                 if( isset( $a['class'] ) && isset( $b['class'] )
 801                 && is_string( $a['class'] ) && is_string( $b['class'] )
 802                 && $a['class'] !== $b['class'] ) {
 803                         $classes = preg_split( '/\s+/', "{$a['class']} {$b['class']}",
 804                                 -1, PREG_SPLIT_NO_EMPTY );
 805                         $out['class'] = implode( ' ', array_unique( $classes ) );
 806                 }
 807                 return $out;
 808         }
 809
 810         /**
 811          * Pick apart some CSS and check it for forbidden or unsafe structures.
 812          * Returns a sanitized string. This sanitized string will have
 813          * character references and escape sequences decoded, and comments
 814          * stripped. If the input is just too evil, only a comment complaining
 815          * about evilness will be returned.
 816          *
 817          * Currently URL references, 'expression', 'tps' are forbidden.
 818          *
 819          * NOTE: Despite the fact that character references are decoded, the
 820          * returned string may contain character references given certain
 821          * clever input strings. These character references must
 822          * be escaped before the return value is embedded in HTML.
 823          *
 824          * @param $value String
 825          * @return String
 826          */
 827         static function checkCss( $value ) {
 828                 // Decode character references like &#123;
 829                 $value = Sanitizer::decodeCharReferences( $value );
 830
 831                 // Decode escape sequences and line continuation
 832                 // See the grammar in the CSS 2 spec, appendix D.
 833                 // This has to be done AFTER decoding character references.
 834                 // This means it isn't possible for this function to return
 835                 // unsanitized escape sequences. It is possible to manufacture
 836                 // input that contains character references that decode to
 837                 // escape sequences that decode to character references, but
 838                 // it's OK for the return value to contain character references
 839                 // because the caller is supposed to escape those anyway.
 840                 static $decodeRegex;
 841                 if ( !$decodeRegex ) {
 842                         $space = '[\\x20\\t\\r\\n\\f]';
 843                         $nl = '(?:\\n|\\r\\n|\\r|\\f)';
 844                         $backslash = '\\\\';
 845                         $decodeRegex = "/ $backslash
 846                                 (?:
 847                                         ($nl) |  # 1. Line continuation
 848                                         ([0-9A-Fa-f]{1,6})$space? |  # 2. character number
 849                                         (.) | # 3. backslash cancelling special meaning
 850                                         () | # 4. backslash at end of string
 851                                 )/xu";
 852                 }
 853                 $value = preg_replace_callback( $decodeRegex,
 854                         array( __CLASS__, 'cssDecodeCallback' ), $value );
 855
 856                 // Remove any comments; IE gets token splitting wrong
 857                 // This must be done AFTER decoding character references and
 858                 // escape sequences, because those steps can introduce comments
 859                 // This step cannot introduce character references or escape
 860                 // sequences, because it replaces comments with spaces rather
 861                 // than removing them completely.
 862                 $value = StringUtils::delimiterReplace( '/*', '*/', ' ', $value );
 863
 864                 // Remove anything after a comment-start token, to guard against
 865                 // incorrect client implementations.
 866                 $commentPos = strpos( $value, '/*' );
 867                 if ( $commentPos !== false ) {
 868                         $value = substr( $value, 0, $commentPos );
 869                 }
 870
 871                 // Reject problematic keywords and control characters
 872                 if ( preg_match( '/[\000-\010\016-\037\177]/', $value ) ) {
 873                         return '/* invalid control char */';
 874                 } elseif ( preg_match( '! expression | filter\s*: | accelerator\s*: | url\s*\( | image\s*\( | image-set\s*\( !ix', $value ) ) {
 875                         return '/* insecure input */';
 876                 }
 877                 return $value;
 878         }
 879
 880         /**
 881          * @param $matches array
 882          * @return String
 883          */
 884         static function cssDecodeCallback( $matches ) {
 885                 if ( $matches[1] !== '' ) {
 886                         // Line continuation
 887                         return '';
 888                 } elseif ( $matches[2] !== '' ) {
 889                         $char = codepointToUtf8( hexdec( $matches[2] ) );
 890                 } elseif ( $matches[3] !== '' ) {
 891                         $char = $matches[3];
 892                 } else {
 893                         $char = '\\';
 894                 }
 895                 if ( $char == "\n" || $char == '"' || $char == "'" || $char == '\\' ) {
 896                         // These characters need to be escaped in strings
 897                         // Clean up the escape sequence to avoid parsing errors by clients
 898                         return '\\' . dechex( ord( $char ) ) . ' ';
 899                 } else {
 900                         // Decode unnecessary escape
 901                         return $char;
 902                 }
 903         }
 904
 905         /**
 906          * Take a tag soup fragment listing an HTML element's attributes
 907          * and normalize it to well-formed XML, discarding unwanted attributes.
 908          * Output is safe for further wikitext processing, with escaping of
 909          * values that could trigger problems.
 910          *
 911          * - Normalizes attribute names to lowercase
 912          * - Discards attributes not on a whitelist for the given element
 913          * - Turns broken or invalid entities into plaintext
 914          * - Double-quotes all attribute values
 915          * - Attributes without values are given the name as attribute
 916          * - Double attributes are discarded
 917          * - Unsafe style attributes are discarded
 918          * - Prepends space if there are attributes.
 919          *
 920          * @param $text String
 921          * @param $element String
 922          * @return String
 923          */
 924         static function fixTagAttributes( $text, $element ) {
 925                 if( trim( $text ) == '' ) {
 926                         return '';
 927                 }
 928
 929                 $decoded = Sanitizer::decodeTagAttributes( $text );
 930                 $stripped = Sanitizer::validateTagAttributes( $decoded, $element );
 931
 932                 $attribs = array();
 933                 foreach( $stripped as $attribute => $value ) {
 934                         $encAttribute = htmlspecialchars( $attribute );
 935                         $encValue = Sanitizer::safeEncodeAttribute( $value );
 936
 937                         $attribs[] = "$encAttribute=\"$encValue\"";
 938                 }
 939                 return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
 940         }
 941
 942         /**
 943          * Encode an attribute value for HTML output.
 944          * @param $text String
 945          * @return HTML-encoded text fragment
 946          */
 947         static function encodeAttribute( $text ) {
 948                 $encValue = htmlspecialchars( $text, ENT_QUOTES );
 949
 950                 // Whitespace is normalized during attribute decoding,
 951                 // so if we've been passed non-spaces we must encode them
 952                 // ahead of time or they won't be preserved.
 953                 $encValue = strtr( $encValue, array(
 954                         "\n" => '&#10;',
 955                         "\r" => '&#13;',
 956                         "\t" => '&#9;',
 957                 ) );
 958
 959                 return $encValue;
 960         }
 961
 962         /**
 963          * Encode an attribute value for HTML tags, with extra armoring
 964          * against further wiki processing.
 965          * @param $text String
 966          * @return HTML-encoded text fragment
 967          */
 968         static function safeEncodeAttribute( $text ) {
 969                 $encValue = Sanitizer::encodeAttribute( $text );
 970
 971                 # Templates and links may be expanded in later parsing,
 972                 # creating invalid or dangerous output. Suppress this.
 973                 $encValue = strtr( $encValue, array(
 974                         '<'    => '&lt;',   // This should never happen,
 975                         '>'    => '&gt;',   // we've received invalid input
 976                         '"'    => '&quot;', // which should have been escaped.
 977                         '{'    => '&#123;',
 978                         '['    => '&#91;',
 979                         "''"   => '&#39;&#39;',
 980                         'ISBN' => '&#73;SBN',
 981                         'RFC'  => '&#82;FC',
 982                         'PMID' => '&#80;MID',
 983                         '|'    => '&#124;',
 984                         '__'   => '&#95;_',
 985                 ) );
 986
 987                 # Stupid hack
 988                 $encValue = preg_replace_callback(
 989                         '/((?i)' . wfUrlProtocols() . ')/',
 990                         array( 'Sanitizer', 'armorLinksCallback' ),
 991                         $encValue );
 992                 return $encValue;
 993         }
 994
 995         /**
 996          * Given a value, escape it so that it can be used in an id attribute and
 997          * return it.  This will use HTML5 validation if $wgExperimentalHtmlIds is
 998          * true, allowing anything but ASCII whitespace.  Otherwise it will use
 999          * HTML 4 rules, which means a narrow subset of ASCII, with bad characters
1000          * escaped with lots of dots.
1001          *
1002          * To ensure we don't have to bother escaping anything, we also strip ', ",
1003          * & even if $wgExperimentalIds is true.  TODO: Is this the best tactic?
1004          * We also strip # because it upsets IE, and % because it could be
1005          * ambiguous if it's part of something that looks like a percent escape
1006          * (which don't work reliably in fragments cross-browser).
1007          *
1008          * @see http://www.w3.org/TR/html401/types.html#type-name Valid characters
1009          *                                                          in the id and
1010          *                                                          name attributes
1011          * @see http://www.w3.org/TR/html401/struct/links.html#h-12.2.3 Anchors with the id attribute
1012          * @see http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#the-id-attribute
1013          *   HTML5 definition of id attribute
1014          *
1015          * @param $id String: id to escape
1016          * @param $options Mixed: string or array of strings (default is array()):
1017          *   'noninitial': This is a non-initial fragment of an id, not a full id,
1018          *       so don't pay attention if the first character isn't valid at the
1019          *       beginning of an id.  Only matters if $wgExperimentalHtmlIds is
1020          *       false.
1021          *   'legacy': Behave the way the old HTML 4-based ID escaping worked even
1022          *       if $wgExperimentalHtmlIds is used, so we can generate extra
1023          *       anchors and links won't break.
1024          * @return String
1025          */
1026         static function escapeId( $id, $options = array() ) {
1027                 global $wgHtml5, $wgExperimentalHtmlIds;
1028                 $options = (array)$options;
1029
1030                 if ( $wgHtml5 && $wgExperimentalHtmlIds && !in_array( 'legacy', $options ) ) {
1031                         $id = Sanitizer::decodeCharReferences( $id );
1032                         $id = preg_replace( '/[ \t\n\r\f_\'"&#%]+/', '_', $id );
1033                         $id = trim( $id, '_' );
1034                         if ( $id === '' ) {
1035                                 # Must have been all whitespace to start with.
1036                                 return '_';
1037                         } else {
1038                                 return $id;
1039                         }
1040                 }
1041
1042                 # HTML4-style escaping
1043                 static $replace = array(
1044                         '%3A' => ':',
1045                         '%' => '.'
1046                 );
1047
1048                 $id = urlencode( Sanitizer::decodeCharReferences( strtr( $id, ' ', '_' ) ) );
1049                 $id = str_replace( array_keys( $replace ), array_values( $replace ), $id );
1050
1051                 if ( !preg_match( '/^[a-zA-Z]/', $id )
1052                 && !in_array( 'noninitial', $options ) ) {
1053                         // Initial character must be a letter!
1054                         $id = "x$id";
1055                 }
1056                 return $id;
1057         }
1058
1059         /**
1060          * Given a value, escape it so that it can be used as a CSS class and
1061          * return it.
1062          *
1063          * @todo For extra validity, input should be validated UTF-8.
1064          *
1065          * @see http://www.w3.org/TR/CSS21/syndata.html Valid characters/format
1066          *
1067          * @param $class String
1068          * @return String
1069          */
1070         static function escapeClass( $class ) {
1071                 // Convert ugly stuff to underscores and kill underscores in ugly places
1072                 return rtrim( preg_replace(
1073                         array( '/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/', '/_+/' ),
1074                         '_',
1075                         $class ), '_' );
1076         }
1077
1078         /**
1079          * Given HTML input, escape with htmlspecialchars but un-escape entites.
1080          * This allows (generally harmless) entities like &#160; to survive.
1081          *
1082          * @param $html String to escape
1083          * @return String: escaped input
1084          */
1085         static function escapeHtmlAllowEntities( $html ) {
1086                 $html = Sanitizer::decodeCharReferences( $html );
1087                 # It seems wise to escape ' as well as ", as a matter of course.  Can't
1088                 # hurt.
1089                 $html = htmlspecialchars( $html, ENT_QUOTES );
1090                 return $html;
1091         }
1092
1093         /**
1094          * Regex replace callback for armoring links against further processing.
1095          * @param $matches Array
1096          * @return string
1097          */
1098         private static function armorLinksCallback( $matches ) {
1099                 return str_replace( ':', '&#58;', $matches[1] );
1100         }
1101
1102         /**
1103          * Return an associative array of attribute names and values from
1104          * a partial tag string. Attribute names are forces to lowercase,
1105          * character references are decoded to UTF-8 text.
1106          *
1107          * @param $text String
1108          * @return Array
1109          */
1110         public static function decodeTagAttributes( $text ) {
1111                 if( trim( $text ) == '' ) {
1112                         return array();
1113                 }
1114
1115                 $attribs = array();
1116                 $pairs = array();
1117                 if( !preg_match_all(
1118                         self::getAttribsRegex(),
1119                         $text,
1120                         $pairs,
1121                         PREG_SET_ORDER ) ) {
1122                         return $attribs;
1123                 }
1124
1125                 foreach( $pairs as $set ) {
1126                         $attribute = strtolower( $set[1] );
1127                         $value = Sanitizer::getTagAttributeCallback( $set );
1128
1129                         // Normalize whitespace
1130                         $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
1131                         $value = trim( $value );
1132
1133                         // Decode character references
1134                         $attribs[$attribute] = Sanitizer::decodeCharReferences( $value );
1135                 }
1136                 return $attribs;
1137         }
1138
1139         /**
1140          * Pick the appropriate attribute value from a match set from the
1141          * attribs regex matches.
1142          *
1143          * @param $set Array
1144          * @throws MWException
1145          * @return String
1146          */
1147         private static function getTagAttributeCallback( $set ) {
1148                 if( isset( $set[6] ) ) {
1149                         # Illegal #XXXXXX color with no quotes.
1150                         return $set[6];
1151                 } elseif( isset( $set[5] ) ) {
1152                         # No quotes.
1153                         return $set[5];
1154                 } elseif( isset( $set[4] ) ) {
1155                         # Single-quoted
1156                         return $set[4];
1157                 } elseif( isset( $set[3] ) ) {
1158                         # Double-quoted
1159                         return $set[3];
1160                 } elseif( !isset( $set[2] ) ) {
1161                         # In XHTML, attributes must have a value.
1162                         # For 'reduced' form, return explicitly the attribute name here.
1163                         return $set[1];
1164                 } else {
1165                         throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
1166                 }
1167         }
1168
1169         /**
1170          * Normalize whitespace and character references in an XML source-
1171          * encoded text for an attribute value.
1172          *
1173          * See http://www.w3.org/TR/REC-xml/#AVNormalize for background,
1174          * but note that we're not returning the value, but are returning
1175          * XML source fragments that will be slapped into output.
1176          *
1177          * @param $text String
1178          * @return String
1179          */
1180         private static function normalizeAttributeValue( $text ) {
1181                 return str_replace( '"', '&quot;',
1182                         self::normalizeWhitespace(
1183                                 Sanitizer::normalizeCharReferences( $text ) ) );
1184         }
1185
1186         /**
1187          * @param $text string
1188          * @return mixed
1189          */
1190         private static function normalizeWhitespace( $text ) {
1191                 return preg_replace(
1192                         '/\r\n|[\x20\x0d\x0a\x09]/',
1193                         ' ',
1194                         $text );
1195         }
1196
1197         /**
1198          * Normalizes whitespace in a section name, such as might be returned
1199          * by Parser::stripSectionName(), for use in the id's that are used for
1200          * section links.
1201          *
1202          * @param $section String
1203          * @return String
1204          */
1205         static function normalizeSectionNameWhitespace( $section ) {
1206                 return trim( preg_replace( '/[ _]+/', ' ', $section ) );
1207         }
1208
1209         /**
1210          * Ensure that any entities and character references are legal
1211          * for XML and XHTML specifically. Any stray bits will be
1212          * &amp;-escaped to result in a valid text fragment.
1213          *
1214          * a. named char refs can only be &lt; &gt; &amp; &quot;, others are
1215          *   numericized (this way we're well-formed even without a DTD)
1216          * b. any numeric char refs must be legal chars, not invalid or forbidden
1217          * c. use lower cased "&#x", not "&#X"
1218          * d. fix or reject non-valid attributes
1219          *
1220          * @param $text String
1221          * @return String
1222          * @private
1223          */
1224         static function normalizeCharReferences( $text ) {
1225                 return preg_replace_callback(
1226                         self::CHAR_REFS_REGEX,
1227                         array( 'Sanitizer', 'normalizeCharReferencesCallback' ),
1228                         $text );
1229         }
1230         /**
1231          * @param $matches String
1232          * @return String
1233          */
1234         static function normalizeCharReferencesCallback( $matches ) {
1235                 $ret = null;
1236                 if( $matches[1] != '' ) {
1237                         $ret = Sanitizer::normalizeEntity( $matches[1] );
1238                 } elseif( $matches[2] != '' ) {
1239                         $ret = Sanitizer::decCharReference( $matches[2] );
1240                 } elseif( $matches[3] != '' ) {
1241                         $ret = Sanitizer::hexCharReference( $matches[3] );
1242                 }
1243                 if( is_null( $ret ) ) {
1244                         return htmlspecialchars( $matches[0] );
1245                 } else {
1246                         return $ret;
1247                 }
1248         }
1249
1250         /**
1251          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1252          * return the equivalent numeric entity reference (except for the core &lt;
1253          * &gt; &amp; &quot;). If the entity is a MediaWiki-specific alias, returns
1254          * the HTML equivalent. Otherwise, returns HTML-escaped text of
1255          * pseudo-entity source (eg &amp;foo;)
1256          *
1257          * @param $name String
1258          * @return String
1259          */
1260         static function normalizeEntity( $name ) {
1261                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1262                         return '&' . self::$htmlEntityAliases[$name] . ';';
1263                 } elseif ( in_array( $name,
1264                 array( 'lt', 'gt', 'amp', 'quot' ) ) ) {
1265                         return "&$name;";
1266                 } elseif ( isset( self::$htmlEntities[$name] ) ) {
1267                         return '&#' . self::$htmlEntities[$name] . ';';
1268                 } else {
1269                         return "&amp;$name;";
1270                 }
1271         }
1272
1273         /**
1274          * @param $codepoint
1275          * @return null|string
1276          */
1277         static function decCharReference( $codepoint ) {
1278                 $point = intval( $codepoint );
1279                 if( Sanitizer::validateCodepoint( $point ) ) {
1280                         return sprintf( '&#%d;', $point );
1281                 } else {
1282                         return null;
1283                 }
1284         }
1285
1286         /**
1287          * @param $codepoint
1288          * @return null|string
1289          */
1290         static function hexCharReference( $codepoint ) {
1291                 $point = hexdec( $codepoint );
1292                 if( Sanitizer::validateCodepoint( $point ) ) {
1293                         return sprintf( '&#x%x;', $point );
1294                 } else {
1295                         return null;
1296                 }
1297         }
1298
1299         /**
1300          * Returns true if a given Unicode codepoint is a valid character in XML.
1301          * @param $codepoint Integer
1302          * @return Boolean
1303          */
1304         private static function validateCodepoint( $codepoint ) {
1305                 return ($codepoint ==    0x09)
1306                         || ($codepoint ==    0x0a)
1307                         || ($codepoint ==    0x0d)
1308                         || ($codepoint >=    0x20 && $codepoint <=   0xd7ff)
1309                         || ($codepoint >=  0xe000 && $codepoint <=   0xfffd)
1310                         || ($codepoint >= 0x10000 && $codepoint <= 0x10ffff);
1311         }
1312
1313         /**
1314          * Decode any character references, numeric or named entities,
1315          * in the text and return a UTF-8 string.
1316          *
1317          * @param $text String
1318          * @return String
1319          */
1320         public static function decodeCharReferences( $text ) {
1321                 return preg_replace_callback(
1322                         self::CHAR_REFS_REGEX,
1323                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1324                         $text );
1325         }
1326
1327         /**
1328          * Decode any character references, numeric or named entities,
1329          * in the next and normalize the resulting string. (bug 14952)
1330          *
1331          * This is useful for page titles, not for text to be displayed,
1332          * MediaWiki allows HTML entities to escape normalization as a feature.
1333          *
1334          * @param $text String (already normalized, containing entities)
1335          * @return String (still normalized, without entities)
1336          */
1337         public static function decodeCharReferencesAndNormalize( $text ) {
1338                 global $wgContLang;
1339                 $text = preg_replace_callback(
1340                         self::CHAR_REFS_REGEX,
1341                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1342                         $text, /* limit */ -1, $count );
1343
1344                 if ( $count ) {
1345                         return $wgContLang->normalize( $text );
1346                 } else {
1347                         return $text;
1348                 }
1349         }
1350
1351         /**
1352          * @param $matches String
1353          * @return String
1354          */
1355         static function decodeCharReferencesCallback( $matches ) {
1356                 if( $matches[1] != '' ) {
1357                         return Sanitizer::decodeEntity( $matches[1] );
1358                 } elseif( $matches[2] != '' ) {
1359                         return  Sanitizer::decodeChar( intval( $matches[2] ) );
1360                 } elseif( $matches[3] != '' ) {
1361                         return  Sanitizer::decodeChar( hexdec( $matches[3] ) );
1362                 }
1363                 # Last case should be an ampersand by itself
1364                 return $matches[0];
1365         }
1366
1367         /**
1368          * Return UTF-8 string for a codepoint if that is a valid
1369          * character reference, otherwise U+FFFD REPLACEMENT CHARACTER.
1370          * @param $codepoint Integer
1371          * @return String
1372          * @private
1373          */
1374         static function decodeChar( $codepoint ) {
1375                 if( Sanitizer::validateCodepoint( $codepoint ) ) {
1376                         return codepointToUtf8( $codepoint );
1377                 } else {
1378                         return UTF8_REPLACEMENT;
1379                 }
1380         }
1381
1382         /**
1383          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1384          * return the UTF-8 encoding of that character. Otherwise, returns
1385          * pseudo-entity source (eg "&foo;")
1386          *
1387          * @param $name String
1388          * @return String
1389          */
1390         static function decodeEntity( $name ) {
1391                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1392                         $name = self::$htmlEntityAliases[$name];
1393                 }
1394                 if( isset( self::$htmlEntities[$name] ) ) {
1395                         return codepointToUtf8( self::$htmlEntities[$name] );
1396                 } else {
1397                         return "&$name;";
1398                 }
1399         }
1400
1401         /**
1402          * Fetch the whitelist of acceptable attributes for a given element name.
1403          *
1404          * @param $element String
1405          * @return Array
1406          */
1407         static function attributeWhitelist( $element ) {
1408                 $list = Sanitizer::setupAttributeWhitelist();
1409                 return isset( $list[$element] )
1410                         ? $list[$element]
1411                         : array();
1412         }
1413
1414         /**
1415          * Foreach array key (an allowed HTML element), return an array
1416          * of allowed attributes
1417          * @return Array
1418          */
1419         static function setupAttributeWhitelist() {
1420                 global $wgAllowRdfaAttributes, $wgHtml5, $wgAllowMicrodataAttributes;
1421
1422                 static $whitelist, $staticInitialised;
1423                 $globalContext = implode( '-', compact( 'wgAllowRdfaAttributes', 'wgHtml5', 'wgAllowMicrodataAttributes' ) );
1424
1425                 if ( isset( $whitelist ) && $staticInitialised == $globalContext ) {
1426                         return $whitelist;
1427                 }
1428
1429                 $common = array(
1430                         # HTML
1431                         'id',
1432                         'class',
1433                         'style',
1434                         'lang',
1435                         'dir',
1436                         'title',
1437
1438                         # WAI-ARIA
1439                         'role',
1440                 );
1441
1442                 if ( $wgAllowRdfaAttributes ) {
1443                         #RDFa attributes as specified in section 9 of http://www.w3.org/TR/2008/REC-rdfa-syntax-20081014
1444                         $common = array_merge( $common, array(
1445                             'about', 'property', 'resource', 'datatype', 'typeof',
1446                         ) );
1447                 }
1448
1449                 if ( $wgHtml5 && $wgAllowMicrodataAttributes ) {
1450                         # add HTML5 microdata tages as pecified by http://www.whatwg.org/specs/web-apps/current-work/multipage/microdata.html#the-microdata-model
1451                         $common = array_merge( $common, array(
1452                             'itemid', 'itemprop', 'itemref', 'itemscope', 'itemtype'
1453                         ) );
1454                 }
1455
1456                 $block = array_merge( $common, array( 'align' ) );
1457                 $tablealign = array( 'align', 'char', 'charoff', 'valign' );
1458                 $tablecell = array( 'abbr',
1459                                     'axis',
1460                                     'headers',
1461                                     'scope',
1462                                     'rowspan',
1463                                     'colspan',
1464                                     'nowrap', # deprecated
1465                                     'width',  # deprecated
1466                                     'height', # deprecated
1467                                     'bgcolor' # deprecated
1468                                     );
1469
1470                 # Numbers refer to sections in HTML 4.01 standard describing the element.
1471                 # See: http://www.w3.org/TR/html4/
1472                 $whitelist = array(
1473                         # 7.5.4
1474                         'div'        => $block,
1475                         'center'     => $common, # deprecated
1476                         'span'       => $block, # ??
1477
1478                         # 7.5.5
1479                         'h1'         => $block,
1480                         'h2'         => $block,
1481                         'h3'         => $block,
1482                         'h4'         => $block,
1483                         'h5'         => $block,
1484                         'h6'         => $block,
1485
1486                         # 7.5.6
1487                         # address
1488
1489                         # 8.2.4
1490                         # bdo
1491
1492                         # 9.2.1
1493                         'em'         => $common,
1494                         'strong'     => $common,
1495                         'cite'       => $common,
1496                         'dfn'        => $common,
1497                         'code'       => $common,
1498                         'samp'       => $common,
1499                         'kbd'        => $common,
1500                         'var'        => $common,
1501                         'abbr'       => $common,
1502                         # acronym
1503
1504                         # 9.2.2
1505                         'blockquote' => array_merge( $common, array( 'cite' ) ),
1506                         # q
1507
1508                         # 9.2.3
1509                         'sub'        => $common,
1510                         'sup'        => $common,
1511
1512                         # 9.3.1
1513                         'p'          => $block,
1514
1515                         # 9.3.2
1516                         'br'         => array( 'id', 'class', 'title', 'style', 'clear' ),
1517
1518                         # 9.3.4
1519                         'pre'        => array_merge( $common, array( 'width' ) ),
1520
1521                         # 9.4
1522                         'ins'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1523                         'del'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1524
1525                         # 10.2
1526                         'ul'         => array_merge( $common, array( 'type' ) ),
1527                         'ol'         => array_merge( $common, array( 'type', 'start' ) ),
1528                         'li'         => array_merge( $common, array( 'type', 'value' ) ),
1529
1530                         # 10.3
1531                         'dl'         => $common,
1532                         'dd'         => $common,
1533                         'dt'         => $common,
1534
1535                         # 11.2.1
1536                         'table'      => array_merge( $common,
1537                                                                 array( 'summary', 'width', 'border', 'frame',
1538                                                                                 'rules', 'cellspacing', 'cellpadding',
1539                                                                                 'align', 'bgcolor',
1540                                                                 ) ),
1541
1542                         # 11.2.2
1543                         'caption'    => array_merge( $common, array( 'align' ) ),
1544
1545                         # 11.2.3
1546                         'thead'      => array_merge( $common, $tablealign ),
1547                         'tfoot'      => array_merge( $common, $tablealign ),
1548                         'tbody'      => array_merge( $common, $tablealign ),
1549
1550                         # 11.2.4
1551                         'colgroup'   => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1552                         'col'        => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1553
1554                         # 11.2.5
1555                         'tr'         => array_merge( $common, array( 'bgcolor' ), $tablealign ),
1556
1557                         # 11.2.6
1558                         'td'         => array_merge( $common, $tablecell, $tablealign ),
1559                         'th'         => array_merge( $common, $tablecell, $tablealign ),
1560
1561                         # 12.2 # NOTE: <a> is not allowed directly, but the attrib whitelist is used from the Parser object
1562                         'a'          => array_merge( $common, array( 'href', 'rel', 'rev' ) ), # rel/rev esp. for RDFa
1563
1564                         # 13.2
1565                         # Not usually allowed, but may be used for extension-style hooks
1566                         # such as <math> when it is rasterized, or if $wgAllowImageTag is
1567                         # true
1568                         'img'        => array_merge( $common, array( 'alt', 'src', 'width', 'height' ) ),
1569
1570                         # 15.2.1
1571                         'tt'         => $common,
1572                         'b'          => $common,
1573                         'i'          => $common,
1574                         'big'        => $common,
1575                         'small'      => $common,
1576                         'strike'     => $common,
1577                         's'          => $common,
1578                         'u'          => $common,
1579
1580                         # 15.2.2
1581                         'font'       => array_merge( $common, array( 'size', 'color', 'face' ) ),
1582                         # basefont
1583
1584                         # 15.3
1585                         'hr'         => array_merge( $common, array( 'noshade', 'size', 'width' ) ),
1586
1587                         # XHTML Ruby annotation text module, simple ruby only.
1588                         # http://www.w3c.org/TR/ruby/
1589                         'ruby'       => $common,
1590                         # rbc
1591                         # rtc
1592                         'rb'         => $common,
1593                         'rt'         => $common, #array_merge( $common, array( 'rbspan' ) ),
1594                         'rp'         => $common,
1595
1596                         # MathML root element, where used for extensions
1597                         # 'title' may not be 100% valid here; it's XHTML
1598                         # http://www.w3.org/TR/REC-MathML/
1599                         'math'       => array( 'class', 'style', 'id', 'title' ),
1600
1601                         # HTML 5 section 4.6
1602                         'bdi' => $common,
1603
1604                 );
1605
1606                 if ( $wgHtml5 ) {
1607                         # HTML5 elements, defined by:
1608                         # http://www.whatwg.org/specs/web-apps/current-work/multipage/
1609                         $whitelist += array(
1610                                 'data' => array_merge( $common, array( 'value' ) ),
1611                                 'time' => array_merge( $common, array( 'datetime' ) ),
1612                                 'mark' => $common,
1613
1614                                 // meta and link are only permitted by removeHTMLtags when Microdata
1615                                 // is enabled so we don't bother adding a conditional to hide these
1616                                 // Also meta and link are only valid in WikiText as Microdata elements
1617                                 // (ie: validateTag rejects tags missing the attributes needed for Microdata)
1618                                 // So we don't bother including $common attributes that have no purpose.
1619                                 'meta' => array( 'itemprop', 'content' ),
1620                                 'link' => array( 'itemprop', 'href' ),
1621                         );
1622                 }
1623
1624                 $staticInitialised = $globalContext;
1625
1626                 return $whitelist;
1627         }
1628
1629         /**
1630          * Take a fragment of (potentially invalid) HTML and return
1631          * a version with any tags removed, encoded as plain text.
1632          *
1633          * Warning: this return value must be further escaped for literal
1634          * inclusion in HTML output as of 1.10!
1635          *
1636          * @param $text String: HTML fragment
1637          * @return String
1638          */
1639         static function stripAllTags( $text ) {
1640                 # Actual <tags>
1641                 $text = StringUtils::delimiterReplace( '<', '>', '', $text );
1642
1643                 # Normalize &entities and whitespace
1644                 $text = self::decodeCharReferences( $text );
1645                 $text = self::normalizeWhitespace( $text );
1646
1647                 return $text;
1648         }
1649
1650         /**
1651          * Hack up a private DOCTYPE with HTML's standard entity declarations.
1652          * PHP 4 seemed to know these if you gave it an HTML doctype, but
1653          * PHP 5.1 doesn't.
1654          *
1655          * Use for passing XHTML fragments to PHP's XML parsing functions
1656          *
1657          * @return String
1658          */
1659         static function hackDocType() {
1660                 $out = "<!DOCTYPE html [\n";
1661                 foreach( self::$htmlEntities as $entity => $codepoint ) {
1662                         $out .= "<!ENTITY $entity \"&#$codepoint;\">";
1663                 }
1664                 $out .= "]>\n";
1665                 return $out;
1666         }
1667
1668         /**
1669          * @param $url string
1670          * @return mixed|string
1671          */
1672         static function cleanUrl( $url ) {
1673                 # Normalize any HTML entities in input. They will be
1674                 # re-escaped by makeExternalLink().
1675                 $url = Sanitizer::decodeCharReferences( $url );
1676
1677                 # Escape any control characters introduced by the above step
1678                 $url = preg_replace_callback( '/[\][<>"\\x00-\\x20\\x7F\|]/',
1679                         array( __CLASS__, 'cleanUrlCallback' ), $url );
1680
1681                 # Validate hostname portion
1682                 $matches = array();
1683                 if( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
1684                         list( /* $whole */, $protocol, $host, $rest ) = $matches;
1685
1686                         // Characters that will be ignored in IDNs.
1687                         // http://tools.ietf.org/html/3454#section-3.1
1688                         // Strip them before further processing so blacklists and such work.
1689                         $strip = "/
1690                                 \\s|          # general whitespace
1691                                 \xc2\xad|     # 00ad SOFT HYPHEN
1692                                 \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
1693                                 \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
1694                                 \xe2\x81\xa0| # 2060 WORD JOINER
1695                                 \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
1696                                 \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
1697                                 \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
1698                                 \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
1699                                 \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
1700                                 \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
1701                                 \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
1702                                 [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe0f VARIATION SELECTOR-1-16
1703                                 /xuD";
1704
1705                         $host = preg_replace( $strip, '', $host );
1706
1707                         // @todo FIXME: Validate hostnames here
1708
1709                         return $protocol . $host . $rest;
1710                 } else {
1711                         return $url;
1712                 }
1713         }
1714
1715         /**
1716          * @param $matches array
1717          * @return string
1718          */
1719         static function cleanUrlCallback( $matches ) {
1720                 return urlencode( $matches[0] );
1721         }
1722
1723         /**
1724          * Does a string look like an e-mail address?
1725          *
1726          * This validates an email address using an HTML5 specification found at:
1727          * http://www.whatwg.org/specs/web-apps/current-work/multipage/states-of-the-type-attribute.html#valid-e-mail-address
1728          * Which as of 2011-01-24 says:
1729          *
1730          *   A valid e-mail address is a string that matches the ABNF production
1731          *   1*( atext / "." ) "@" ldh-str *( "." ldh-str ) where atext is defined
1732          *   in RFC 5322 section 3.2.3, and ldh-str is defined in RFC 1034 section
1733          *   3.5.
1734          *
1735          * This function is an implementation of the specification as requested in
1736          * bug 22449.
1737          *
1738          * Client-side forms will use the same standard validation rules via JS or
1739          * HTML 5 validation; additional restrictions can be enforced server-side
1740          * by extensions via the 'isValidEmailAddr' hook.
1741          *
1742          * Note that this validation doesn't 100% match RFC 2822, but is believed
1743          * to be liberal enough for wide use. Some invalid addresses will still
1744          * pass validation here.
1745          *
1746          * @since 1.18
1747          *
1748          * @param $addr String E-mail address
1749          * @return Bool
1750          */
1751         public static function validateEmail( $addr ) {
1752                 $result = null;
1753                 if( !wfRunHooks( 'isValidEmailAddr', array( $addr, &$result ) ) ) {
1754                         return $result;
1755                 }
1756
1757                 // Please note strings below are enclosed in brackets [], this make the
1758                 // hyphen "-" a range indicator. Hence it is double backslashed below.
1759                 // See bug 26948
1760                 $rfc5322_atext   = "a-z0-9!#$%&'*+\\-\/=?^_`{|}~";
1761                 $rfc1034_ldh_str = "a-z0-9\\-";
1762
1763                 $HTML5_email_regexp = "/
1764                 ^                      # start of string
1765                 [$rfc5322_atext\\.]+    # user part which is liberal :p
1766                 @                      # 'apostrophe'
1767                 [$rfc1034_ldh_str]+       # First domain part
1768                 (\\.[$rfc1034_ldh_str]+)*  # Following part prefixed with a dot
1769                 $                      # End of string
1770                 /ix"; // case Insensitive, eXtended
1771
1772                 return (bool) preg_match( $HTML5_email_regexp, $addr );
1773         }
1774 }