includes/Sanitizer.php

   1 <?php
   2 /**
   3  * XHTML sanitizer for MediaWiki
   4  *
   5  * Copyright (C) 2002-2005 Brion Vibber <brion@pobox.com> et al
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @addtogroup Parser
  24  */
  25
  26 /**
  27  * Regular expression to match various types of character references in
  28  * Sanitizer::normalizeCharReferences and Sanitizer::decodeCharReferences
  29  */
  30 define( 'MW_CHAR_REFS_REGEX',
  31         '/&([A-Za-z0-9\x80-\xff]+);
  32          |&\#([0-9]+);
  33          |&\#x([0-9A-Za-z]+);
  34          |&\#X([0-9A-Za-z]+);
  35          |(&)/x' );
  36
  37 /**
  38  * Regular expression to match HTML/XML attribute pairs within a tag.
  39  * Allows some... latitude.
  40  * Used in Sanitizer::fixTagAttributes and Sanitizer::decodeTagAttributes
  41  */
  42 $attrib = '[A-Za-z0-9]';
  43 $space = '[\x09\x0a\x0d\x20]';
  44 define( 'MW_ATTRIBS_REGEX',
  45         "/(?:^|$space)($attrib+)
  46           ($space*=$space*
  47                 (?:
  48                  # The attribute value: quoted or alone
  49                   \"([^<\"]*)\"
  50                  | '([^<']*)'
  51                  |  ([a-zA-Z0-9!#$%&()*,\\-.\\/:;<>?@[\\]^_`{|}~]+)
  52                  |  (\#[0-9a-fA-F]+) # Technically wrong, but lots of
  53                                                          # colors are specified like this.
  54                                                          # We'll be normalizing it.
  55                 )
  56            )?(?=$space|\$)/sx" );
  57
  58 /**
  59  * List of all named character entities defined in HTML 4.01
  60  * http://www.w3.org/TR/html4/sgml/entities.html
  61  * @private
  62  */
  63 global $wgHtmlEntities;
  64 $wgHtmlEntities = array(
  65         'Aacute'   => 193,
  66         'aacute'   => 225,
  67         'Acirc'    => 194,
  68         'acirc'    => 226,
  69         'acute'    => 180,
  70         'AElig'    => 198,
  71         'aelig'    => 230,
  72         'Agrave'   => 192,
  73         'agrave'   => 224,
  74         'alefsym'  => 8501,
  75         'Alpha'    => 913,
  76         'alpha'    => 945,
  77         'amp'      => 38,
  78         'and'      => 8743,
  79         'ang'      => 8736,
  80         'Aring'    => 197,
  81         'aring'    => 229,
  82         'asymp'    => 8776,
  83         'Atilde'   => 195,
  84         'atilde'   => 227,
  85         'Auml'     => 196,
  86         'auml'     => 228,
  87         'bdquo'    => 8222,
  88         'Beta'     => 914,
  89         'beta'     => 946,
  90         'brvbar'   => 166,
  91         'bull'     => 8226,
  92         'cap'      => 8745,
  93         'Ccedil'   => 199,
  94         'ccedil'   => 231,
  95         'cedil'    => 184,
  96         'cent'     => 162,
  97         'Chi'      => 935,
  98         'chi'      => 967,
  99         'circ'     => 710,
 100         'clubs'    => 9827,
 101         'cong'     => 8773,
 102         'copy'     => 169,
 103         'crarr'    => 8629,
 104         'cup'      => 8746,
 105         'curren'   => 164,
 106         'dagger'   => 8224,
 107         'Dagger'   => 8225,
 108         'darr'     => 8595,
 109         'dArr'     => 8659,
 110         'deg'      => 176,
 111         'Delta'    => 916,
 112         'delta'    => 948,
 113         'diams'    => 9830,
 114         'divide'   => 247,
 115         'Eacute'   => 201,
 116         'eacute'   => 233,
 117         'Ecirc'    => 202,
 118         'ecirc'    => 234,
 119         'Egrave'   => 200,
 120         'egrave'   => 232,
 121         'empty'    => 8709,
 122         'emsp'     => 8195,
 123         'ensp'     => 8194,
 124         'Epsilon'  => 917,
 125         'epsilon'  => 949,
 126         'equiv'    => 8801,
 127         'Eta'      => 919,
 128         'eta'      => 951,
 129         'ETH'      => 208,
 130         'eth'      => 240,
 131         'Euml'     => 203,
 132         'euml'     => 235,
 133         'euro'     => 8364,
 134         'exist'    => 8707,
 135         'fnof'     => 402,
 136         'forall'   => 8704,
 137         'frac12'   => 189,
 138         'frac14'   => 188,
 139         'frac34'   => 190,
 140         'frasl'    => 8260,
 141         'Gamma'    => 915,
 142         'gamma'    => 947,
 143         'ge'       => 8805,
 144         'gt'       => 62,
 145         'harr'     => 8596,
 146         'hArr'     => 8660,
 147         'hearts'   => 9829,
 148         'hellip'   => 8230,
 149         'Iacute'   => 205,
 150         'iacute'   => 237,
 151         'Icirc'    => 206,
 152         'icirc'    => 238,
 153         'iexcl'    => 161,
 154         'Igrave'   => 204,
 155         'igrave'   => 236,
 156         'image'    => 8465,
 157         'infin'    => 8734,
 158         'int'      => 8747,
 159         'Iota'     => 921,
 160         'iota'     => 953,
 161         'iquest'   => 191,
 162         'isin'     => 8712,
 163         'Iuml'     => 207,
 164         'iuml'     => 239,
 165         'Kappa'    => 922,
 166         'kappa'    => 954,
 167         'Lambda'   => 923,
 168         'lambda'   => 955,
 169         'lang'     => 9001,
 170         'laquo'    => 171,
 171         'larr'     => 8592,
 172         'lArr'     => 8656,
 173         'lceil'    => 8968,
 174         'ldquo'    => 8220,
 175         'le'       => 8804,
 176         'lfloor'   => 8970,
 177         'lowast'   => 8727,
 178         'loz'      => 9674,
 179         'lrm'      => 8206,
 180         'lsaquo'   => 8249,
 181         'lsquo'    => 8216,
 182         'lt'       => 60,
 183         'macr'     => 175,
 184         'mdash'    => 8212,
 185         'micro'    => 181,
 186         'middot'   => 183,
 187         'minus'    => 8722,
 188         'Mu'       => 924,
 189         'mu'       => 956,
 190         'nabla'    => 8711,
 191         'nbsp'     => 160,
 192         'ndash'    => 8211,
 193         'ne'       => 8800,
 194         'ni'       => 8715,
 195         'not'      => 172,
 196         'notin'    => 8713,
 197         'nsub'     => 8836,
 198         'Ntilde'   => 209,
 199         'ntilde'   => 241,
 200         'Nu'       => 925,
 201         'nu'       => 957,
 202         'Oacute'   => 211,
 203         'oacute'   => 243,
 204         'Ocirc'    => 212,
 205         'ocirc'    => 244,
 206         'OElig'    => 338,
 207         'oelig'    => 339,
 208         'Ograve'   => 210,
 209         'ograve'   => 242,
 210         'oline'    => 8254,
 211         'Omega'    => 937,
 212         'omega'    => 969,
 213         'Omicron'  => 927,
 214         'omicron'  => 959,
 215         'oplus'    => 8853,
 216         'or'       => 8744,
 217         'ordf'     => 170,
 218         'ordm'     => 186,
 219         'Oslash'   => 216,
 220         'oslash'   => 248,
 221         'Otilde'   => 213,
 222         'otilde'   => 245,
 223         'otimes'   => 8855,
 224         'Ouml'     => 214,
 225         'ouml'     => 246,
 226         'para'     => 182,
 227         'part'     => 8706,
 228         'permil'   => 8240,
 229         'perp'     => 8869,
 230         'Phi'      => 934,
 231         'phi'      => 966,
 232         'Pi'       => 928,
 233         'pi'       => 960,
 234         'piv'      => 982,
 235         'plusmn'   => 177,
 236         'pound'    => 163,
 237         'prime'    => 8242,
 238         'Prime'    => 8243,
 239         'prod'     => 8719,
 240         'prop'     => 8733,
 241         'Psi'      => 936,
 242         'psi'      => 968,
 243         'quot'     => 34,
 244         'radic'    => 8730,
 245         'rang'     => 9002,
 246         'raquo'    => 187,
 247         'rarr'     => 8594,
 248         'rArr'     => 8658,
 249         'rceil'    => 8969,
 250         'rdquo'    => 8221,
 251         'real'     => 8476,
 252         'reg'      => 174,
 253         'rfloor'   => 8971,
 254         'Rho'      => 929,
 255         'rho'      => 961,
 256         'rlm'      => 8207,
 257         'rsaquo'   => 8250,
 258         'rsquo'    => 8217,
 259         'sbquo'    => 8218,
 260         'Scaron'   => 352,
 261         'scaron'   => 353,
 262         'sdot'     => 8901,
 263         'sect'     => 167,
 264         'shy'      => 173,
 265         'Sigma'    => 931,
 266         'sigma'    => 963,
 267         'sigmaf'   => 962,
 268         'sim'      => 8764,
 269         'spades'   => 9824,
 270         'sub'      => 8834,
 271         'sube'     => 8838,
 272         'sum'      => 8721,
 273         'sup'      => 8835,
 274         'sup1'     => 185,
 275         'sup2'     => 178,
 276         'sup3'     => 179,
 277         'supe'     => 8839,
 278         'szlig'    => 223,
 279         'Tau'      => 932,
 280         'tau'      => 964,
 281         'there4'   => 8756,
 282         'Theta'    => 920,
 283         'theta'    => 952,
 284         'thetasym' => 977,
 285         'thinsp'   => 8201,
 286         'THORN'    => 222,
 287         'thorn'    => 254,
 288         'tilde'    => 732,
 289         'times'    => 215,
 290         'trade'    => 8482,
 291         'Uacute'   => 218,
 292         'uacute'   => 250,
 293         'uarr'     => 8593,
 294         'uArr'     => 8657,
 295         'Ucirc'    => 219,
 296         'ucirc'    => 251,
 297         'Ugrave'   => 217,
 298         'ugrave'   => 249,
 299         'uml'      => 168,
 300         'upsih'    => 978,
 301         'Upsilon'  => 933,
 302         'upsilon'  => 965,
 303         'Uuml'     => 220,
 304         'uuml'     => 252,
 305         'weierp'   => 8472,
 306         'Xi'       => 926,
 307         'xi'       => 958,
 308         'Yacute'   => 221,
 309         'yacute'   => 253,
 310         'yen'      => 165,
 311         'Yuml'     => 376,
 312         'yuml'     => 255,
 313         'Zeta'     => 918,
 314         'zeta'     => 950,
 315         'zwj'      => 8205,
 316         'zwnj'     => 8204 );
 317
 318 /**
 319  * Character entity aliases accepted by MediaWiki
 320  */
 321 global $wgHtmlEntityAliases;
 322 $wgHtmlEntityAliases = array(
 323         'רלמ' => 'rlm',
 324         'رلم' => 'rlm',
 325 );
 326
 327
 328 /**
 329  * XHTML sanitizer for MediaWiki
 330  * @addtogroup Parser
 331  */
 332 class Sanitizer {
 333         const NONE = 0;
 334         const INITIAL_NONLETTER = 1;
 335
 336         /**
 337          * Cleans up HTML, removes dangerous tags and attributes, and
 338          * removes HTML comments
 339          * @private
 340          * @param string $text
 341          * @param callback $processCallback to do any variable or parameter replacements in HTML attribute values
 342          * @param array $args for the processing callback
 343          * @return string
 344          */
 345         static function removeHTMLtags( $text, $processCallback = null, $args = array(), $extratags = array() ) {
 346                 global $wgUseTidy;
 347
 348                 static $htmlpairs, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
 349                         $htmllist, $listtags, $htmlsingleallowed, $htmlelements, $staticInitialised;
 350
 351                 wfProfileIn( __METHOD__ );
 352
 353                 if ( !$staticInitialised ) {
 354
 355                         $htmlpairs = array_merge( $extratags, array( # Tags that must be closed
 356                                 'b', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
 357                                 'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
 358                                 'strike', 'strong', 'tt', 'var', 'div', 'center',
 359                                 'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
 360                                 'ruby', 'rt' , 'rb' , 'rp', 'p', 'span', 'u'
 361                         ) );
 362                         $htmlsingle = array(
 363                                 'br', 'hr', 'li', 'dt', 'dd'
 364                         );
 365                         $htmlsingleonly = array( # Elements that cannot have close tags
 366                                 'br', 'hr'
 367                         );
 368                         $htmlnest = array( # Tags that can be nested--??
 369                                 'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
 370                                 'dl', 'font', 'big', 'small', 'sub', 'sup', 'span'
 371                         );
 372                         $tabletags = array( # Can only appear inside table, we will close them
 373                                 'td', 'th', 'tr',
 374                         );
 375                         $htmllist = array( # Tags used by list
 376                                 'ul','ol',
 377                         );
 378                         $listtags = array( # Tags that can appear in a list
 379                                 'li',
 380                         );
 381
 382                         $htmlsingleallowed = array_merge( $htmlsingle, $tabletags );
 383                         $htmlelements = array_merge( $htmlsingle, $htmlpairs, $htmlnest );
 384
 385                         # Convert them all to hashtables for faster lookup
 386                         $vars = array( 'htmlpairs', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
 387                                 'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelements' );
 388                         foreach ( $vars as $var ) {
 389                                 $$var = array_flip( $$var );
 390                         }
 391                         $staticInitialised = true;
 392                 }
 393
 394                 # Remove HTML comments
 395                 $text = Sanitizer::removeHTMLcomments( $text );
 396                 $bits = explode( '<', $text );
 397                 $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 398                 if(!$wgUseTidy) {
 399                         $tagstack = $tablestack = array();
 400                         foreach ( $bits as $x ) {
 401                                 $regs = array();
 402                                 if( preg_match( '!^(/?)(\\w+)([^>]*?)(/{0,1}>)([^<]*)$!', $x, $regs ) ) {
 403                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 404                                 } else {
 405                                         $slash = $t = $params = $brace = $rest = null;
 406                                 }
 407
 408                                 $badtag = 0 ;
 409                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 410                                         # Check our stack
 411                                         if ( $slash ) {
 412                                                 # Closing a tag...
 413                                                 if( isset( $htmlsingleonly[$t] ) ) {
 414                                                         $badtag = 1;
 415                                                 } elseif ( ( $ot = @array_pop( $tagstack ) ) != $t ) {
 416                                                         if ( isset( $htmlsingleallowed[$ot] ) ) {
 417                                                                 # Pop all elements with an optional close tag
 418                                                                 # and see if we find a match below them
 419                                                                 $optstack = array();
 420                                                                 array_push ($optstack, $ot);
 421                                                                 while ( ( ( $ot = @array_pop( $tagstack ) ) != $t ) &&
 422                                                                                 isset( $htmlsingleallowed[$ot] ) )
 423                                                                 {
 424                                                                         array_push ($optstack, $ot);
 425                                                                 }
 426                                                                 if ( $t != $ot ) {
 427                                                                         # No match. Push the optinal elements back again
 428                                                                         $badtag = 1;
 429                                                                         while ( $ot = @array_pop( $optstack ) ) {
 430                                                                                 array_push( $tagstack, $ot );
 431                                                                         }
 432                                                                 }
 433                                                         } else {
 434                                                                 @array_push( $tagstack, $ot );
 435                                                                 # <li> can be nested in <ul> or <ol>, skip those cases:
 436                                                                 if(!(isset( $htmllist[$ot] ) && isset( $listtags[$t] ) )) {
 437                                                                         $badtag = 1;
 438                                                                 }
 439                                                         }
 440                                                 } else {
 441                                                         if ( $t == 'table' ) {
 442                                                                 $tagstack = array_pop( $tablestack );
 443                                                         }
 444                                                 }
 445                                                 $newparams = '';
 446                                         } else {
 447                                                 # Keep track for later
 448                                                 if ( isset( $tabletags[$t] ) &&
 449                                                 ! in_array( 'table', $tagstack ) ) {
 450                                                         $badtag = 1;
 451                                                 } else if ( in_array( $t, $tagstack ) &&
 452                                                 ! isset( $htmlnest [$t ] ) ) {
 453                                                         $badtag = 1 ;
 454                                                 # Is it a self closed htmlpair ? (bug 5487)
 455                                                 } else if( $brace == '/>' &&
 456                                                 isset( $htmlpairs[$t] ) ) {
 457                                                         $badtag = 1;
 458                                                 } elseif( isset( $htmlsingleonly[$t] ) ) {
 459                                                         # Hack to force empty tag for uncloseable elements
 460                                                         $brace = '/>';
 461                                                 } else if( isset( $htmlsingle[$t] ) ) {
 462                                                         # Hack to not close $htmlsingle tags
 463                                                         $brace = NULL;
 464                                                 } else if( isset( $tabletags[$t] )
 465                                                 &&  in_array($t ,$tagstack) ) {
 466                                                         // New table tag but forgot to close the previous one
 467                                                         $text .= "</$t>";
 468                                                 } else {
 469                                                         if ( $t == 'table' ) {
 470                                                                 array_push( $tablestack, $tagstack );
 471                                                                 $tagstack = array();
 472                                                         }
 473                                                         array_push( $tagstack, $t );
 474                                                 }
 475
 476                                                 # Replace any variables or template parameters with
 477                                                 # plaintext results.
 478                                                 if( is_callable( $processCallback ) ) {
 479                                                         call_user_func_array( $processCallback, array( &$params, $args ) );
 480                                                 }
 481
 482                                                 # Strip non-approved attributes from the tag
 483                                                 $newparams = Sanitizer::fixTagAttributes( $params, $t );
 484                                         }
 485                                         if ( ! $badtag ) {
 486                                                 $rest = str_replace( '>', '&gt;', $rest );
 487                                                 $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
 488                                                 $text .= "<$slash$t$newparams$close>$rest";
 489                                                 continue;
 490                                         }
 491                                 }
 492                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 493                         }
 494                         # Close off any remaining tags
 495                         while ( is_array( $tagstack ) && ($t = array_pop( $tagstack )) ) {
 496                                 $text .= "</$t>\n";
 497                                 if ( $t == 'table' ) { $tagstack = array_pop( $tablestack ); }
 498                         }
 499                 } else {
 500                         # this might be possible using tidy itself
 501                         foreach ( $bits as $x ) {
 502                                 preg_match( '/^(\\/?)(\\w+)([^>]*?)(\\/{0,1}>)([^<]*)$/',
 503                                 $x, $regs );
 504                                 @list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 505                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 506                                         if( is_callable( $processCallback ) ) {
 507                                                 call_user_func_array( $processCallback, array( &$params, $args ) );
 508                                         }
 509                                         $newparams = Sanitizer::fixTagAttributes( $params, $t );
 510                                         $rest = str_replace( '>', '&gt;', $rest );
 511                                         $text .= "<$slash$t$newparams$brace$rest";
 512                                 } else {
 513                                         $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 514                                 }
 515                         }
 516                 }
 517                 wfProfileOut( __METHOD__ );
 518                 return $text;
 519         }
 520
 521         /**
 522          * Remove '<!--', '-->', and everything between.
 523          * To avoid leaving blank lines, when a comment is both preceded
 524          * and followed by a newline (ignoring spaces), trim leading and
 525          * trailing spaces and one of the newlines.
 526          *
 527          * @private
 528          * @param string $text
 529          * @return string
 530          */
 531         static function removeHTMLcomments( $text ) {
 532                 wfProfileIn( __METHOD__ );
 533                 while (($start = strpos($text, '<!--')) !== false) {
 534                         $end = strpos($text, '-->', $start + 4);
 535                         if ($end === false) {
 536                                 # Unterminated comment; bail out
 537                                 break;
 538                         }
 539
 540                         $end += 3;
 541
 542                         # Trim space and newline if the comment is both
 543                         # preceded and followed by a newline
 544                         $spaceStart = max($start - 1, 0);
 545                         $spaceLen = $end - $spaceStart;
 546                         while (substr($text, $spaceStart, 1) === ' ' && $spaceStart > 0) {
 547                                 $spaceStart--;
 548                                 $spaceLen++;
 549                         }
 550                         while (substr($text, $spaceStart + $spaceLen, 1) === ' ')
 551                                 $spaceLen++;
 552                         if (substr($text, $spaceStart, 1) === "\n" and substr($text, $spaceStart + $spaceLen, 1) === "\n") {
 553                                 # Remove the comment, leading and trailing
 554                                 # spaces, and leave only one newline.
 555                                 $text = substr_replace($text, "\n", $spaceStart, $spaceLen + 1);
 556                         }
 557                         else {
 558                                 # Remove just the comment.
 559                                 $text = substr_replace($text, '', $start, $end - $start);
 560                         }
 561                 }
 562                 wfProfileOut( __METHOD__ );
 563                 return $text;
 564         }
 565
 566         /**
 567          * Take an array of attribute names and values and normalize or discard
 568          * illegal values for the given element type.
 569          *
 570          * - Discards attributes not on a whitelist for the given element
 571          * - Unsafe style attributes are discarded
 572          * - Invalid id attributes are reencoded
 573          *
 574          * @param array $attribs
 575          * @param string $element
 576          * @return array
 577          *
 578          * @todo Check for legal values where the DTD limits things.
 579          * @todo Check for unique id attribute :P
 580          */
 581         static function validateTagAttributes( $attribs, $element ) {
 582                 return Sanitizer::validateAttributes( $attribs,
 583                         Sanitizer::attributeWhitelist( $element ) );
 584         }
 585
 586         /**
 587          * Take an array of attribute names and values and normalize or discard
 588          * illegal values for the given whitelist.
 589          *
 590          * - Discards attributes not the given whitelist
 591          * - Unsafe style attributes are discarded
 592          * - Invalid id attributes are reencoded
 593          *
 594          * @param array $attribs
 595          * @param array $whitelist list of allowed attribute names
 596          * @return array
 597          *
 598          * @todo Check for legal values where the DTD limits things.
 599          * @todo Check for unique id attribute :P
 600          */
 601         static function validateAttributes( $attribs, $whitelist ) {
 602                 $whitelist = array_flip( $whitelist );
 603                 $out = array();
 604                 foreach( $attribs as $attribute => $value ) {
 605                         if( !isset( $whitelist[$attribute] ) ) {
 606                                 continue;
 607                         }
 608                         # Strip javascript "expression" from stylesheets.
 609                         # http://msdn.microsoft.com/workshop/author/dhtml/overview/recalc.asp
 610                         if( $attribute == 'style' ) {
 611                                 $value = Sanitizer::checkCss( $value );
 612                                 if( $value === false ) {
 613                                         # haxx0r
 614                                         continue;
 615                                 }
 616                         }
 617
 618                         if ( $attribute === 'id' )
 619                                 $value = Sanitizer::escapeId( $value );
 620
 621                         // If this attribute was previously set, override it.
 622                         // Output should only have one attribute of each name.
 623                         $out[$attribute] = $value;
 624                 }
 625                 return $out;
 626         }
 627
 628         /**
 629          * Merge two sets of HTML attributes.
 630          * Conflicting items in the second set will override those
 631          * in the first, except for 'class' attributes which will be
 632          * combined.
 633          *
 634          * @todo implement merging for other attributes such as style
 635          * @param array $a
 636          * @param array $b
 637          * @return array
 638          */
 639         static function mergeAttributes( $a, $b ) {
 640                 $out = array_merge( $a, $b );
 641                 if( isset( $a['class'] )
 642                         && isset( $b['class'] )
 643                         && $a['class'] !== $b['class'] ) {
 644
 645                         $out['class'] = implode( ' ',
 646                                 array_unique(
 647                                         preg_split( '/\s+/',
 648                                                 $a['class'] . ' ' . $b['class'],
 649                                                 -1,
 650                                                 PREG_SPLIT_NO_EMPTY ) ) );
 651                 }
 652                 return $out;
 653         }
 654
 655         /**
 656          * Pick apart some CSS and check it for forbidden or unsafe structures.
 657          * Returns a sanitized string, or false if it was just too evil.
 658          *
 659          * Currently URL references, 'expression', 'tps' are forbidden.
 660          *
 661          * @param string $value
 662          * @return mixed
 663          */
 664         static function checkCss( $value ) {
 665                 $stripped = Sanitizer::decodeCharReferences( $value );
 666
 667                 // Remove any comments; IE gets token splitting wrong
 668                 $stripped = StringUtils::delimiterReplace( '/*', '*/', ' ', $stripped );
 669
 670                 $value = $stripped;
 671
 672                 // ... and continue checks
 673                 $stripped = preg_replace( '!\\\\([0-9A-Fa-f]{1,6})[ \\n\\r\\t\\f]?!e',
 674                         'codepointToUtf8(hexdec("$1"))', $stripped );
 675                 $stripped = str_replace( '\\', '', $stripped );
 676                 if( preg_match( '/(?:expression|tps*:\/\/|url\\s*\().*/is',
 677                                 $stripped ) ) {
 678                         # haxx0r
 679                         return false;
 680                 }
 681
 682                 return $value;
 683         }
 684
 685         /**
 686          * Take a tag soup fragment listing an HTML element's attributes
 687          * and normalize it to well-formed XML, discarding unwanted attributes.
 688          * Output is safe for further wikitext processing, with escaping of
 689          * values that could trigger problems.
 690          *
 691          * - Normalizes attribute names to lowercase
 692          * - Discards attributes not on a whitelist for the given element
 693          * - Turns broken or invalid entities into plaintext
 694          * - Double-quotes all attribute values
 695          * - Attributes without values are given the name as attribute
 696          * - Double attributes are discarded
 697          * - Unsafe style attributes are discarded
 698          * - Prepends space if there are attributes.
 699          *
 700          * @param string $text
 701          * @param string $element
 702          * @return string
 703          */
 704         static function fixTagAttributes( $text, $element ) {
 705                 if( trim( $text ) == '' ) {
 706                         return '';
 707                 }
 708
 709                 $stripped = Sanitizer::validateTagAttributes(
 710                         Sanitizer::decodeTagAttributes( $text ), $element );
 711
 712                 $attribs = array();
 713                 foreach( $stripped as $attribute => $value ) {
 714                         $encAttribute = htmlspecialchars( $attribute );
 715                         $encValue = Sanitizer::safeEncodeAttribute( $value );
 716
 717                         $attribs[] = "$encAttribute=\"$encValue\"";
 718                 }
 719
 720         if ( !wfRunHooks( 'SanitizerAfterFixTagAttributes', array( $text, $element, &$attribs ) ) ) {
 721                 return ''; # They want to abort this.
 722         }
 723                 return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
 724         }
 725
 726         /**
 727          * Encode an attribute value for HTML output.
 728          * @param $text
 729          * @return HTML-encoded text fragment
 730          */
 731         static function encodeAttribute( $text ) {
 732                 $encValue = htmlspecialchars( $text, ENT_QUOTES );
 733
 734                 // Whitespace is normalized during attribute decoding,
 735                 // so if we've been passed non-spaces we must encode them
 736                 // ahead of time or they won't be preserved.
 737                 $encValue = strtr( $encValue, array(
 738                         "\n" => '&#10;',
 739                         "\r" => '&#13;',
 740                         "\t" => '&#9;',
 741                 ) );
 742
 743                 return $encValue;
 744         }
 745
 746         /**
 747          * Encode an attribute value for HTML tags, with extra armoring
 748          * against further wiki processing.
 749          * @param $text
 750          * @return HTML-encoded text fragment
 751          */
 752         static function safeEncodeAttribute( $text ) {
 753                 $encValue = Sanitizer::encodeAttribute( $text );
 754
 755                 # Templates and links may be expanded in later parsing,
 756                 # creating invalid or dangerous output. Suppress this.
 757                 $encValue = strtr( $encValue, array(
 758                         '<'    => '&lt;',   // This should never happen,
 759                         '>'    => '&gt;',   // we've received invalid input
 760                         '"'    => '&quot;', // which should have been escaped.
 761                         '{'    => '&#123;',
 762                         '['    => '&#91;',
 763                         "''"   => '&#39;&#39;',
 764                         'ISBN' => '&#73;SBN',
 765                         'RFC'  => '&#82;FC',
 766                         'PMID' => '&#80;MID',
 767                         '|'    => '&#124;',
 768                         '__'   => '&#95;_',
 769                 ) );
 770
 771                 # Stupid hack
 772                 $encValue = preg_replace_callback(
 773                         '/(' . wfUrlProtocols() . ')/',
 774                         array( 'Sanitizer', 'armorLinksCallback' ),
 775                         $encValue );
 776                 return $encValue;
 777         }
 778
 779         /**
 780          * Given a value escape it so that it can be used in an id attribute and
 781          * return it, this does not validate the value however (see first link)
 782          *
 783          * @see http://www.w3.org/TR/html401/types.html#type-name Valid characters
 784          *                                                          in the id and
 785          *                                                          name attributes
 786          * @see http://www.w3.org/TR/html401/struct/links.html#h-12.2.3 Anchors with the id attribute
 787          *
 788          * @param string $id    Id to validate
 789          * @param int    $flags Currently only two values: Sanitizer::INITIAL_NONLETTER
 790          *                      (default) permits initial non-letter characters,
 791          *                      such as if you're adding a prefix to them.
 792          *                      Sanitizer::NONE will prepend an 'x' if the id
 793          *                      would otherwise start with a nonletter.
 794          * @return string
 795          */
 796         static function escapeId( $id, $flags = Sanitizer::INITIAL_NONLETTER ) {
 797                 static $replace = array(
 798                         '%3A' => ':',
 799                         '%' => '.'
 800                 );
 801
 802                 $id = urlencode( Sanitizer::decodeCharReferences( strtr( $id, ' ', '_' ) ) );
 803                 $id = str_replace( array_keys( $replace ), array_values( $replace ), $id );
 804
 805                 if( ~$flags & Sanitizer::INITIAL_NONLETTER
 806                 && !preg_match( '/[a-zA-Z]/', $id[0] ) ) {
 807                         // Initial character must be a letter!
 808                         $id = "x$id";
 809                 }
 810                 return $id;
 811         }
 812
 813         /**
 814          * Given a value, escape it so that it can be used as a CSS class and
 815          * return it.
 816          *
 817          * @todo For extra validity, input should be validated UTF-8.
 818          *
 819          * @see http://www.w3.org/TR/CSS21/syndata.html Valid characters/format
 820          *
 821          * @param string $class
 822          * @return string
 823          */
 824         static function escapeClass( $class ) {
 825                 // Convert ugly stuff to underscores and kill underscores in ugly places
 826                 return rtrim(preg_replace(
 827                         array('/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/','/_+/'),
 828                         '_',
 829                         $class ), '_');
 830         }
 831
 832         /**
 833          * Regex replace callback for armoring links against further processing.
 834          * @param array $matches
 835          * @return string
 836          * @private
 837          */
 838         private static function armorLinksCallback( $matches ) {
 839                 return str_replace( ':', '&#58;', $matches[1] );
 840         }
 841
 842         /**
 843          * Return an associative array of attribute names and values from
 844          * a partial tag string. Attribute names are forces to lowercase,
 845          * character references are decoded to UTF-8 text.
 846          *
 847          * @param string
 848          * @return array
 849          */
 850         static function decodeTagAttributes( $text ) {
 851                 $attribs = array();
 852
 853                 if( trim( $text ) == '' ) {
 854                         return $attribs;
 855                 }
 856
 857                 $pairs = array();
 858                 if( !preg_match_all(
 859                         MW_ATTRIBS_REGEX,
 860                         $text,
 861                         $pairs,
 862                         PREG_SET_ORDER ) ) {
 863                         return $attribs;
 864                 }
 865
 866                 foreach( $pairs as $set ) {
 867                         $attribute = strtolower( $set[1] );
 868                         $value = Sanitizer::getTagAttributeCallback( $set );
 869
 870                         // Normalize whitespace
 871                         $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
 872                         $value = trim( $value );
 873
 874                         // Decode character references
 875                         $attribs[$attribute] = Sanitizer::decodeCharReferences( $value );
 876                 }
 877                 return $attribs;
 878         }
 879
 880         /**
 881          * Pick the appropriate attribute value from a match set from the
 882          * MW_ATTRIBS_REGEX matches.
 883          *
 884          * @param array $set
 885          * @return string
 886          * @private
 887          */
 888         private static function getTagAttributeCallback( $set ) {
 889                 if( isset( $set[6] ) ) {
 890                         # Illegal #XXXXXX color with no quotes.
 891                         return $set[6];
 892                 } elseif( isset( $set[5] ) ) {
 893                         # No quotes.
 894                         return $set[5];
 895                 } elseif( isset( $set[4] ) ) {
 896                         # Single-quoted
 897                         return $set[4];
 898                 } elseif( isset( $set[3] ) ) {
 899                         # Double-quoted
 900                         return $set[3];
 901                 } elseif( !isset( $set[2] ) ) {
 902                         # In XHTML, attributes must have a value.
 903                         # For 'reduced' form, return explicitly the attribute name here.
 904                         return $set[1];
 905                 } else {
 906                         throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
 907                 }
 908         }
 909
 910         /**
 911          * Normalize whitespace and character references in an XML source-
 912          * encoded text for an attribute value.
 913          *
 914          * See http://www.w3.org/TR/REC-xml/#AVNormalize for background,
 915          * but note that we're not returning the value, but are returning
 916          * XML source fragments that will be slapped into output.
 917          *
 918          * @param string $text
 919          * @return string
 920          * @private
 921          */
 922         private static function normalizeAttributeValue( $text ) {
 923                 return str_replace( '"', '&quot;',
 924                         self::normalizeWhitespace(
 925                                 Sanitizer::normalizeCharReferences( $text ) ) );
 926         }
 927
 928         private static function normalizeWhitespace( $text ) {
 929                 return preg_replace(
 930                         '/\r\n|[\x20\x0d\x0a\x09]/',
 931                         ' ',
 932                         $text );
 933         }
 934
 935         /**
 936          * Ensure that any entities and character references are legal
 937          * for XML and XHTML specifically. Any stray bits will be
 938          * &amp;-escaped to result in a valid text fragment.
 939          *
 940          * a. any named char refs must be known in XHTML
 941          * b. any numeric char refs must be legal chars, not invalid or forbidden
 942          * c. use &#x, not &#X
 943          * d. fix or reject non-valid attributes
 944          *
 945          * @param string $text
 946          * @return string
 947          * @private
 948          */
 949         static function normalizeCharReferences( $text ) {
 950                 return preg_replace_callback(
 951                         MW_CHAR_REFS_REGEX,
 952                         array( 'Sanitizer', 'normalizeCharReferencesCallback' ),
 953                         $text );
 954         }
 955         /**
 956          * @param string $matches
 957          * @return string
 958          */
 959         static function normalizeCharReferencesCallback( $matches ) {
 960                 $ret = null;
 961                 if( $matches[1] != '' ) {
 962                         $ret = Sanitizer::normalizeEntity( $matches[1] );
 963                 } elseif( $matches[2] != '' ) {
 964                         $ret = Sanitizer::decCharReference( $matches[2] );
 965                 } elseif( $matches[3] != ''  ) {
 966                         $ret = Sanitizer::hexCharReference( $matches[3] );
 967                 } elseif( $matches[4] != '' ) {
 968                         $ret = Sanitizer::hexCharReference( $matches[4] );
 969                 }
 970                 if( is_null( $ret ) ) {
 971                         return htmlspecialchars( $matches[0] );
 972                 } else {
 973                         return $ret;
 974                 }
 975         }
 976
 977         /**
 978          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
 979          * return the named entity reference as is. If the entity is a
 980          * MediaWiki-specific alias, returns the HTML equivalent. Otherwise,
 981          * returns HTML-escaped text of pseudo-entity source (eg &amp;foo;)
 982          *
 983          * @param string $name
 984          * @return string
 985          * @static
 986          */
 987         static function normalizeEntity( $name ) {
 988                 global $wgHtmlEntities, $wgHtmlEntityAliases;
 989                 if ( isset( $wgHtmlEntityAliases[$name] ) ) {
 990                         return "&{$wgHtmlEntityAliases[$name]};";
 991                 } elseif( isset( $wgHtmlEntities[$name] ) ) {
 992                         return "&$name;";
 993                 } else {
 994                         return "&amp;$name;";
 995                 }
 996         }
 997
 998         static function decCharReference( $codepoint ) {
 999                 $point = intval( $codepoint );
1000                 if( Sanitizer::validateCodepoint( $point ) ) {
1001                         return sprintf( '&#%d;', $point );
1002                 } else {
1003                         return null;
1004                 }
1005         }
1006
1007         static function hexCharReference( $codepoint ) {
1008                 $point = hexdec( $codepoint );
1009                 if( Sanitizer::validateCodepoint( $point ) ) {
1010                         return sprintf( '&#x%x;', $point );
1011                 } else {
1012                         return null;
1013                 }
1014         }
1015
1016         /**
1017          * Returns true if a given Unicode codepoint is a valid character in XML.
1018          * @param int $codepoint
1019          * @return bool
1020          */
1021         private static function validateCodepoint( $codepoint ) {
1022                 return ($codepoint ==    0x09)
1023                         || ($codepoint ==    0x0a)
1024                         || ($codepoint ==    0x0d)
1025                         || ($codepoint >=    0x20 && $codepoint <=   0xd7ff)
1026                         || ($codepoint >=  0xe000 && $codepoint <=   0xfffd)
1027                         || ($codepoint >= 0x10000 && $codepoint <= 0x10ffff);
1028         }
1029
1030         /**
1031          * Decode any character references, numeric or named entities,
1032          * in the text and return a UTF-8 string.
1033          *
1034          * @param string $text
1035          * @return string
1036          * @public
1037          * @static
1038          */
1039         public static function decodeCharReferences( $text ) {
1040                 return preg_replace_callback(
1041                         MW_CHAR_REFS_REGEX,
1042                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1043                         $text );
1044         }
1045
1046         /**
1047          * @param string $matches
1048          * @return string
1049          */
1050         static function decodeCharReferencesCallback( $matches ) {
1051                 if( $matches[1] != '' ) {
1052                         return Sanitizer::decodeEntity( $matches[1] );
1053                 } elseif( $matches[2] != '' ) {
1054                         return  Sanitizer::decodeChar( intval( $matches[2] ) );
1055                 } elseif( $matches[3] != ''  ) {
1056                         return  Sanitizer::decodeChar( hexdec( $matches[3] ) );
1057                 } elseif( $matches[4] != '' ) {
1058                         return  Sanitizer::decodeChar( hexdec( $matches[4] ) );
1059                 }
1060                 # Last case should be an ampersand by itself
1061                 return $matches[0];
1062         }
1063
1064         /**
1065          * Return UTF-8 string for a codepoint if that is a valid
1066          * character reference, otherwise U+FFFD REPLACEMENT CHARACTER.
1067          * @param int $codepoint
1068          * @return string
1069          * @private
1070          */
1071         static function decodeChar( $codepoint ) {
1072                 if( Sanitizer::validateCodepoint( $codepoint ) ) {
1073                         return codepointToUtf8( $codepoint );
1074                 } else {
1075                         return UTF8_REPLACEMENT;
1076                 }
1077         }
1078
1079         /**
1080          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1081          * return the UTF-8 encoding of that character. Otherwise, returns
1082          * pseudo-entity source (eg &foo;)
1083          *
1084          * @param string $name
1085          * @return string
1086          */
1087         static function decodeEntity( $name ) {
1088                 global $wgHtmlEntities, $wgHtmlEntityAliases;
1089                 if ( isset( $wgHtmlEntityAliases[$name] ) ) {
1090                         $name = $wgHtmlEntityAliases[$name];
1091                 }
1092                 if( isset( $wgHtmlEntities[$name] ) ) {
1093                         return codepointToUtf8( $wgHtmlEntities[$name] );
1094                 } else {
1095                         return "&$name;";
1096                 }
1097         }
1098
1099         /**
1100          * Fetch the whitelist of acceptable attributes for a given
1101          * element name.
1102          *
1103          * @param string $element
1104          * @return array
1105          */
1106         static function attributeWhitelist( $element ) {
1107                 static $list;
1108                 if( !isset( $list ) ) {
1109                         $list = Sanitizer::setupAttributeWhitelist();
1110                 }
1111                 return isset( $list[$element] )
1112                         ? $list[$element]
1113                         : array();
1114         }
1115
1116         /**
1117          * @todo Document it a bit
1118          * @return array
1119          */
1120         static function setupAttributeWhitelist() {
1121                 $common = array( 'id', 'class', 'lang', 'dir', 'title', 'style' );
1122                 $block = array_merge( $common, array( 'align' ) );
1123                 $tablealign = array( 'align', 'char', 'charoff', 'valign' );
1124                 $tablecell = array( 'abbr',
1125                                     'axis',
1126                                     'headers',
1127                                     'scope',
1128                                     'rowspan',
1129                                     'colspan',
1130                                     'nowrap', # deprecated
1131                                     'width',  # deprecated
1132                                     'height', # deprecated
1133                                     'bgcolor' # deprecated
1134                                     );
1135
1136                 # Numbers refer to sections in HTML 4.01 standard describing the element.
1137                 # See: http://www.w3.org/TR/html4/
1138                 $whitelist = array (
1139                         # 7.5.4
1140                         'div'        => $block,
1141                         'center'     => $common, # deprecated
1142                         'span'       => $block, # ??
1143
1144                         # 7.5.5
1145                         'h1'         => $block,
1146                         'h2'         => $block,
1147                         'h3'         => $block,
1148                         'h4'         => $block,
1149                         'h5'         => $block,
1150                         'h6'         => $block,
1151
1152                         # 7.5.6
1153                         # address
1154
1155                         # 8.2.4
1156                         # bdo
1157
1158                         # 9.2.1
1159                         'em'         => $common,
1160                         'strong'     => $common,
1161                         'cite'       => $common,
1162                         # dfn
1163                         'code'       => $common,
1164                         # samp
1165                         # kbd
1166                         'var'        => $common,
1167                         # abbr
1168                         # acronym
1169
1170                         # 9.2.2
1171                         'blockquote' => array_merge( $common, array( 'cite' ) ),
1172                         # q
1173
1174                         # 9.2.3
1175                         'sub'        => $common,
1176                         'sup'        => $common,
1177
1178                         # 9.3.1
1179                         'p'          => $block,
1180
1181                         # 9.3.2
1182                         'br'         => array( 'id', 'class', 'title', 'style', 'clear' ),
1183
1184                         # 9.3.4
1185                         'pre'        => array_merge( $common, array( 'width' ) ),
1186
1187                         # 9.4
1188                         'ins'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1189                         'del'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1190
1191                         # 10.2
1192                         'ul'         => array_merge( $common, array( 'type' ) ),
1193                         'ol'         => array_merge( $common, array( 'type', 'start' ) ),
1194                         'li'         => array_merge( $common, array( 'type', 'value' ) ),
1195
1196                         # 10.3
1197                         'dl'         => $common,
1198                         'dd'         => $common,
1199                         'dt'         => $common,
1200
1201                         # 11.2.1
1202                         'table'      => array_merge( $common,
1203                                                                 array( 'summary', 'width', 'border', 'frame',
1204                                                                                 'rules', 'cellspacing', 'cellpadding',
1205                                                                                 'align', 'bgcolor',
1206                                                                 ) ),
1207
1208                         # 11.2.2
1209                         'caption'    => array_merge( $common, array( 'align' ) ),
1210
1211                         # 11.2.3
1212                         'thead'      => array_merge( $common, $tablealign ),
1213                         'tfoot'      => array_merge( $common, $tablealign ),
1214                         'tbody'      => array_merge( $common, $tablealign ),
1215
1216                         # 11.2.4
1217                         'colgroup'   => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1218                         'col'        => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1219
1220                         # 11.2.5
1221                         'tr'         => array_merge( $common, array( 'bgcolor' ), $tablealign ),
1222
1223                         # 11.2.6
1224                         'td'         => array_merge( $common, $tablecell, $tablealign ),
1225                         'th'         => array_merge( $common, $tablecell, $tablealign ),
1226
1227                         # 13.2
1228                         # Not usually allowed, but may be used for extension-style hooks
1229                         # such as <math> when it is rasterized
1230                         'img'        => array_merge( $common, array( 'alt' ) ),
1231
1232                         # 15.2.1
1233                         'tt'         => $common,
1234                         'b'          => $common,
1235                         'i'          => $common,
1236                         'big'        => $common,
1237                         'small'      => $common,
1238                         'strike'     => $common,
1239                         's'          => $common,
1240                         'u'          => $common,
1241
1242                         # 15.2.2
1243                         'font'       => array_merge( $common, array( 'size', 'color', 'face' ) ),
1244                         # basefont
1245
1246                         # 15.3
1247                         'hr'         => array_merge( $common, array( 'noshade', 'size', 'width' ) ),
1248
1249                         # XHTML Ruby annotation text module, simple ruby only.
1250                         # http://www.w3c.org/TR/ruby/
1251                         'ruby'       => $common,
1252                         # rbc
1253                         # rtc
1254                         'rb'         => $common,
1255                         'rt'         => $common, #array_merge( $common, array( 'rbspan' ) ),
1256                         'rp'         => $common,
1257
1258                         # MathML root element, where used for extensions
1259                         # 'title' may not be 100% valid here; it's XHTML
1260                         # http://www.w3.org/TR/REC-MathML/
1261                         'math'       => array( 'class', 'style', 'id', 'title' ),
1262                         );
1263                 return $whitelist;
1264         }
1265
1266         /**
1267          * Take a fragment of (potentially invalid) HTML and return
1268          * a version with any tags removed, encoded as plain text.
1269          *
1270          * Warning: this return value must be further escaped for literal
1271          * inclusion in HTML output as of 1.10!
1272          *
1273          * @param string $text HTML fragment
1274          * @return string
1275          */
1276         static function stripAllTags( $text ) {
1277                 # Actual <tags>
1278                 $text = StringUtils::delimiterReplace( '<', '>', '', $text );
1279
1280                 # Normalize &entities and whitespace
1281                 $text = self::decodeCharReferences( $text );
1282                 $text = self::normalizeWhitespace( $text );
1283
1284                 return $text;
1285         }
1286
1287         /**
1288          * Hack up a private DOCTYPE with HTML's standard entity declarations.
1289          * PHP 4 seemed to know these if you gave it an HTML doctype, but
1290          * PHP 5.1 doesn't.
1291          *
1292          * Use for passing XHTML fragments to PHP's XML parsing functions
1293          *
1294          * @return string
1295          * @static
1296          */
1297         static function hackDocType() {
1298                 global $wgHtmlEntities;
1299                 $out = "<!DOCTYPE html [\n";
1300                 foreach( $wgHtmlEntities as $entity => $codepoint ) {
1301                         $out .= "<!ENTITY $entity \"&#$codepoint;\">";
1302                 }
1303                 $out .= "]>\n";
1304                 return $out;
1305         }
1306
1307         static function cleanUrl( $url, $hostname=true ) {
1308                 # Normalize any HTML entities in input. They will be
1309                 # re-escaped by makeExternalLink().
1310                 $url = Sanitizer::decodeCharReferences( $url );
1311
1312                 # Escape any control characters introduced by the above step
1313                 $url = preg_replace( '/[\][<>"\\x00-\\x20\\x7F]/e', "urlencode('\\0')", $url );
1314
1315                 # Validate hostname portion
1316                 $matches = array();
1317                 if( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
1318                         list( /* $whole */, $protocol, $host, $rest ) = $matches;
1319
1320                         // Characters that will be ignored in IDNs.
1321                         // http://tools.ietf.org/html/3454#section-3.1
1322                         // Strip them before further processing so blacklists and such work.
1323                         $strip = "/
1324                                 \\s|          # general whitespace
1325                                 \xc2\xad|     # 00ad SOFT HYPHEN
1326                                 \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
1327                                 \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
1328                                 \xe2\x81\xa0| # 2060 WORD JOINER
1329                                 \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
1330                                 \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
1331                                 \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
1332                                 \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
1333                                 \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
1334                                 \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
1335                                 \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
1336                                 [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe00f VARIATION SELECTOR-1-16
1337                                 /xuD";
1338
1339                         $host = preg_replace( $strip, '', $host );
1340
1341                         // @fixme: validate hostnames here
1342
1343                         return $protocol . $host . $rest;
1344                 } else {
1345                         return $url;
1346                 }
1347         }
1348
1349 }