includes/Sanitizer.php

   1 <?php
   2 /**
   3  * XHTML sanitizer for MediaWiki
   4  *
   5  * Copyright (C) 2002-2005 Brion Vibber <brion@pobox.com> et al
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @addtogroup Parser
  24  */
  25
  26 /**
  27  * Regular expression to match various types of character references in
  28  * Sanitizer::normalizeCharReferences and Sanitizer::decodeCharReferences
  29  */
  30 define( 'MW_CHAR_REFS_REGEX',
  31         '/&([A-Za-z0-9\x80-\xff]+);
  32          |&\#([0-9]+);
  33          |&\#x([0-9A-Za-z]+);
  34          |&\#X([0-9A-Za-z]+);
  35          |(&)/x' );
  36
  37 /**
  38  * Regular expression to match HTML/XML attribute pairs within a tag.
  39  * Allows some... latitude.
  40  * Used in Sanitizer::fixTagAttributes and Sanitizer::decodeTagAttributes
  41  */
  42 $attrib = '[A-Za-z0-9]';
  43 $space = '[\x09\x0a\x0d\x20]';
  44 define( 'MW_ATTRIBS_REGEX',
  45         "/(?:^|$space)($attrib+)
  46           ($space*=$space*
  47                 (?:
  48                  # The attribute value: quoted or alone
  49                   \"([^<\"]*)\"
  50                  | '([^<']*)'
  51                  |  ([a-zA-Z0-9!#$%&()*,\\-.\\/:;<>?@[\\]^_`{|}~]+)
  52                  |  (\#[0-9a-fA-F]+) # Technically wrong, but lots of
  53                                                          # colors are specified like this.
  54                                                          # We'll be normalizing it.
  55                 )
  56            )?(?=$space|\$)/sx" );
  57
  58 /**
  59  * List of all named character entities defined in HTML 4.01
  60  * http://www.w3.org/TR/html4/sgml/entities.html
  61  * @private
  62  */
  63 global $wgHtmlEntities;
  64 $wgHtmlEntities = array(
  65         'Aacute'   => 193,
  66         'aacute'   => 225,
  67         'Acirc'    => 194,
  68         'acirc'    => 226,
  69         'acute'    => 180,
  70         'AElig'    => 198,
  71         'aelig'    => 230,
  72         'Agrave'   => 192,
  73         'agrave'   => 224,
  74         'alefsym'  => 8501,
  75         'Alpha'    => 913,
  76         'alpha'    => 945,
  77         'amp'      => 38,
  78         'and'      => 8743,
  79         'ang'      => 8736,
  80         'Aring'    => 197,
  81         'aring'    => 229,
  82         'asymp'    => 8776,
  83         'Atilde'   => 195,
  84         'atilde'   => 227,
  85         'Auml'     => 196,
  86         'auml'     => 228,
  87         'bdquo'    => 8222,
  88         'Beta'     => 914,
  89         'beta'     => 946,
  90         'brvbar'   => 166,
  91         'bull'     => 8226,
  92         'cap'      => 8745,
  93         'Ccedil'   => 199,
  94         'ccedil'   => 231,
  95         'cedil'    => 184,
  96         'cent'     => 162,
  97         'Chi'      => 935,
  98         'chi'      => 967,
  99         'circ'     => 710,
 100         'clubs'    => 9827,
 101         'cong'     => 8773,
 102         'copy'     => 169,
 103         'crarr'    => 8629,
 104         'cup'      => 8746,
 105         'curren'   => 164,
 106         'dagger'   => 8224,
 107         'Dagger'   => 8225,
 108         'darr'     => 8595,
 109         'dArr'     => 8659,
 110         'deg'      => 176,
 111         'Delta'    => 916,
 112         'delta'    => 948,
 113         'diams'    => 9830,
 114         'divide'   => 247,
 115         'Eacute'   => 201,
 116         'eacute'   => 233,
 117         'Ecirc'    => 202,
 118         'ecirc'    => 234,
 119         'Egrave'   => 200,
 120         'egrave'   => 232,
 121         'empty'    => 8709,
 122         'emsp'     => 8195,
 123         'ensp'     => 8194,
 124         'Epsilon'  => 917,
 125         'epsilon'  => 949,
 126         'equiv'    => 8801,
 127         'Eta'      => 919,
 128         'eta'      => 951,
 129         'ETH'      => 208,
 130         'eth'      => 240,
 131         'Euml'     => 203,
 132         'euml'     => 235,
 133         'euro'     => 8364,
 134         'exist'    => 8707,
 135         'fnof'     => 402,
 136         'forall'   => 8704,
 137         'frac12'   => 189,
 138         'frac14'   => 188,
 139         'frac34'   => 190,
 140         'frasl'    => 8260,
 141         'Gamma'    => 915,
 142         'gamma'    => 947,
 143         'ge'       => 8805,
 144         'gt'       => 62,
 145         'harr'     => 8596,
 146         'hArr'     => 8660,
 147         'hearts'   => 9829,
 148         'hellip'   => 8230,
 149         'Iacute'   => 205,
 150         'iacute'   => 237,
 151         'Icirc'    => 206,
 152         'icirc'    => 238,
 153         'iexcl'    => 161,
 154         'Igrave'   => 204,
 155         'igrave'   => 236,
 156         'image'    => 8465,
 157         'infin'    => 8734,
 158         'int'      => 8747,
 159         'Iota'     => 921,
 160         'iota'     => 953,
 161         'iquest'   => 191,
 162         'isin'     => 8712,
 163         'Iuml'     => 207,
 164         'iuml'     => 239,
 165         'Kappa'    => 922,
 166         'kappa'    => 954,
 167         'Lambda'   => 923,
 168         'lambda'   => 955,
 169         'lang'     => 9001,
 170         'laquo'    => 171,
 171         'larr'     => 8592,
 172         'lArr'     => 8656,
 173         'lceil'    => 8968,
 174         'ldquo'    => 8220,
 175         'le'       => 8804,
 176         'lfloor'   => 8970,
 177         'lowast'   => 8727,
 178         'loz'      => 9674,
 179         'lrm'      => 8206,
 180         'lsaquo'   => 8249,
 181         'lsquo'    => 8216,
 182         'lt'       => 60,
 183         'macr'     => 175,
 184         'mdash'    => 8212,
 185         'micro'    => 181,
 186         'middot'   => 183,
 187         'minus'    => 8722,
 188         'Mu'       => 924,
 189         'mu'       => 956,
 190         'nabla'    => 8711,
 191         'nbsp'     => 160,
 192         'ndash'    => 8211,
 193         'ne'       => 8800,
 194         'ni'       => 8715,
 195         'not'      => 172,
 196         'notin'    => 8713,
 197         'nsub'     => 8836,
 198         'Ntilde'   => 209,
 199         'ntilde'   => 241,
 200         'Nu'       => 925,
 201         'nu'       => 957,
 202         'Oacute'   => 211,
 203         'oacute'   => 243,
 204         'Ocirc'    => 212,
 205         'ocirc'    => 244,
 206         'OElig'    => 338,
 207         'oelig'    => 339,
 208         'Ograve'   => 210,
 209         'ograve'   => 242,
 210         'oline'    => 8254,
 211         'Omega'    => 937,
 212         'omega'    => 969,
 213         'Omicron'  => 927,
 214         'omicron'  => 959,
 215         'oplus'    => 8853,
 216         'or'       => 8744,
 217         'ordf'     => 170,
 218         'ordm'     => 186,
 219         'Oslash'   => 216,
 220         'oslash'   => 248,
 221         'Otilde'   => 213,
 222         'otilde'   => 245,
 223         'otimes'   => 8855,
 224         'Ouml'     => 214,
 225         'ouml'     => 246,
 226         'para'     => 182,
 227         'part'     => 8706,
 228         'permil'   => 8240,
 229         'perp'     => 8869,
 230         'Phi'      => 934,
 231         'phi'      => 966,
 232         'Pi'       => 928,
 233         'pi'       => 960,
 234         'piv'      => 982,
 235         'plusmn'   => 177,
 236         'pound'    => 163,
 237         'prime'    => 8242,
 238         'Prime'    => 8243,
 239         'prod'     => 8719,
 240         'prop'     => 8733,
 241         'Psi'      => 936,
 242         'psi'      => 968,
 243         'quot'     => 34,
 244         'radic'    => 8730,
 245         'rang'     => 9002,
 246         'raquo'    => 187,
 247         'rarr'     => 8594,
 248         'rArr'     => 8658,
 249         'rceil'    => 8969,
 250         'rdquo'    => 8221,
 251         'real'     => 8476,
 252         'reg'      => 174,
 253         'rfloor'   => 8971,
 254         'Rho'      => 929,
 255         'rho'      => 961,
 256         'rlm'      => 8207,
 257         'rsaquo'   => 8250,
 258         'rsquo'    => 8217,
 259         'sbquo'    => 8218,
 260         'Scaron'   => 352,
 261         'scaron'   => 353,
 262         'sdot'     => 8901,
 263         'sect'     => 167,
 264         'shy'      => 173,
 265         'Sigma'    => 931,
 266         'sigma'    => 963,
 267         'sigmaf'   => 962,
 268         'sim'      => 8764,
 269         'spades'   => 9824,
 270         'sub'      => 8834,
 271         'sube'     => 8838,
 272         'sum'      => 8721,
 273         'sup'      => 8835,
 274         'sup1'     => 185,
 275         'sup2'     => 178,
 276         'sup3'     => 179,
 277         'supe'     => 8839,
 278         'szlig'    => 223,
 279         'Tau'      => 932,
 280         'tau'      => 964,
 281         'there4'   => 8756,
 282         'Theta'    => 920,
 283         'theta'    => 952,
 284         'thetasym' => 977,
 285         'thinsp'   => 8201,
 286         'THORN'    => 222,
 287         'thorn'    => 254,
 288         'tilde'    => 732,
 289         'times'    => 215,
 290         'trade'    => 8482,
 291         'Uacute'   => 218,
 292         'uacute'   => 250,
 293         'uarr'     => 8593,
 294         'uArr'     => 8657,
 295         'Ucirc'    => 219,
 296         'ucirc'    => 251,
 297         'Ugrave'   => 217,
 298         'ugrave'   => 249,
 299         'uml'      => 168,
 300         'upsih'    => 978,
 301         'Upsilon'  => 933,
 302         'upsilon'  => 965,
 303         'Uuml'     => 220,
 304         'uuml'     => 252,
 305         'weierp'   => 8472,
 306         'Xi'       => 926,
 307         'xi'       => 958,
 308         'Yacute'   => 221,
 309         'yacute'   => 253,
 310         'yen'      => 165,
 311         'Yuml'     => 376,
 312         'yuml'     => 255,
 313         'Zeta'     => 918,
 314         'zeta'     => 950,
 315         'zwj'      => 8205,
 316         'zwnj'     => 8204 );
 317
 318 /**
 319  * Character entity aliases accepted by MediaWiki
 320  */
 321 global $wgHtmlEntityAliases;
 322 $wgHtmlEntityAliases = array(
 323         'רלמ' => 'rlm',
 324         'رلم' => 'rlm',
 325 );
 326
 327
 328 /**
 329  * XHTML sanitizer for MediaWiki
 330  * @addtogroup Parser
 331  */
 332 class Sanitizer {
 333         /**
 334          * Cleans up HTML, removes dangerous tags and attributes, and
 335          * removes HTML comments
 336          * @private
 337          * @param string $text
 338          * @param callback $processCallback to do any variable or parameter replacements in HTML attribute values
 339          * @param array $args for the processing callback
 340          * @return string
 341          */
 342         static function removeHTMLtags( $text, $processCallback = null, $args = array() ) {
 343                 global $wgUseTidy;
 344
 345                 static $htmlpairs, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
 346                         $htmllist, $listtags, $htmlsingleallowed, $htmlelements, $staticInitialised;
 347
 348                 wfProfileIn( __METHOD__ );
 349
 350                 if ( !$staticInitialised ) {
 351
 352                         $htmlpairs = array( # Tags that must be closed
 353                                 'b', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
 354                                 'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
 355                                 'strike', 'strong', 'tt', 'var', 'div', 'center',
 356                                 'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
 357                                 'ruby', 'rt' , 'rb' , 'rp', 'p', 'span', 'u'
 358                         );
 359                         $htmlsingle = array(
 360                                 'br', 'hr', 'li', 'dt', 'dd'
 361                         );
 362                         $htmlsingleonly = array( # Elements that cannot have close tags
 363                                 'br', 'hr'
 364                         );
 365                         $htmlnest = array( # Tags that can be nested--??
 366                                 'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
 367                                 'dl', 'font', 'big', 'small', 'sub', 'sup', 'span'
 368                         );
 369                         $tabletags = array( # Can only appear inside table, we will close them
 370                                 'td', 'th', 'tr',
 371                         );
 372                         $htmllist = array( # Tags used by list
 373                                 'ul','ol',
 374                         );
 375                         $listtags = array( # Tags that can appear in a list
 376                                 'li',
 377                         );
 378
 379                         $htmlsingleallowed = array_merge( $htmlsingle, $tabletags );
 380                         $htmlelements = array_merge( $htmlsingle, $htmlpairs, $htmlnest );
 381
 382                         # Convert them all to hashtables for faster lookup
 383                         $vars = array( 'htmlpairs', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
 384                                 'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelements' );
 385                         foreach ( $vars as $var ) {
 386                                 $$var = array_flip( $$var );
 387                         }
 388                         $staticInitialised = true;
 389                 }
 390
 391                 # Remove HTML comments
 392                 $text = Sanitizer::removeHTMLcomments( $text );
 393                 $bits = explode( '<', $text );
 394                 $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 395                 if(!$wgUseTidy) {
 396                         $tagstack = $tablestack = array();
 397                         foreach ( $bits as $x ) {
 398                                 $regs = array();
 399                                 if( preg_match( '!^(/?)(\\w+)([^>]*?)(/{0,1}>)([^<]*)$!', $x, $regs ) ) {
 400                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 401                                 } else {
 402                                         $slash = $t = $params = $brace = $rest = null;
 403                                 }
 404
 405                                 $badtag = 0 ;
 406                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 407                                         # Check our stack
 408                                         if ( $slash ) {
 409                                                 # Closing a tag...
 410                                                 if( isset( $htmlsingleonly[$t] ) ) {
 411                                                         $badtag = 1;
 412                                                 } elseif ( ( $ot = @array_pop( $tagstack ) ) != $t ) {
 413                                                         if ( isset( $htmlsingleallowed[$ot] ) ) {
 414                                                                 # Pop all elements with an optional close tag
 415                                                                 # and see if we find a match below them
 416                                                                 $optstack = array();
 417                                                                 array_push ($optstack, $ot);
 418                                                                 while ( ( ( $ot = @array_pop( $tagstack ) ) != $t ) &&
 419                                                                                 isset( $htmlsingleallowed[$ot] ) )
 420                                                                 {
 421                                                                         array_push ($optstack, $ot);
 422                                                                 }
 423                                                                 if ( $t != $ot ) {
 424                                                                         # No match. Push the optinal elements back again
 425                                                                         $badtag = 1;
 426                                                                         while ( $ot = @array_pop( $optstack ) ) {
 427                                                                                 array_push( $tagstack, $ot );
 428                                                                         }
 429                                                                 }
 430                                                         } else {
 431                                                                 @array_push( $tagstack, $ot );
 432                                                                 # <li> can be nested in <ul> or <ol>, skip those cases:
 433                                                                 if(!(isset( $htmllist[$ot] ) && isset( $listtags[$t] ) )) {
 434                                                                         $badtag = 1;
 435                                                                 }
 436                                                         }
 437                                                 } else {
 438                                                         if ( $t == 'table' ) {
 439                                                                 $tagstack = array_pop( $tablestack );
 440                                                         }
 441                                                 }
 442                                                 $newparams = '';
 443                                         } else {
 444                                                 # Keep track for later
 445                                                 if ( isset( $tabletags[$t] ) &&
 446                                                 ! in_array( 'table', $tagstack ) ) {
 447                                                         $badtag = 1;
 448                                                 } else if ( in_array( $t, $tagstack ) &&
 449                                                 ! isset( $htmlnest [$t ] ) ) {
 450                                                         $badtag = 1 ;
 451                                                 # Is it a self closed htmlpair ? (bug 5487)
 452                                                 } else if( $brace == '/>' &&
 453                                                 isset( $htmlpairs[$t] ) ) {
 454                                                         $badtag = 1;
 455                                                 } elseif( isset( $htmlsingleonly[$t] ) ) {
 456                                                         # Hack to force empty tag for uncloseable elements
 457                                                         $brace = '/>';
 458                                                 } else if( isset( $htmlsingle[$t] ) ) {
 459                                                         # Hack to not close $htmlsingle tags
 460                                                         $brace = NULL;
 461                                                 } else if( isset( $tabletags[$t] )
 462                                                 &&  in_array($t ,$tagstack) ) {
 463                                                         // New table tag but forgot to close the previous one
 464                                                         $text .= "</$t>";
 465                                                 } else {
 466                                                         if ( $t == 'table' ) {
 467                                                                 array_push( $tablestack, $tagstack );
 468                                                                 $tagstack = array();
 469                                                         }
 470                                                         array_push( $tagstack, $t );
 471                                                 }
 472
 473                                                 # Replace any variables or template parameters with
 474                                                 # plaintext results.
 475                                                 if( is_callable( $processCallback ) ) {
 476                                                         call_user_func_array( $processCallback, array( &$params, $args ) );
 477                                                 }
 478
 479                                                 # Strip non-approved attributes from the tag
 480                                                 $newparams = Sanitizer::fixTagAttributes( $params, $t );
 481                                         }
 482                                         if ( ! $badtag ) {
 483                                                 $rest = str_replace( '>', '&gt;', $rest );
 484                                                 $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
 485                                                 $text .= "<$slash$t$newparams$close>$rest";
 486                                                 continue;
 487                                         }
 488                                 }
 489                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 490                         }
 491                         # Close off any remaining tags
 492                         while ( is_array( $tagstack ) && ($t = array_pop( $tagstack )) ) {
 493                                 $text .= "</$t>\n";
 494                                 if ( $t == 'table' ) { $tagstack = array_pop( $tablestack ); }
 495                         }
 496                 } else {
 497                         # this might be possible using tidy itself
 498                         foreach ( $bits as $x ) {
 499                                 preg_match( '/^(\\/?)(\\w+)([^>]*?)(\\/{0,1}>)([^<]*)$/',
 500                                 $x, $regs );
 501                                 @list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 502                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 503                                         if( is_callable( $processCallback ) ) {
 504                                                 call_user_func_array( $processCallback, array( &$params, $args ) );
 505                                         }
 506                                         $newparams = Sanitizer::fixTagAttributes( $params, $t );
 507                                         $rest = str_replace( '>', '&gt;', $rest );
 508                                         $text .= "<$slash$t$newparams$brace$rest";
 509                                 } else {
 510                                         $text .= '&lt;' . str_replace( '>', '&gt;', $x);
 511                                 }
 512                         }
 513                 }
 514                 wfProfileOut( __METHOD__ );
 515                 return $text;
 516         }
 517
 518         /**
 519          * Remove '<!--', '-->', and everything between.
 520          * To avoid leaving blank lines, when a comment is both preceded
 521          * and followed by a newline (ignoring spaces), trim leading and
 522          * trailing spaces and one of the newlines.
 523          *
 524          * @private
 525          * @param string $text
 526          * @return string
 527          */
 528         static function removeHTMLcomments( $text ) {
 529                 wfProfileIn( __METHOD__ );
 530                 while (($start = strpos($text, '<!--')) !== false) {
 531                         $end = strpos($text, '-->', $start + 4);
 532                         if ($end === false) {
 533                                 # Unterminated comment; bail out
 534                                 break;
 535                         }
 536
 537                         $end += 3;
 538
 539                         # Trim space and newline if the comment is both
 540                         # preceded and followed by a newline
 541                         $spaceStart = max($start - 1, 0);
 542                         $spaceLen = $end - $spaceStart;
 543                         while (substr($text, $spaceStart, 1) === ' ' && $spaceStart > 0) {
 544                                 $spaceStart--;
 545                                 $spaceLen++;
 546                         }
 547                         while (substr($text, $spaceStart + $spaceLen, 1) === ' ')
 548                                 $spaceLen++;
 549                         if (substr($text, $spaceStart, 1) === "\n" and substr($text, $spaceStart + $spaceLen, 1) === "\n") {
 550                                 # Remove the comment, leading and trailing
 551                                 # spaces, and leave only one newline.
 552                                 $text = substr_replace($text, "\n", $spaceStart, $spaceLen + 1);
 553                         }
 554                         else {
 555                                 # Remove just the comment.
 556                                 $text = substr_replace($text, '', $start, $end - $start);
 557                         }
 558                 }
 559                 wfProfileOut( __METHOD__ );
 560                 return $text;
 561         }
 562
 563         /**
 564          * Take an array of attribute names and values and normalize or discard
 565          * illegal values for the given element type.
 566          *
 567          * - Discards attributes not on a whitelist for the given element
 568          * - Unsafe style attributes are discarded
 569          * - Invalid id attributes are reencoded
 570          *
 571          * @param array $attribs
 572          * @param string $element
 573          * @return array
 574          *
 575          * @todo Check for legal values where the DTD limits things.
 576          * @todo Check for unique id attribute :P
 577          */
 578         static function validateTagAttributes( $attribs, $element ) {
 579                 return Sanitizer::validateAttributes( $attribs,
 580                         Sanitizer::attributeWhitelist( $element ) );
 581         }
 582
 583         /**
 584          * Take an array of attribute names and values and normalize or discard
 585          * illegal values for the given whitelist.
 586          *
 587          * - Discards attributes not the given whitelist
 588          * - Unsafe style attributes are discarded
 589          * - Invalid id attributes are reencoded
 590          *
 591          * @param array $attribs
 592          * @param array $whitelist list of allowed attribute names
 593          * @return array
 594          *
 595          * @todo Check for legal values where the DTD limits things.
 596          * @todo Check for unique id attribute :P
 597          */
 598         static function validateAttributes( $attribs, $whitelist ) {
 599                 $whitelist = array_flip( $whitelist );
 600                 $out = array();
 601                 foreach( $attribs as $attribute => $value ) {
 602                         if( !isset( $whitelist[$attribute] ) ) {
 603                                 continue;
 604                         }
 605                         # Strip javascript "expression" from stylesheets.
 606                         # http://msdn.microsoft.com/workshop/author/dhtml/overview/recalc.asp
 607                         if( $attribute == 'style' ) {
 608                                 $value = Sanitizer::checkCss( $value );
 609                                 if( $value === false ) {
 610                                         # haxx0r
 611                                         continue;
 612                                 }
 613                         }
 614
 615                         if ( $attribute === 'id' )
 616                                 $value = Sanitizer::escapeId( $value );
 617
 618                         // If this attribute was previously set, override it.
 619                         // Output should only have one attribute of each name.
 620                         $out[$attribute] = $value;
 621                 }
 622                 return $out;
 623         }
 624
 625         /**
 626          * Merge two sets of HTML attributes.
 627          * Conflicting items in the second set will override those
 628          * in the first, except for 'class' attributes which will be
 629          * combined.
 630          *
 631          * @todo implement merging for other attributes such as style
 632          * @param array $a
 633          * @param array $b
 634          * @return array
 635          */
 636         static function mergeAttributes( $a, $b ) {
 637                 $out = array_merge( $a, $b );
 638                 if( isset( $a['class'] )
 639                         && isset( $b['class'] )
 640                         && $a['class'] !== $b['class'] ) {
 641
 642                         $out['class'] = implode( ' ',
 643                                 array_unique(
 644                                         preg_split( '/\s+/',
 645                                                 $a['class'] . ' ' . $b['class'],
 646                                                 -1,
 647                                                 PREG_SPLIT_NO_EMPTY ) ) );
 648                 }
 649                 return $out;
 650         }
 651
 652         /**
 653          * Pick apart some CSS and check it for forbidden or unsafe structures.
 654          * Returns a sanitized string, or false if it was just too evil.
 655          *
 656          * Currently URL references, 'expression', 'tps' are forbidden.
 657          *
 658          * @param string $value
 659          * @return mixed
 660          */
 661         static function checkCss( $value ) {
 662                 $stripped = Sanitizer::decodeCharReferences( $value );
 663
 664                 // Remove any comments; IE gets token splitting wrong
 665                 $stripped = StringUtils::delimiterReplace( '/*', '*/', ' ', $stripped );
 666
 667                 $value = $stripped;
 668
 669                 // ... and continue checks
 670                 $stripped = preg_replace( '!\\\\([0-9A-Fa-f]{1,6})[ \\n\\r\\t\\f]?!e',
 671                         'codepointToUtf8(hexdec("$1"))', $stripped );
 672                 $stripped = str_replace( '\\', '', $stripped );
 673                 if( preg_match( '/(?:expression|tps*:\/\/|url\\s*\().*/is',
 674                                 $stripped ) ) {
 675                         # haxx0r
 676                         return false;
 677                 }
 678
 679                 return $value;
 680         }
 681
 682         /**
 683          * Take a tag soup fragment listing an HTML element's attributes
 684          * and normalize it to well-formed XML, discarding unwanted attributes.
 685          * Output is safe for further wikitext processing, with escaping of
 686          * values that could trigger problems.
 687          *
 688          * - Normalizes attribute names to lowercase
 689          * - Discards attributes not on a whitelist for the given element
 690          * - Turns broken or invalid entities into plaintext
 691          * - Double-quotes all attribute values
 692          * - Attributes without values are given the name as attribute
 693          * - Double attributes are discarded
 694          * - Unsafe style attributes are discarded
 695          * - Prepends space if there are attributes.
 696          *
 697          * @param string $text
 698          * @param string $element
 699          * @return string
 700          */
 701         static function fixTagAttributes( $text, $element ) {
 702                 if( trim( $text ) == '' ) {
 703                         return '';
 704                 }
 705
 706                 $stripped = Sanitizer::validateTagAttributes(
 707                         Sanitizer::decodeTagAttributes( $text ), $element );
 708
 709                 $attribs = array();
 710                 foreach( $stripped as $attribute => $value ) {
 711                         $encAttribute = htmlspecialchars( $attribute );
 712                         $encValue = Sanitizer::safeEncodeAttribute( $value );
 713
 714                         $attribs[] = "$encAttribute=\"$encValue\"";
 715                 }
 716                 return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
 717         }
 718
 719         /**
 720          * Encode an attribute value for HTML output.
 721          * @param $text
 722          * @return HTML-encoded text fragment
 723          */
 724         static function encodeAttribute( $text ) {
 725                 $encValue = htmlspecialchars( $text );
 726
 727                 // Whitespace is normalized during attribute decoding,
 728                 // so if we've been passed non-spaces we must encode them
 729                 // ahead of time or they won't be preserved.
 730                 $encValue = strtr( $encValue, array(
 731                         "\n" => '&#10;',
 732                         "\r" => '&#13;',
 733                         "\t" => '&#9;',
 734                 ) );
 735
 736                 return $encValue;
 737         }
 738
 739         /**
 740          * Encode an attribute value for HTML tags, with extra armoring
 741          * against further wiki processing.
 742          * @param $text
 743          * @return HTML-encoded text fragment
 744          */
 745         static function safeEncodeAttribute( $text ) {
 746                 $encValue = Sanitizer::encodeAttribute( $text );
 747
 748                 # Templates and links may be expanded in later parsing,
 749                 # creating invalid or dangerous output. Suppress this.
 750                 $encValue = strtr( $encValue, array(
 751                         '<'    => '&lt;',   // This should never happen,
 752                         '>'    => '&gt;',   // we've received invalid input
 753                         '"'    => '&quot;', // which should have been escaped.
 754                         '{'    => '&#123;',
 755                         '['    => '&#91;',
 756                         "''"   => '&#39;&#39;',
 757                         'ISBN' => '&#73;SBN',
 758                         'RFC'  => '&#82;FC',
 759                         'PMID' => '&#80;MID',
 760                         '|'    => '&#124;',
 761                         '__'   => '&#95;_',
 762                 ) );
 763
 764                 # Stupid hack
 765                 $encValue = preg_replace_callback(
 766                         '/(' . wfUrlProtocols() . ')/',
 767                         array( 'Sanitizer', 'armorLinksCallback' ),
 768                         $encValue );
 769                 return $encValue;
 770         }
 771
 772         /**
 773          * Given a value escape it so that it can be used in an id attribute and
 774          * return it, this does not validate the value however (see first link)
 775          *
 776          * @see http://www.w3.org/TR/html401/types.html#type-name Valid characters
 777          *                                                          in the id and
 778          *                                                          name attributes
 779          * @see http://www.w3.org/TR/html401/struct/links.html#h-12.2.3 Anchors with the id attribute
 780          *
 781          * @static
 782          *
 783          * @param string $id
 784          * @return string
 785          */
 786         static function escapeId( $id ) {
 787                 static $replace = array(
 788                         '%3A' => ':',
 789                         '%' => '.'
 790                 );
 791
 792                 $id = urlencode( Sanitizer::decodeCharReferences( strtr( $id, ' ', '_' ) ) );
 793
 794                 return str_replace( array_keys( $replace ), array_values( $replace ), $id );
 795         }
 796
 797         /**
 798          * Given a value, escape it so that it can be used as a CSS class and
 799          * return it.
 800          *
 801          * @todo For extra validity, input should be validated UTF-8.
 802          *
 803          * @see http://www.w3.org/TR/CSS21/syndata.html Valid characters/format
 804          *
 805          * @param string $class
 806          * @return string
 807          */
 808         static function escapeClass( $class ) {
 809                 // Convert ugly stuff to underscores and kill underscores in ugly places
 810                 return rtrim(preg_replace(
 811                         array('/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/','/_+/'),
 812                         '_',
 813                         $class ), '_');
 814         }
 815
 816         /**
 817          * Regex replace callback for armoring links against further processing.
 818          * @param array $matches
 819          * @return string
 820          * @private
 821          */
 822         private static function armorLinksCallback( $matches ) {
 823                 return str_replace( ':', '&#58;', $matches[1] );
 824         }
 825
 826         /**
 827          * Return an associative array of attribute names and values from
 828          * a partial tag string. Attribute names are forces to lowercase,
 829          * character references are decoded to UTF-8 text.
 830          *
 831          * @param string
 832          * @return array
 833          */
 834         static function decodeTagAttributes( $text ) {
 835                 $attribs = array();
 836
 837                 if( trim( $text ) == '' ) {
 838                         return $attribs;
 839                 }
 840
 841                 $pairs = array();
 842                 if( !preg_match_all(
 843                         MW_ATTRIBS_REGEX,
 844                         $text,
 845                         $pairs,
 846                         PREG_SET_ORDER ) ) {
 847                         return $attribs;
 848                 }
 849
 850                 foreach( $pairs as $set ) {
 851                         $attribute = strtolower( $set[1] );
 852                         $value = Sanitizer::getTagAttributeCallback( $set );
 853
 854                         // Normalize whitespace
 855                         $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
 856                         $value = trim( $value );
 857
 858                         // Decode character references
 859                         $attribs[$attribute] = Sanitizer::decodeCharReferences( $value );
 860                 }
 861                 return $attribs;
 862         }
 863
 864         /**
 865          * Pick the appropriate attribute value from a match set from the
 866          * MW_ATTRIBS_REGEX matches.
 867          *
 868          * @param array $set
 869          * @return string
 870          * @private
 871          */
 872         private static function getTagAttributeCallback( $set ) {
 873                 if( isset( $set[6] ) ) {
 874                         # Illegal #XXXXXX color with no quotes.
 875                         return $set[6];
 876                 } elseif( isset( $set[5] ) ) {
 877                         # No quotes.
 878                         return $set[5];
 879                 } elseif( isset( $set[4] ) ) {
 880                         # Single-quoted
 881                         return $set[4];
 882                 } elseif( isset( $set[3] ) ) {
 883                         # Double-quoted
 884                         return $set[3];
 885                 } elseif( !isset( $set[2] ) ) {
 886                         # In XHTML, attributes must have a value.
 887                         # For 'reduced' form, return explicitly the attribute name here.
 888                         return $set[1];
 889                 } else {
 890                         throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
 891                 }
 892         }
 893
 894         /**
 895          * Normalize whitespace and character references in an XML source-
 896          * encoded text for an attribute value.
 897          *
 898          * See http://www.w3.org/TR/REC-xml/#AVNormalize for background,
 899          * but note that we're not returning the value, but are returning
 900          * XML source fragments that will be slapped into output.
 901          *
 902          * @param string $text
 903          * @return string
 904          * @private
 905          */
 906         private static function normalizeAttributeValue( $text ) {
 907                 return str_replace( '"', '&quot;',
 908                         self::normalizeWhitespace(
 909                                 Sanitizer::normalizeCharReferences( $text ) ) );
 910         }
 911
 912         private static function normalizeWhitespace( $text ) {
 913                 return preg_replace(
 914                         '/\r\n|[\x20\x0d\x0a\x09]/',
 915                         ' ',
 916                         $text );
 917         }
 918
 919         /**
 920          * Ensure that any entities and character references are legal
 921          * for XML and XHTML specifically. Any stray bits will be
 922          * &amp;-escaped to result in a valid text fragment.
 923          *
 924          * a. any named char refs must be known in XHTML
 925          * b. any numeric char refs must be legal chars, not invalid or forbidden
 926          * c. use &#x, not &#X
 927          * d. fix or reject non-valid attributes
 928          *
 929          * @param string $text
 930          * @return string
 931          * @private
 932          */
 933         static function normalizeCharReferences( $text ) {
 934                 return preg_replace_callback(
 935                         MW_CHAR_REFS_REGEX,
 936                         array( 'Sanitizer', 'normalizeCharReferencesCallback' ),
 937                         $text );
 938         }
 939         /**
 940          * @param string $matches
 941          * @return string
 942          */
 943         static function normalizeCharReferencesCallback( $matches ) {
 944                 $ret = null;
 945                 if( $matches[1] != '' ) {
 946                         $ret = Sanitizer::normalizeEntity( $matches[1] );
 947                 } elseif( $matches[2] != '' ) {
 948                         $ret = Sanitizer::decCharReference( $matches[2] );
 949                 } elseif( $matches[3] != ''  ) {
 950                         $ret = Sanitizer::hexCharReference( $matches[3] );
 951                 } elseif( $matches[4] != '' ) {
 952                         $ret = Sanitizer::hexCharReference( $matches[4] );
 953                 }
 954                 if( is_null( $ret ) ) {
 955                         return htmlspecialchars( $matches[0] );
 956                 } else {
 957                         return $ret;
 958                 }
 959         }
 960
 961         /**
 962          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
 963          * return the named entity reference as is. If the entity is a
 964          * MediaWiki-specific alias, returns the HTML equivalent. Otherwise,
 965          * returns HTML-escaped text of pseudo-entity source (eg &amp;foo;)
 966          *
 967          * @param string $name
 968          * @return string
 969          * @static
 970          */
 971         static function normalizeEntity( $name ) {
 972                 global $wgHtmlEntities, $wgHtmlEntityAliases;
 973                 if ( isset( $wgHtmlEntityAliases[$name] ) ) {
 974                         return "&{$wgHtmlEntityAliases[$name]};";
 975                 } elseif( isset( $wgHtmlEntities[$name] ) ) {
 976                         return "&$name;";
 977                 } else {
 978                         return "&amp;$name;";
 979                 }
 980         }
 981
 982         static function decCharReference( $codepoint ) {
 983                 $point = intval( $codepoint );
 984                 if( Sanitizer::validateCodepoint( $point ) ) {
 985                         return sprintf( '&#%d;', $point );
 986                 } else {
 987                         return null;
 988                 }
 989         }
 990
 991         static function hexCharReference( $codepoint ) {
 992                 $point = hexdec( $codepoint );
 993                 if( Sanitizer::validateCodepoint( $point ) ) {
 994                         return sprintf( '&#x%x;', $point );
 995                 } else {
 996                         return null;
 997                 }
 998         }
 999
1000         /**
1001          * Returns true if a given Unicode codepoint is a valid character in XML.
1002          * @param int $codepoint
1003          * @return bool
1004          */
1005         private static function validateCodepoint( $codepoint ) {
1006                 return ($codepoint ==    0x09)
1007                         || ($codepoint ==    0x0a)
1008                         || ($codepoint ==    0x0d)
1009                         || ($codepoint >=    0x20 && $codepoint <=   0xd7ff)
1010                         || ($codepoint >=  0xe000 && $codepoint <=   0xfffd)
1011                         || ($codepoint >= 0x10000 && $codepoint <= 0x10ffff);
1012         }
1013
1014         /**
1015          * Decode any character references, numeric or named entities,
1016          * in the text and return a UTF-8 string.
1017          *
1018          * @param string $text
1019          * @return string
1020          * @public
1021          * @static
1022          */
1023         public static function decodeCharReferences( $text ) {
1024                 return preg_replace_callback(
1025                         MW_CHAR_REFS_REGEX,
1026                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1027                         $text );
1028         }
1029
1030         /**
1031          * @param string $matches
1032          * @return string
1033          */
1034         static function decodeCharReferencesCallback( $matches ) {
1035                 if( $matches[1] != '' ) {
1036                         return Sanitizer::decodeEntity( $matches[1] );
1037                 } elseif( $matches[2] != '' ) {
1038                         return  Sanitizer::decodeChar( intval( $matches[2] ) );
1039                 } elseif( $matches[3] != ''  ) {
1040                         return  Sanitizer::decodeChar( hexdec( $matches[3] ) );
1041                 } elseif( $matches[4] != '' ) {
1042                         return  Sanitizer::decodeChar( hexdec( $matches[4] ) );
1043                 }
1044                 # Last case should be an ampersand by itself
1045                 return $matches[0];
1046         }
1047
1048         /**
1049          * Return UTF-8 string for a codepoint if that is a valid
1050          * character reference, otherwise U+FFFD REPLACEMENT CHARACTER.
1051          * @param int $codepoint
1052          * @return string
1053          * @private
1054          */
1055         static function decodeChar( $codepoint ) {
1056                 if( Sanitizer::validateCodepoint( $codepoint ) ) {
1057                         return codepointToUtf8( $codepoint );
1058                 } else {
1059                         return UTF8_REPLACEMENT;
1060                 }
1061         }
1062
1063         /**
1064          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1065          * return the UTF-8 encoding of that character. Otherwise, returns
1066          * pseudo-entity source (eg &foo;)
1067          *
1068          * @param string $name
1069          * @return string
1070          */
1071         static function decodeEntity( $name ) {
1072                 global $wgHtmlEntities, $wgHtmlEntityAliases;
1073                 if ( isset( $wgHtmlEntityAliases[$name] ) ) {
1074                         $name = $wgHtmlEntityAliases[$name];
1075                 }
1076                 if( isset( $wgHtmlEntities[$name] ) ) {
1077                         return codepointToUtf8( $wgHtmlEntities[$name] );
1078                 } else {
1079                         return "&$name;";
1080                 }
1081         }
1082
1083         /**
1084          * Fetch the whitelist of acceptable attributes for a given
1085          * element name.
1086          *
1087          * @param string $element
1088          * @return array
1089          */
1090         static function attributeWhitelist( $element ) {
1091                 static $list;
1092                 if( !isset( $list ) ) {
1093                         $list = Sanitizer::setupAttributeWhitelist();
1094                 }
1095                 return isset( $list[$element] )
1096                         ? $list[$element]
1097                         : array();
1098         }
1099
1100         /**
1101          * @todo Document it a bit
1102          * @return array
1103          */
1104         static function setupAttributeWhitelist() {
1105                 $common = array( 'id', 'class', 'lang', 'dir', 'title', 'style' );
1106                 $block = array_merge( $common, array( 'align' ) );
1107                 $tablealign = array( 'align', 'char', 'charoff', 'valign' );
1108                 $tablecell = array( 'abbr',
1109                                     'axis',
1110                                     'headers',
1111                                     'scope',
1112                                     'rowspan',
1113                                     'colspan',
1114                                     'nowrap', # deprecated
1115                                     'width',  # deprecated
1116                                     'height', # deprecated
1117                                     'bgcolor' # deprecated
1118                                     );
1119
1120                 # Numbers refer to sections in HTML 4.01 standard describing the element.
1121                 # See: http://www.w3.org/TR/html4/
1122                 $whitelist = array (
1123                         # 7.5.4
1124                         'div'        => $block,
1125                         'center'     => $common, # deprecated
1126                         'span'       => $block, # ??
1127
1128                         # 7.5.5
1129                         'h1'         => $block,
1130                         'h2'         => $block,
1131                         'h3'         => $block,
1132                         'h4'         => $block,
1133                         'h5'         => $block,
1134                         'h6'         => $block,
1135
1136                         # 7.5.6
1137                         # address
1138
1139                         # 8.2.4
1140                         # bdo
1141
1142                         # 9.2.1
1143                         'em'         => $common,
1144                         'strong'     => $common,
1145                         'cite'       => $common,
1146                         # dfn
1147                         'code'       => $common,
1148                         # samp
1149                         # kbd
1150                         'var'        => $common,
1151                         # abbr
1152                         # acronym
1153
1154                         # 9.2.2
1155                         'blockquote' => array_merge( $common, array( 'cite' ) ),
1156                         # q
1157
1158                         # 9.2.3
1159                         'sub'        => $common,
1160                         'sup'        => $common,
1161
1162                         # 9.3.1
1163                         'p'          => $block,
1164
1165                         # 9.3.2
1166                         'br'         => array( 'id', 'class', 'title', 'style', 'clear' ),
1167
1168                         # 9.3.4
1169                         'pre'        => array_merge( $common, array( 'width' ) ),
1170
1171                         # 9.4
1172                         'ins'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1173                         'del'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1174
1175                         # 10.2
1176                         'ul'         => array_merge( $common, array( 'type' ) ),
1177                         'ol'         => array_merge( $common, array( 'type', 'start' ) ),
1178                         'li'         => array_merge( $common, array( 'type', 'value' ) ),
1179
1180                         # 10.3
1181                         'dl'         => $common,
1182                         'dd'         => $common,
1183                         'dt'         => $common,
1184
1185                         # 11.2.1
1186                         'table'      => array_merge( $common,
1187                                                                 array( 'summary', 'width', 'border', 'frame',
1188                                                                                 'rules', 'cellspacing', 'cellpadding',
1189                                                                                 'align', 'bgcolor',
1190                                                                 ) ),
1191
1192                         # 11.2.2
1193                         'caption'    => array_merge( $common, array( 'align' ) ),
1194
1195                         # 11.2.3
1196                         'thead'      => array_merge( $common, $tablealign ),
1197                         'tfoot'      => array_merge( $common, $tablealign ),
1198                         'tbody'      => array_merge( $common, $tablealign ),
1199
1200                         # 11.2.4
1201                         'colgroup'   => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1202                         'col'        => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1203
1204                         # 11.2.5
1205                         'tr'         => array_merge( $common, array( 'bgcolor' ), $tablealign ),
1206
1207                         # 11.2.6
1208                         'td'         => array_merge( $common, $tablecell, $tablealign ),
1209                         'th'         => array_merge( $common, $tablecell, $tablealign ),
1210
1211                         # 13.2
1212                         # Not usually allowed, but may be used for extension-style hooks
1213                         # such as <math> when it is rasterized
1214                         'img'        => array_merge( $common, array( 'alt' ) ),
1215
1216                         # 15.2.1
1217                         'tt'         => $common,
1218                         'b'          => $common,
1219                         'i'          => $common,
1220                         'big'        => $common,
1221                         'small'      => $common,
1222                         'strike'     => $common,
1223                         's'          => $common,
1224                         'u'          => $common,
1225
1226                         # 15.2.2
1227                         'font'       => array_merge( $common, array( 'size', 'color', 'face' ) ),
1228                         # basefont
1229
1230                         # 15.3
1231                         'hr'         => array_merge( $common, array( 'noshade', 'size', 'width' ) ),
1232
1233                         # XHTML Ruby annotation text module, simple ruby only.
1234                         # http://www.w3c.org/TR/ruby/
1235                         'ruby'       => $common,
1236                         # rbc
1237                         # rtc
1238                         'rb'         => $common,
1239                         'rt'         => $common, #array_merge( $common, array( 'rbspan' ) ),
1240                         'rp'         => $common,
1241
1242                         # MathML root element, where used for extensions
1243                         # 'title' may not be 100% valid here; it's XHTML
1244                         # http://www.w3.org/TR/REC-MathML/
1245                         'math'       => array( 'class', 'style', 'id', 'title' ),
1246                         );
1247                 return $whitelist;
1248         }
1249
1250         /**
1251          * Take a fragment of (potentially invalid) HTML and return
1252          * a version with any tags removed, encoded as plain text.
1253          *
1254          * Warning: this return value must be further escaped for literal
1255          * inclusion in HTML output as of 1.10!
1256          *
1257          * @param string $text HTML fragment
1258          * @return string
1259          */
1260         static function stripAllTags( $text ) {
1261                 # Actual <tags>
1262                 $text = StringUtils::delimiterReplace( '<', '>', '', $text );
1263
1264                 # Normalize &entities and whitespace
1265                 $text = self::decodeCharReferences( $text );
1266                 $text = self::normalizeWhitespace( $text );
1267
1268                 return $text;
1269         }
1270
1271         /**
1272          * Hack up a private DOCTYPE with HTML's standard entity declarations.
1273          * PHP 4 seemed to know these if you gave it an HTML doctype, but
1274          * PHP 5.1 doesn't.
1275          *
1276          * Use for passing XHTML fragments to PHP's XML parsing functions
1277          *
1278          * @return string
1279          * @static
1280          */
1281         static function hackDocType() {
1282                 global $wgHtmlEntities;
1283                 $out = "<!DOCTYPE html [\n";
1284                 foreach( $wgHtmlEntities as $entity => $codepoint ) {
1285                         $out .= "<!ENTITY $entity \"&#$codepoint;\">";
1286                 }
1287                 $out .= "]>\n";
1288                 return $out;
1289         }
1290
1291         static function cleanUrl( $url, $hostname=true ) {
1292                 # Normalize any HTML entities in input. They will be
1293                 # re-escaped by makeExternalLink().
1294                 $url = Sanitizer::decodeCharReferences( $url );
1295
1296                 # Escape any control characters introduced by the above step
1297                 $url = preg_replace( '/[\][<>"\\x00-\\x20\\x7F]/e', "urlencode('\\0')", $url );
1298
1299                 # Validate hostname portion
1300                 $matches = array();
1301                 if( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
1302                         list( /* $whole */, $protocol, $host, $rest ) = $matches;
1303
1304                         // Characters that will be ignored in IDNs.
1305                         // http://tools.ietf.org/html/3454#section-3.1
1306                         // Strip them before further processing so blacklists and such work.
1307                         $strip = "/
1308                                 \\s|          # general whitespace
1309                                 \xc2\xad|     # 00ad SOFT HYPHEN
1310                                 \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
1311                                 \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
1312                                 \xe2\x81\xa0| # 2060 WORD JOINER
1313                                 \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
1314                                 \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
1315                                 \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
1316                                 \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
1317                                 \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
1318                                 \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
1319                                 \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
1320                                 [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe00f VARIATION SELECTOR-1-16
1321                                 /xuD";
1322
1323                         $host = preg_replace( $strip, '', $host );
1324
1325                         // @fixme: validate hostnames here
1326
1327                         return $protocol . $host . $rest;
1328                 } else {
1329                         return $url;
1330                 }
1331         }
1332
1333 }
1334
1335