includes/parser/Sanitizer.php

   1 <?php
   2 /**
   3  * HTML sanitizer for %MediaWiki.
   4  *
   5  * Copyright © 2002-2005 Brion Vibber <brion@pobox.com> et al
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Parser
  25  */
  26
  27 use MediaWiki\MediaWikiServices;
  28
  29 /**
  30  * HTML sanitizer for MediaWiki
  31  * @ingroup Parser
  32  */
  33 class Sanitizer {
  34         /**
  35          * Regular expression to match various types of character references in
  36          * Sanitizer::normalizeCharReferences and Sanitizer::decodeCharReferences
  37          */
  38         const CHAR_REFS_REGEX =
  39                 '/&([A-Za-z0-9\x80-\xff]+);
  40                  |&\#([0-9]+);
  41                  |&\#[xX]([0-9A-Fa-f]+);
  42                  |(&)/x';
  43
  44         /**
  45          * Acceptable tag name charset from HTML5 parsing spec
  46          * https://www.w3.org/TR/html5/syntax.html#tag-open-state
  47          */
  48         const ELEMENT_BITS_REGEX = '!^(/?)([A-Za-z][^\t\n\v />\0]*+)([^>]*?)(/?>)([^<]*)$!';
  49
  50         /**
  51          * Blacklist for evil uris like javascript:
  52          * WARNING: DO NOT use this in any place that actually requires blacklisting
  53          * for security reasons. There are NUMEROUS[1] ways to bypass blacklisting, the
  54          * only way to be secure from javascript: uri based xss vectors is to whitelist
  55          * things that you know are safe and deny everything else.
  56          * [1]: http://ha.ckers.org/xss.html
  57          */
  58         const EVIL_URI_PATTERN = '!(^|\s|\*/\s*)(javascript|vbscript)([^\w]|$)!i';
  59         const XMLNS_ATTRIBUTE_PATTERN = "/^xmlns:[:A-Z_a-z-.0-9]+$/";
  60
  61         /**
  62          * Tells escapeUrlForHtml() to encode the ID using the wiki's primary encoding.
  63          *
  64          * @since 1.30
  65          */
  66         const ID_PRIMARY = 0;
  67
  68         /**
  69          * Tells escapeUrlForHtml() to encode the ID using the fallback encoding, or return false
  70          * if no fallback is configured.
  71          *
  72          * @since 1.30
  73          */
  74         const ID_FALLBACK = 1;
  75
  76         /**
  77          * List of all named character entities defined in HTML 4.01
  78          * https://www.w3.org/TR/html4/sgml/entities.html
  79          * As well as &apos; which is only defined starting in XHTML1.
  80          */
  81         private static $htmlEntities = [
  82                 'Aacute'   => 193,
  83                 'aacute'   => 225,
  84                 'Acirc'    => 194,
  85                 'acirc'    => 226,
  86                 'acute'    => 180,
  87                 'AElig'    => 198,
  88                 'aelig'    => 230,
  89                 'Agrave'   => 192,
  90                 'agrave'   => 224,
  91                 'alefsym'  => 8501,
  92                 'Alpha'    => 913,
  93                 'alpha'    => 945,
  94                 'amp'      => 38,
  95                 'and'      => 8743,
  96                 'ang'      => 8736,
  97                 'apos'     => 39, // New in XHTML & HTML 5; avoid in output for compatibility with IE.
  98                 'Aring'    => 197,
  99                 'aring'    => 229,
 100                 'asymp'    => 8776,
 101                 'Atilde'   => 195,
 102                 'atilde'   => 227,
 103                 'Auml'     => 196,
 104                 'auml'     => 228,
 105                 'bdquo'    => 8222,
 106                 'Beta'     => 914,
 107                 'beta'     => 946,
 108                 'brvbar'   => 166,
 109                 'bull'     => 8226,
 110                 'cap'      => 8745,
 111                 'Ccedil'   => 199,
 112                 'ccedil'   => 231,
 113                 'cedil'    => 184,
 114                 'cent'     => 162,
 115                 'Chi'      => 935,
 116                 'chi'      => 967,
 117                 'circ'     => 710,
 118                 'clubs'    => 9827,
 119                 'cong'     => 8773,
 120                 'copy'     => 169,
 121                 'crarr'    => 8629,
 122                 'cup'      => 8746,
 123                 'curren'   => 164,
 124                 'dagger'   => 8224,
 125                 'Dagger'   => 8225,
 126                 'darr'     => 8595,
 127                 'dArr'     => 8659,
 128                 'deg'      => 176,
 129                 'Delta'    => 916,
 130                 'delta'    => 948,
 131                 'diams'    => 9830,
 132                 'divide'   => 247,
 133                 'Eacute'   => 201,
 134                 'eacute'   => 233,
 135                 'Ecirc'    => 202,
 136                 'ecirc'    => 234,
 137                 'Egrave'   => 200,
 138                 'egrave'   => 232,
 139                 'empty'    => 8709,
 140                 'emsp'     => 8195,
 141                 'ensp'     => 8194,
 142                 'Epsilon'  => 917,
 143                 'epsilon'  => 949,
 144                 'equiv'    => 8801,
 145                 'Eta'      => 919,
 146                 'eta'      => 951,
 147                 'ETH'      => 208,
 148                 'eth'      => 240,
 149                 'Euml'     => 203,
 150                 'euml'     => 235,
 151                 'euro'     => 8364,
 152                 'exist'    => 8707,
 153                 'fnof'     => 402,
 154                 'forall'   => 8704,
 155                 'frac12'   => 189,
 156                 'frac14'   => 188,
 157                 'frac34'   => 190,
 158                 'frasl'    => 8260,
 159                 'Gamma'    => 915,
 160                 'gamma'    => 947,
 161                 'ge'       => 8805,
 162                 'gt'       => 62,
 163                 'harr'     => 8596,
 164                 'hArr'     => 8660,
 165                 'hearts'   => 9829,
 166                 'hellip'   => 8230,
 167                 'Iacute'   => 205,
 168                 'iacute'   => 237,
 169                 'Icirc'    => 206,
 170                 'icirc'    => 238,
 171                 'iexcl'    => 161,
 172                 'Igrave'   => 204,
 173                 'igrave'   => 236,
 174                 'image'    => 8465,
 175                 'infin'    => 8734,
 176                 'int'      => 8747,
 177                 'Iota'     => 921,
 178                 'iota'     => 953,
 179                 'iquest'   => 191,
 180                 'isin'     => 8712,
 181                 'Iuml'     => 207,
 182                 'iuml'     => 239,
 183                 'Kappa'    => 922,
 184                 'kappa'    => 954,
 185                 'Lambda'   => 923,
 186                 'lambda'   => 955,
 187                 'lang'     => 9001,
 188                 'laquo'    => 171,
 189                 'larr'     => 8592,
 190                 'lArr'     => 8656,
 191                 'lceil'    => 8968,
 192                 'ldquo'    => 8220,
 193                 'le'       => 8804,
 194                 'lfloor'   => 8970,
 195                 'lowast'   => 8727,
 196                 'loz'      => 9674,
 197                 'lrm'      => 8206,
 198                 'lsaquo'   => 8249,
 199                 'lsquo'    => 8216,
 200                 'lt'       => 60,
 201                 'macr'     => 175,
 202                 'mdash'    => 8212,
 203                 'micro'    => 181,
 204                 'middot'   => 183,
 205                 'minus'    => 8722,
 206                 'Mu'       => 924,
 207                 'mu'       => 956,
 208                 'nabla'    => 8711,
 209                 'nbsp'     => 160,
 210                 'ndash'    => 8211,
 211                 'ne'       => 8800,
 212                 'ni'       => 8715,
 213                 'not'      => 172,
 214                 'notin'    => 8713,
 215                 'nsub'     => 8836,
 216                 'Ntilde'   => 209,
 217                 'ntilde'   => 241,
 218                 'Nu'       => 925,
 219                 'nu'       => 957,
 220                 'Oacute'   => 211,
 221                 'oacute'   => 243,
 222                 'Ocirc'    => 212,
 223                 'ocirc'    => 244,
 224                 'OElig'    => 338,
 225                 'oelig'    => 339,
 226                 'Ograve'   => 210,
 227                 'ograve'   => 242,
 228                 'oline'    => 8254,
 229                 'Omega'    => 937,
 230                 'omega'    => 969,
 231                 'Omicron'  => 927,
 232                 'omicron'  => 959,
 233                 'oplus'    => 8853,
 234                 'or'       => 8744,
 235                 'ordf'     => 170,
 236                 'ordm'     => 186,
 237                 'Oslash'   => 216,
 238                 'oslash'   => 248,
 239                 'Otilde'   => 213,
 240                 'otilde'   => 245,
 241                 'otimes'   => 8855,
 242                 'Ouml'     => 214,
 243                 'ouml'     => 246,
 244                 'para'     => 182,
 245                 'part'     => 8706,
 246                 'permil'   => 8240,
 247                 'perp'     => 8869,
 248                 'Phi'      => 934,
 249                 'phi'      => 966,
 250                 'Pi'       => 928,
 251                 'pi'       => 960,
 252                 'piv'      => 982,
 253                 'plusmn'   => 177,
 254                 'pound'    => 163,
 255                 'prime'    => 8242,
 256                 'Prime'    => 8243,
 257                 'prod'     => 8719,
 258                 'prop'     => 8733,
 259                 'Psi'      => 936,
 260                 'psi'      => 968,
 261                 'quot'     => 34,
 262                 'radic'    => 8730,
 263                 'rang'     => 9002,
 264                 'raquo'    => 187,
 265                 'rarr'     => 8594,
 266                 'rArr'     => 8658,
 267                 'rceil'    => 8969,
 268                 'rdquo'    => 8221,
 269                 'real'     => 8476,
 270                 'reg'      => 174,
 271                 'rfloor'   => 8971,
 272                 'Rho'      => 929,
 273                 'rho'      => 961,
 274                 'rlm'      => 8207,
 275                 'rsaquo'   => 8250,
 276                 'rsquo'    => 8217,
 277                 'sbquo'    => 8218,
 278                 'Scaron'   => 352,
 279                 'scaron'   => 353,
 280                 'sdot'     => 8901,
 281                 'sect'     => 167,
 282                 'shy'      => 173,
 283                 'Sigma'    => 931,
 284                 'sigma'    => 963,
 285                 'sigmaf'   => 962,
 286                 'sim'      => 8764,
 287                 'spades'   => 9824,
 288                 'sub'      => 8834,
 289                 'sube'     => 8838,
 290                 'sum'      => 8721,
 291                 'sup'      => 8835,
 292                 'sup1'     => 185,
 293                 'sup2'     => 178,
 294                 'sup3'     => 179,
 295                 'supe'     => 8839,
 296                 'szlig'    => 223,
 297                 'Tau'      => 932,
 298                 'tau'      => 964,
 299                 'there4'   => 8756,
 300                 'Theta'    => 920,
 301                 'theta'    => 952,
 302                 'thetasym' => 977,
 303                 'thinsp'   => 8201,
 304                 'THORN'    => 222,
 305                 'thorn'    => 254,
 306                 'tilde'    => 732,
 307                 'times'    => 215,
 308                 'trade'    => 8482,
 309                 'Uacute'   => 218,
 310                 'uacute'   => 250,
 311                 'uarr'     => 8593,
 312                 'uArr'     => 8657,
 313                 'Ucirc'    => 219,
 314                 'ucirc'    => 251,
 315                 'Ugrave'   => 217,
 316                 'ugrave'   => 249,
 317                 'uml'      => 168,
 318                 'upsih'    => 978,
 319                 'Upsilon'  => 933,
 320                 'upsilon'  => 965,
 321                 'Uuml'     => 220,
 322                 'uuml'     => 252,
 323                 'weierp'   => 8472,
 324                 'Xi'       => 926,
 325                 'xi'       => 958,
 326                 'Yacute'   => 221,
 327                 'yacute'   => 253,
 328                 'yen'      => 165,
 329                 'Yuml'     => 376,
 330                 'yuml'     => 255,
 331                 'Zeta'     => 918,
 332                 'zeta'     => 950,
 333                 'zwj'      => 8205,
 334                 'zwnj'     => 8204
 335         ];
 336
 337         /**
 338          * Character entity aliases accepted by MediaWiki
 339          */
 340         private static $htmlEntityAliases = [
 341                 'רלמ' => 'rlm',
 342                 'رلم' => 'rlm',
 343         ];
 344
 345         /**
 346          * Lazy-initialised attributes regex, see getAttribsRegex()
 347          */
 348         private static $attribsRegex;
 349
 350         /**
 351          * Regular expression to match HTML/XML attribute pairs within a tag.
 352          * Based on https://www.w3.org/TR/html5/syntax.html#before-attribute-name-state
 353          * Used in Sanitizer::decodeTagAttributes
 354          * @return string
 355          */
 356         static function getAttribsRegex() {
 357                 if ( self::$attribsRegex === null ) {
 358                         $spaceChars = '\x09\x0a\x0c\x0d\x20';
 359                         $space = "[{$spaceChars}]";
 360                         $attrib = "[^{$spaceChars}\/>=]";
 361                         $attribFirst = "(?:{$attrib}|=)";
 362                         self::$attribsRegex =
 363                                 "/({$attribFirst}{$attrib}*)
 364                                         ($space*=$space*
 365                                         (?:
 366                                                 # The attribute value: quoted or alone
 367                                                 \"([^\"]*)(?:\"|\$)
 368                                                 | '([^']*)(?:'|\$)
 369                                                 | (((?!$space|>).)*)
 370                                         )
 371                                 )?/sxu";
 372                 }
 373                 return self::$attribsRegex;
 374         }
 375
 376         /**
 377          * Lazy-initialised attribute name regex, see getAttribNameRegex()
 378          */
 379         private static $attribNameRegex;
 380
 381         /**
 382          * Used in Sanitizer::decodeTagAttributes to filter attributes.
 383          * @return string
 384          */
 385         static function getAttribNameRegex() {
 386                 if ( self::$attribNameRegex === null ) {
 387                         $attribFirst = "[:_\p{L}\p{N}]";
 388                         $attrib = "[:_\.\-\p{L}\p{N}]";
 389                         self::$attribNameRegex = "/^({$attribFirst}{$attrib}*)$/sxu";
 390                 }
 391                 return self::$attribNameRegex;
 392         }
 393
 394         /**
 395          * Return the various lists of recognized tags
 396          * @param array $extratags For any extra tags to include
 397          * @param array $removetags For any tags (default or extra) to exclude
 398          * @return array
 399          */
 400         public static function getRecognizedTagData( $extratags = [], $removetags = [] ) {
 401                 global $wgAllowImageTag;
 402
 403                 static $htmlpairsStatic, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
 404                         $htmllist, $listtags, $htmlsingleallowed, $htmlelementsStatic, $staticInitialised;
 405
 406                 // Base our staticInitialised variable off of the global config state so that if the globals
 407                 // are changed (like in the screwed up test system) we will re-initialise the settings.
 408                 $globalContext = $wgAllowImageTag;
 409                 if ( !$staticInitialised || $staticInitialised != $globalContext ) {
 410                         $htmlpairsStatic = [ # Tags that must be closed
 411                                 'b', 'bdi', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
 412                                 'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
 413                                 'strike', 'strong', 'tt', 'var', 'div', 'center',
 414                                 'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
 415                                 'ruby', 'rb', 'rp', 'rt', 'rtc', 'p', 'span', 'abbr', 'dfn',
 416                                 'kbd', 'samp', 'data', 'time', 'mark'
 417                         ];
 418                         $htmlsingle = [
 419                                 'br', 'wbr', 'hr', 'li', 'dt', 'dd', 'meta', 'link'
 420                         ];
 421
 422                         # Elements that cannot have close tags. This is (not coincidentally)
 423                         # also the list of tags for which the HTML 5 parsing algorithm
 424                         # requires you to "acknowledge the token's self-closing flag", i.e.
 425                         # a self-closing tag like <br/> is not an HTML 5 parse error only
 426                         # for this list.
 427                         $htmlsingleonly = [
 428                                 'br', 'wbr', 'hr', 'meta', 'link'
 429                         ];
 430
 431                         $htmlnest = [ # Tags that can be nested--??
 432                                 'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
 433                                 'li', 'dl', 'dt', 'dd', 'font', 'big', 'small', 'sub', 'sup', 'span',
 434                                 'var', 'kbd', 'samp', 'em', 'strong', 'q', 'ruby', 'bdo'
 435                         ];
 436                         $tabletags = [ # Can only appear inside table, we will close them
 437                                 'td', 'th', 'tr',
 438                         ];
 439                         $htmllist = [ # Tags used by list
 440                                 'ul', 'ol',
 441                         ];
 442                         $listtags = [ # Tags that can appear in a list
 443                                 'li',
 444                         ];
 445
 446                         if ( $wgAllowImageTag ) {
 447                                 $htmlsingle[] = 'img';
 448                                 $htmlsingleonly[] = 'img';
 449                         }
 450
 451                         $htmlsingleallowed = array_unique( array_merge( $htmlsingle, $tabletags ) );
 452                         $htmlelementsStatic = array_unique( array_merge( $htmlsingle, $htmlpairsStatic, $htmlnest ) );
 453
 454                         # Convert them all to hashtables for faster lookup
 455                         $vars = [ 'htmlpairsStatic', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
 456                                 'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelementsStatic' ];
 457                         foreach ( $vars as $var ) {
 458                                 $$var = array_flip( $$var );
 459                         }
 460                         $staticInitialised = $globalContext;
 461                 }
 462
 463                 # Populate $htmlpairs and $htmlelements with the $extratags and $removetags arrays
 464                 $extratags = array_flip( $extratags );
 465                 $removetags = array_flip( $removetags );
 466                 $htmlpairs = array_merge( $extratags, $htmlpairsStatic );
 467                 $htmlelements = array_diff_key( array_merge( $extratags, $htmlelementsStatic ), $removetags );
 468
 469                 return [
 470                         'htmlpairs' => $htmlpairs,
 471                         'htmlsingle' => $htmlsingle,
 472                         'htmlsingleonly' => $htmlsingleonly,
 473                         'htmlnest' => $htmlnest,
 474                         'tabletags' => $tabletags,
 475                         'htmllist' => $htmllist,
 476                         'listtags' => $listtags,
 477                         'htmlsingleallowed' => $htmlsingleallowed,
 478                         'htmlelements' => $htmlelements,
 479                 ];
 480         }
 481
 482         /**
 483          * Cleans up HTML, removes dangerous tags and attributes, and
 484          * removes HTML comments
 485          * @param string $text
 486          * @param callable|null $processCallback Callback to do any variable or parameter
 487          *   replacements in HTML attribute values
 488          * @param array|bool $args Arguments for the processing callback
 489          * @param array $extratags For any extra tags to include
 490          * @param array $removetags For any tags (default or extra) to exclude
 491          * @param callable|null $warnCallback (Deprecated) Callback allowing the
 492          *   addition of a tracking category when bad input is encountered.
 493          *   DO NOT ADD NEW PARAMETERS AFTER $warnCallback, since it will be
 494          *   removed shortly.
 495          * @return string
 496          */
 497         public static function removeHTMLtags( $text, $processCallback = null,
 498                 $args = [], $extratags = [], $removetags = [], $warnCallback = null
 499         ) {
 500                 $tagData = self::getRecognizedTagData( $extratags, $removetags );
 501                 $htmlpairs = $tagData['htmlpairs'];
 502                 $htmlsingle = $tagData['htmlsingle'];
 503                 $htmlsingleonly = $tagData['htmlsingleonly'];
 504                 $htmlnest = $tagData['htmlnest'];
 505                 $tabletags = $tagData['tabletags'];
 506                 $htmllist = $tagData['htmllist'];
 507                 $listtags = $tagData['listtags'];
 508                 $htmlsingleallowed = $tagData['htmlsingleallowed'];
 509                 $htmlelements = $tagData['htmlelements'];
 510
 511                 # Remove HTML comments
 512                 $text = self::removeHTMLcomments( $text );
 513                 $bits = explode( '<', $text );
 514                 $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 515                 if ( !MWTidy::isEnabled() ) {
 516                         wfDeprecated( 'disabling tidy', '1.33' );
 517                         $tagstack = $tablestack = [];
 518                         foreach ( $bits as $x ) {
 519                                 $regs = [];
 520                                 # $slash: Does the current element start with a '/'?
 521                                 # $t: Current element name
 522                                 # $params: String between element name and >
 523                                 # $brace: Ending '>' or '/>'
 524                                 # $rest: Everything until the next element of $bits
 525                                 if ( preg_match( self::ELEMENT_BITS_REGEX, $x, $regs ) ) {
 526                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 527                                 } else {
 528                                         $slash = $t = $params = $brace = $rest = null;
 529                                 }
 530
 531                                 $badtag = false;
 532                                 $t = strtolower( $t );
 533                                 if ( isset( $htmlelements[$t] ) ) {
 534                                         # Check our stack
 535                                         if ( $slash && isset( $htmlsingleonly[$t] ) ) {
 536                                                 $badtag = true;
 537                                         } elseif ( $slash ) {
 538                                                 # Closing a tag... is it the one we just opened?
 539                                                 Wikimedia\suppressWarnings();
 540                                                 $ot = array_pop( $tagstack );
 541                                                 Wikimedia\restoreWarnings();
 542
 543                                                 if ( $ot != $t ) {
 544                                                         if ( isset( $htmlsingleallowed[$ot] ) ) {
 545                                                                 # Pop all elements with an optional close tag
 546                                                                 # and see if we find a match below them
 547                                                                 $optstack = [];
 548                                                                 array_push( $optstack, $ot );
 549                                                                 Wikimedia\suppressWarnings();
 550                                                                 $ot = array_pop( $tagstack );
 551                                                                 Wikimedia\restoreWarnings();
 552                                                                 while ( $ot != $t && isset( $htmlsingleallowed[$ot] ) ) {
 553                                                                         array_push( $optstack, $ot );
 554                                                                         Wikimedia\suppressWarnings();
 555                                                                         $ot = array_pop( $tagstack );
 556                                                                         Wikimedia\restoreWarnings();
 557                                                                 }
 558                                                                 if ( $t != $ot ) {
 559                                                                         # No match. Push the optional elements back again
 560                                                                         $badtag = true;
 561                                                                         Wikimedia\suppressWarnings();
 562                                                                         $ot = array_pop( $optstack );
 563                                                                         Wikimedia\restoreWarnings();
 564                                                                         while ( $ot ) {
 565                                                                                 array_push( $tagstack, $ot );
 566                                                                                 Wikimedia\suppressWarnings();
 567                                                                                 $ot = array_pop( $optstack );
 568                                                                                 Wikimedia\restoreWarnings();
 569                                                                         }
 570                                                                 }
 571                                                         } else {
 572                                                                 Wikimedia\suppressWarnings();
 573                                                                 array_push( $tagstack, $ot );
 574                                                                 Wikimedia\restoreWarnings();
 575
 576                                                                 # <li> can be nested in <ul> or <ol>, skip those cases:
 577                                                                 if ( !isset( $htmllist[$ot] ) || !isset( $listtags[$t] ) ) {
 578                                                                         $badtag = true;
 579                                                                 }
 580                                                         }
 581                                                 } elseif ( $t == 'table' ) {
 582                                                         $tagstack = array_pop( $tablestack );
 583                                                 }
 584                                                 $newparams = '';
 585                                         } else {
 586                                                 # Keep track for later
 587                                                 if ( isset( $tabletags[$t] ) && !in_array( 'table', $tagstack ) ) {
 588                                                         $badtag = true;
 589                                                 } elseif ( in_array( $t, $tagstack ) && !isset( $htmlnest[$t] ) ) {
 590                                                         $badtag = true;
 591                                                 #  Is it a self closed htmlpair ? (T7487)
 592                                                 } elseif ( $brace == '/>' && isset( $htmlpairs[$t] ) ) {
 593                                                         // Eventually we'll just remove the self-closing
 594                                                         // slash, in order to be consistent with HTML5
 595                                                         // semantics.
 596                                                         // $brace = '>';
 597                                                         // For now, let's just warn authors to clean up.
 598                                                         if ( is_callable( $warnCallback ) ) {
 599                                                                 call_user_func_array( $warnCallback, [ 'deprecated-self-close-category' ] );
 600                                                         }
 601                                                         $badtag = true;
 602                                                 } elseif ( isset( $htmlsingleonly[$t] ) ) {
 603                                                         # Hack to force empty tag for unclosable elements
 604                                                         $brace = '/>';
 605                                                 } elseif ( isset( $htmlsingle[$t] ) ) {
 606                                                         # Hack to not close $htmlsingle tags
 607                                                         $brace = null;
 608                                                         # Still need to push this optionally-closed tag to
 609                                                         # the tag stack so that we can match end tags
 610                                                         # instead of marking them as bad.
 611                                                         array_push( $tagstack, $t );
 612                                                 } elseif ( isset( $tabletags[$t] ) && in_array( $t, $tagstack ) ) {
 613                                                         // New table tag but forgot to close the previous one
 614                                                         $text .= "</$t>";
 615                                                 } else {
 616                                                         if ( $t == 'table' ) {
 617                                                                 array_push( $tablestack, $tagstack );
 618                                                                 $tagstack = [];
 619                                                         }
 620                                                         array_push( $tagstack, $t );
 621                                                 }
 622
 623                                                 # Replace any variables or template parameters with
 624                                                 # plaintext results.
 625                                                 if ( is_callable( $processCallback ) ) {
 626                                                         call_user_func_array( $processCallback, [ &$params, $args ] );
 627                                                 }
 628
 629                                                 if ( !self::validateTag( $params, $t ) ) {
 630                                                         $badtag = true;
 631                                                 }
 632
 633                                                 # Strip non-approved attributes from the tag
 634                                                 $newparams = self::fixTagAttributes( $params, $t );
 635                                         }
 636                                         if ( !$badtag ) {
 637                                                 $rest = str_replace( '>', '&gt;', $rest );
 638                                                 $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
 639                                                 $text .= "<$slash$t$newparams$close>$rest";
 640                                                 continue;
 641                                         }
 642                                 }
 643                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x );
 644                         }
 645                         # Close off any remaining tags
 646                         while ( is_array( $tagstack ) && ( $t = array_pop( $tagstack ) ) ) {
 647                                 $text .= "</$t>\n";
 648                                 if ( $t == 'table' ) {
 649                                         $tagstack = array_pop( $tablestack );
 650                                 }
 651                         }
 652                 } else {
 653                         # this might be possible using tidy itself
 654                         foreach ( $bits as $x ) {
 655                                 if ( preg_match( self::ELEMENT_BITS_REGEX, $x, $regs ) ) {
 656                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 657
 658                                         $badtag = false;
 659                                         $t = strtolower( $t );
 660                                         if ( isset( $htmlelements[$t] ) ) {
 661                                                 if ( is_callable( $processCallback ) ) {
 662                                                         call_user_func_array( $processCallback, [ &$params, $args ] );
 663                                                 }
 664
 665                                                 if ( $brace == '/>' && !( isset( $htmlsingle[$t] ) || isset( $htmlsingleonly[$t] ) ) ) {
 666                                                         // Eventually we'll just remove the self-closing
 667                                                         // slash, in order to be consistent with HTML5
 668                                                         // semantics.
 669                                                         // $brace = '>';
 670                                                         // For now, let's just warn authors to clean up.
 671                                                         if ( is_callable( $warnCallback ) ) {
 672                                                                 call_user_func_array( $warnCallback, [ 'deprecated-self-close-category' ] );
 673                                                         }
 674                                                 }
 675                                                 if ( !self::validateTag( $params, $t ) ) {
 676                                                         $badtag = true;
 677                                                 }
 678
 679                                                 $newparams = self::fixTagAttributes( $params, $t );
 680                                                 if ( !$badtag ) {
 681                                                         if ( $brace === '/>' && !isset( $htmlsingleonly[$t] ) ) {
 682                                                                 # Interpret self-closing tags as empty tags even when
 683                                                                 # HTML 5 would interpret them as start tags. Such input
 684                                                                 # is commonly seen on Wikimedia wikis with this intention.
 685                                                                 $brace = "></$t>";
 686                                                         }
 687
 688                                                         $rest = str_replace( '>', '&gt;', $rest );
 689                                                         $text .= "<$slash$t$newparams$brace$rest";
 690                                                         continue;
 691                                                 }
 692                                         }
 693                                 }
 694                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x );
 695                         }
 696                 }
 697                 return $text;
 698         }
 699
 700         /**
 701          * Remove '<!--', '-->', and everything between.
 702          * To avoid leaving blank lines, when a comment is both preceded
 703          * and followed by a newline (ignoring spaces), trim leading and
 704          * trailing spaces and one of the newlines.
 705          *
 706          * @param string $text
 707          * @return string
 708          */
 709         public static function removeHTMLcomments( $text ) {
 710                 while ( ( $start = strpos( $text, '<!--' ) ) !== false ) {
 711                         $end = strpos( $text, '-->', $start + 4 );
 712                         if ( $end === false ) {
 713                                 # Unterminated comment; bail out
 714                                 break;
 715                         }
 716
 717                         $end += 3;
 718
 719                         # Trim space and newline if the comment is both
 720                         # preceded and followed by a newline
 721                         $spaceStart = max( $start - 1, 0 );
 722                         $spaceLen = $end - $spaceStart;
 723                         while ( substr( $text, $spaceStart, 1 ) === ' ' && $spaceStart > 0 ) {
 724                                 $spaceStart--;
 725                                 $spaceLen++;
 726                         }
 727                         while ( substr( $text, $spaceStart + $spaceLen, 1 ) === ' ' ) {
 728                                 $spaceLen++;
 729                         }
 730                         if ( substr( $text, $spaceStart, 1 ) === "\n"
 731                                 && substr( $text, $spaceStart + $spaceLen, 1 ) === "\n" ) {
 732                                 # Remove the comment, leading and trailing
 733                                 # spaces, and leave only one newline.
 734                                 $text = substr_replace( $text, "\n", $spaceStart, $spaceLen + 1 );
 735                         } else {
 736                                 # Remove just the comment.
 737                                 $text = substr_replace( $text, '', $start, $end - $start );
 738                         }
 739                 }
 740                 return $text;
 741         }
 742
 743         /**
 744          * Takes attribute names and values for a tag and the tag name and
 745          * validates that the tag is allowed to be present.
 746          * This DOES NOT validate the attributes, nor does it validate the
 747          * tags themselves. This method only handles the special circumstances
 748          * where we may want to allow a tag within content but ONLY when it has
 749          * specific attributes set.
 750          *
 751          * @param string $params
 752          * @param string $element
 753          * @return bool
 754          */
 755         static function validateTag( $params, $element ) {
 756                 $params = self::decodeTagAttributes( $params );
 757
 758                 if ( $element == 'meta' || $element == 'link' ) {
 759                         if ( !isset( $params['itemprop'] ) ) {
 760                                 // <meta> and <link> must have an itemprop="" otherwise they are not valid or safe in content
 761                                 return false;
 762                         }
 763                         if ( $element == 'meta' && !isset( $params['content'] ) ) {
 764                                 // <meta> must have a content="" for the itemprop
 765                                 return false;
 766                         }
 767                         if ( $element == 'link' && !isset( $params['href'] ) ) {
 768                                 // <link> must have an associated href=""
 769                                 return false;
 770                         }
 771                 }
 772
 773                 return true;
 774         }
 775
 776         /**
 777          * Take an array of attribute names and values and normalize or discard
 778          * illegal values for the given element type.
 779          *
 780          * - Discards attributes not on a whitelist for the given element
 781          * - Unsafe style attributes are discarded
 782          * - Invalid id attributes are re-encoded
 783          *
 784          * @param array $attribs
 785          * @param string $element
 786          * @return array
 787          *
 788          * @todo Check for legal values where the DTD limits things.
 789          * @todo Check for unique id attribute :P
 790          */
 791         static function validateTagAttributes( $attribs, $element ) {
 792                 return self::validateAttributes( $attribs,
 793                         self::attributeWhitelistInternal( $element ) );
 794         }
 795
 796         /**
 797          * Take an array of attribute names and values and normalize or discard
 798          * illegal values for the given whitelist.
 799          *
 800          * - Discards attributes not on the given whitelist
 801          * - Unsafe style attributes are discarded
 802          * - Invalid id attributes are re-encoded
 803          *
 804          * @param array $attribs
 805          * @param array $whitelist List of allowed attribute names,
 806          *   either as a sequential array of valid attribute names or
 807          *   as an associative array where keys give valid attribute names
 808          * @return array
 809          *
 810          * @todo Check for legal values where the DTD limits things.
 811          * @todo Check for unique id attribute :P
 812          */
 813         static function validateAttributes( $attribs, $whitelist ) {
 814                 if ( isset( $whitelist[0] ) ) {
 815                         // We would like to eventually deprecate calling this
 816                         // function with a sequential array, but for now just
 817                         // convert it.
 818                         $whitelist = array_flip( $whitelist );
 819                 }
 820                 $hrefExp = '/^(' . wfUrlProtocols() . ')[^\s]+$/';
 821
 822                 $out = [];
 823                 foreach ( $attribs as $attribute => $value ) {
 824                         # Allow XML namespace declaration to allow RDFa
 825                         if ( preg_match( self::XMLNS_ATTRIBUTE_PATTERN, $attribute ) ) {
 826                                 if ( !preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 827                                         $out[$attribute] = $value;
 828                                 }
 829
 830                                 continue;
 831                         }
 832
 833                         # Allow any attribute beginning with "data-"
 834                         # However:
 835                         # * Disallow data attributes used by MediaWiki code
 836                         # * Ensure that the attribute is not namespaced by banning
 837                         #   colons.
 838                         if ( (
 839                                 !preg_match( '/^data-[^:]*$/i', $attribute ) &&
 840                                 !array_key_exists( $attribute, $whitelist )
 841                         ) || self::isReservedDataAttribute( $attribute ) ) {
 842                                 continue;
 843                         }
 844
 845                         # Strip javascript "expression" from stylesheets.
 846                         # https://msdn.microsoft.com/en-us/library/ms537634.aspx
 847                         if ( $attribute == 'style' ) {
 848                                 $value = self::checkCss( $value );
 849                         }
 850
 851                         # Escape HTML id attributes
 852                         if ( $attribute === 'id' ) {
 853                                 $value = self::escapeIdForAttribute( $value, self::ID_PRIMARY );
 854                         }
 855
 856                         # Escape HTML id reference lists
 857                         if ( $attribute === 'aria-describedby'
 858                                 || $attribute === 'aria-flowto'
 859                                 || $attribute === 'aria-labelledby'
 860                                 || $attribute === 'aria-owns'
 861                         ) {
 862                                 $value = self::escapeIdReferenceList( $value );
 863                         }
 864
 865                         // RDFa and microdata properties allow URLs, URIs and/or CURIs.
 866                         // Check them for sanity.
 867                         if ( $attribute === 'rel' || $attribute === 'rev'
 868                                 # RDFa
 869                                 || $attribute === 'about' || $attribute === 'property'
 870                                 || $attribute === 'resource' || $attribute === 'datatype'
 871                                 || $attribute === 'typeof'
 872                                 # HTML5 microdata
 873                                 || $attribute === 'itemid' || $attribute === 'itemprop'
 874                                 || $attribute === 'itemref' || $attribute === 'itemscope'
 875                                 || $attribute === 'itemtype'
 876                         ) {
 877                                 // Paranoia. Allow "simple" values but suppress javascript
 878                                 if ( preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 879                                         continue;
 880                                 }
 881                         }
 882
 883                         # NOTE: even though elements using href/src are not allowed directly, supply
 884                         #       validation code that can be used by tag hook handlers, etc
 885                         if ( $attribute === 'href' || $attribute === 'src' || $attribute === 'poster' ) {
 886                                 if ( !preg_match( $hrefExp, $value ) ) {
 887                                         continue; // drop any href or src attributes not using an allowed protocol.
 888                                         // NOTE: this also drops all relative URLs
 889                                 }
 890                         }
 891
 892                         // If this attribute was previously set, override it.
 893                         // Output should only have one attribute of each name.
 894                         $out[$attribute] = $value;
 895                 }
 896
 897                 # itemtype, itemid, itemref don't make sense without itemscope
 898                 if ( !array_key_exists( 'itemscope', $out ) ) {
 899                         unset( $out['itemtype'] );
 900                         unset( $out['itemid'] );
 901                         unset( $out['itemref'] );
 902                 }
 903                 # TODO: Strip itemprop if we aren't descendants of an itemscope or pointed to by an itemref.
 904
 905                 return $out;
 906         }
 907
 908         /**
 909          * Given an attribute name, checks whether it is a reserved data attribute
 910          * (such as data-mw-foo) which is unavailable to user-generated HTML so MediaWiki
 911          * core and extension code can safely use it to communicate with frontend code.
 912          * @param string $attr Attribute name.
 913          * @return bool
 914          */
 915         public static function isReservedDataAttribute( $attr ) {
 916                 // data-ooui is reserved for ooui.
 917                 // data-mw and data-parsoid are reserved for parsoid.
 918                 // data-mw-<name here> is reserved for extensions (or core) if
 919                 // they need to communicate some data to the client and want to be
 920                 // sure that it isn't coming from an untrusted user.
 921                 // We ignore the possibility of namespaces since user-generated HTML
 922                 // can't use them anymore.
 923                 return (bool)preg_match( '/^data-(ooui|mw|parsoid)/i', $attr );
 924         }
 925
 926         /**
 927          * Merge two sets of HTML attributes.  Conflicting items in the second set
 928          * will override those in the first, except for 'class' attributes which
 929          * will be combined (if they're both strings).
 930          *
 931          * @todo implement merging for other attributes such as style
 932          * @param array $a
 933          * @param array $b
 934          * @return array
 935          */
 936         static function mergeAttributes( $a, $b ) {
 937                 $out = array_merge( $a, $b );
 938                 if ( isset( $a['class'] ) && isset( $b['class'] )
 939                         && is_string( $a['class'] ) && is_string( $b['class'] )
 940                         && $a['class'] !== $b['class']
 941                 ) {
 942                         $classes = preg_split( '/\s+/', "{$a['class']} {$b['class']}",
 943                                 -1, PREG_SPLIT_NO_EMPTY );
 944                         $out['class'] = implode( ' ', array_unique( $classes ) );
 945                 }
 946                 return $out;
 947         }
 948
 949         /**
 950          * Normalize CSS into a format we can easily search for hostile input
 951          *  - decode character references
 952          *  - decode escape sequences
 953          *  - convert characters that IE6 interprets into ascii
 954          *  - remove comments, unless the entire value is one single comment
 955          * @param string $value the css string
 956          * @return string normalized css
 957          */
 958         public static function normalizeCss( $value ) {
 959                 // Decode character references like &#123;
 960                 $value = self::decodeCharReferences( $value );
 961
 962                 // Decode escape sequences and line continuation
 963                 // See the grammar in the CSS 2 spec, appendix D.
 964                 // This has to be done AFTER decoding character references.
 965                 // This means it isn't possible for this function to return
 966                 // unsanitized escape sequences. It is possible to manufacture
 967                 // input that contains character references that decode to
 968                 // escape sequences that decode to character references, but
 969                 // it's OK for the return value to contain character references
 970                 // because the caller is supposed to escape those anyway.
 971                 static $decodeRegex;
 972                 if ( !$decodeRegex ) {
 973                         $space = '[\\x20\\t\\r\\n\\f]';
 974                         $nl = '(?:\\n|\\r\\n|\\r|\\f)';
 975                         $backslash = '\\\\';
 976                         $decodeRegex = "/ $backslash
 977                                 (?:
 978                                         ($nl) |  # 1. Line continuation
 979                                         ([0-9A-Fa-f]{1,6})$space? |  # 2. character number
 980                                         (.) | # 3. backslash cancelling special meaning
 981                                         () | # 4. backslash at end of string
 982                                 )/xu";
 983                 }
 984                 $value = preg_replace_callback( $decodeRegex,
 985                         [ __CLASS__, 'cssDecodeCallback' ], $value );
 986
 987                 // Normalize Halfwidth and Fullwidth Unicode block that IE6 might treat as ascii
 988                 $value = preg_replace_callback(
 989                         '/[！-［］-ｚ]/u', // U+FF01 to U+FF5A, excluding U+FF3C (T60088)
 990                         function ( $matches ) {
 991                                 $cp = UtfNormal\Utils::utf8ToCodepoint( $matches[0] );
 992                                 if ( $cp === false ) {
 993                                         return '';
 994                                 }
 995                                 return chr( $cp - 65248 ); // ASCII range \x21-\x7A
 996                         },
 997                         $value
 998                 );
 999
1000                 // Convert more characters IE6 might treat as ascii
1001                 // U+0280, U+0274, U+207F, U+029F, U+026A, U+207D, U+208D
1002                 $value = str_replace(
1003                         [ 'ʀ', 'ɴ', 'ⁿ', 'ʟ', 'ɪ', '⁽', '₍' ],
1004                         [ 'r', 'n', 'n', 'l', 'i', '(', '(' ],
1005                         $value
1006                 );
1007
1008                 // Let the value through if it's nothing but a single comment, to
1009                 // allow other functions which may reject it to pass some error
1010                 // message through.
1011                 if ( !preg_match( '! ^ \s* /\* [^*\\/]* \*/ \s* $ !x', $value ) ) {
1012                         // Remove any comments; IE gets token splitting wrong
1013                         // This must be done AFTER decoding character references and
1014                         // escape sequences, because those steps can introduce comments
1015                         // This step cannot introduce character references or escape
1016                         // sequences, because it replaces comments with spaces rather
1017                         // than removing them completely.
1018                         $value = StringUtils::delimiterReplace( '/*', '*/', ' ', $value );
1019
1020                         // Remove anything after a comment-start token, to guard against
1021                         // incorrect client implementations.
1022                         $commentPos = strpos( $value, '/*' );
1023                         if ( $commentPos !== false ) {
1024                                 $value = substr( $value, 0, $commentPos );
1025                         }
1026                 }
1027
1028                 // S followed by repeat, iteration, or prolonged sound marks,
1029                 // which IE will treat as "ss"
1030                 $value = preg_replace(
1031                         '/s(?:
1032                                 \xE3\x80\xB1 | # U+3031
1033                                 \xE3\x82\x9D | # U+309D
1034                                 \xE3\x83\xBC | # U+30FC
1035                                 \xE3\x83\xBD | # U+30FD
1036                                 \xEF\xB9\xBC | # U+FE7C
1037                                 \xEF\xB9\xBD | # U+FE7D
1038                                 \xEF\xBD\xB0   # U+FF70
1039                         )/ix',
1040                         'ss',
1041                         $value
1042                 );
1043
1044                 return $value;
1045         }
1046
1047         /**
1048          * Pick apart some CSS and check it for forbidden or unsafe structures.
1049          * Returns a sanitized string. This sanitized string will have
1050          * character references and escape sequences decoded and comments
1051          * stripped (unless it is itself one valid comment, in which case the value
1052          * will be passed through). If the input is just too evil, only a comment
1053          * complaining about evilness will be returned.
1054          *
1055          * Currently URL references, 'expression', 'tps' are forbidden.
1056          *
1057          * NOTE: Despite the fact that character references are decoded, the
1058          * returned string may contain character references given certain
1059          * clever input strings. These character references must
1060          * be escaped before the return value is embedded in HTML.
1061          *
1062          * @param string $value
1063          * @return string
1064          */
1065         static function checkCss( $value ) {
1066                 $value = self::normalizeCss( $value );
1067
1068                 // Reject problematic keywords and control characters
1069                 if ( preg_match( '/[\000-\010\013\016-\037\177]/', $value ) ||
1070                         strpos( $value, UtfNormal\Constants::UTF8_REPLACEMENT ) !== false ) {
1071                         return '/* invalid control char */';
1072                 } elseif ( preg_match(
1073                         '! expression
1074                                 | filter\s*:
1075                                 | accelerator\s*:
1076                                 | -o-link\s*:
1077                                 | -o-link-source\s*:
1078                                 | -o-replace\s*:
1079                                 | url\s*\(
1080                                 | image\s*\(
1081                                 | image-set\s*\(
1082                                 | attr\s*\([^)]+[\s,]+url
1083                                 | var\s*\(
1084                         !ix', $value ) ) {
1085                         return '/* insecure input */';
1086                 }
1087                 return $value;
1088         }
1089
1090         /**
1091          * @param array $matches
1092          * @return string
1093          */
1094         static function cssDecodeCallback( $matches ) {
1095                 if ( $matches[1] !== '' ) {
1096                         // Line continuation
1097                         return '';
1098                 } elseif ( $matches[2] !== '' ) {
1099                         $char = UtfNormal\Utils::codepointToUtf8( hexdec( $matches[2] ) );
1100                 } elseif ( $matches[3] !== '' ) {
1101                         $char = $matches[3];
1102                 } else {
1103                         $char = '\\';
1104                 }
1105                 if ( $char == "\n" || $char == '"' || $char == "'" || $char == '\\' ) {
1106                         // These characters need to be escaped in strings
1107                         // Clean up the escape sequence to avoid parsing errors by clients
1108                         return '\\' . dechex( ord( $char ) ) . ' ';
1109                 } else {
1110                         // Decode unnecessary escape
1111                         return $char;
1112                 }
1113         }
1114
1115         /**
1116          * Take a tag soup fragment listing an HTML element's attributes
1117          * and normalize it to well-formed XML, discarding unwanted attributes.
1118          * Output is safe for further wikitext processing, with escaping of
1119          * values that could trigger problems.
1120          *
1121          * - Normalizes attribute names to lowercase
1122          * - Discards attributes not on a whitelist for the given element
1123          * - Turns broken or invalid entities into plaintext
1124          * - Double-quotes all attribute values
1125          * - Attributes without values are given the name as attribute
1126          * - Double attributes are discarded
1127          * - Unsafe style attributes are discarded
1128          * - Prepends space if there are attributes.
1129          * - (Optionally) Sorts attributes by name.
1130          *
1131          * @param string $text
1132          * @param string $element
1133          * @param bool $sorted Whether to sort the attributes (default: false)
1134          * @return string
1135          */
1136         static function fixTagAttributes( $text, $element, $sorted = false ) {
1137                 if ( trim( $text ) == '' ) {
1138                         return '';
1139                 }
1140
1141                 $decoded = self::decodeTagAttributes( $text );
1142                 $stripped = self::validateTagAttributes( $decoded, $element );
1143
1144                 if ( $sorted ) {
1145                         ksort( $stripped );
1146                 }
1147
1148                 return self::safeEncodeTagAttributes( $stripped );
1149         }
1150
1151         /**
1152          * Encode an attribute value for HTML output.
1153          * @param string $text
1154          * @return string HTML-encoded text fragment
1155          */
1156         static function encodeAttribute( $text ) {
1157                 $encValue = htmlspecialchars( $text, ENT_QUOTES );
1158
1159                 // Whitespace is normalized during attribute decoding,
1160                 // so if we've been passed non-spaces we must encode them
1161                 // ahead of time or they won't be preserved.
1162                 $encValue = strtr( $encValue, [
1163                         "\n" => '&#10;',
1164                         "\r" => '&#13;',
1165                         "\t" => '&#9;',
1166                 ] );
1167
1168                 return $encValue;
1169         }
1170
1171         /**
1172          * Armor French spaces with a replacement character
1173          *
1174          * @since 1.32
1175          * @param string $text Text to armor
1176          * @param string $space Space character for the French spaces, defaults to '&#160;'
1177          * @return string Armored text
1178          */
1179         public static function armorFrenchSpaces( $text, $space = '&#160;' ) {
1180                 // Replace $ with \$ and \ with \\
1181                 $space = preg_replace( '#(?<!\\\\)(\\$|\\\\)#', '\\\\$1', $space );
1182                 $fixtags = [
1183                         # French spaces, last one Guillemet-left
1184                         # only if there is something before the space
1185                         # and a non-word character after the punctuation.
1186                         '/(\S) (?=[?:;!%»›](?!\w))/u' => "\\1$space",
1187                         # French spaces, Guillemet-right
1188                         '/([«‹]) /u' => "\\1$space",
1189                 ];
1190                 return preg_replace( array_keys( $fixtags ), array_values( $fixtags ), $text );
1191         }
1192
1193         /**
1194          * Encode an attribute value for HTML tags, with extra armoring
1195          * against further wiki processing.
1196          * @param string $text
1197          * @return string HTML-encoded text fragment
1198          */
1199         static function safeEncodeAttribute( $text ) {
1200                 $encValue = self::encodeAttribute( $text );
1201
1202                 # Templates and links may be expanded in later parsing,
1203                 # creating invalid or dangerous output. Suppress this.
1204                 $encValue = strtr( $encValue, [
1205                         '<'    => '&lt;',   // This should never happen,
1206                         '>'    => '&gt;',   // we've received invalid input
1207                         '"'    => '&quot;', // which should have been escaped.
1208                         '{'    => '&#123;',
1209                         '}'    => '&#125;', // prevent unpaired language conversion syntax
1210                         '['    => '&#91;',
1211                         ']'    => '&#93;',
1212                         "''"   => '&#39;&#39;',
1213                         'ISBN' => '&#73;SBN',
1214                         'RFC'  => '&#82;FC',
1215                         'PMID' => '&#80;MID',
1216                         '|'    => '&#124;',
1217                         '__'   => '&#95;_',
1218                 ] );
1219
1220                 # Armor against French spaces detection (T5158)
1221                 $encValue = self::armorFrenchSpaces( $encValue, '&#32;' );
1222
1223                 # Stupid hack
1224                 $encValue = preg_replace_callback(
1225                         '/((?i)' . wfUrlProtocols() . ')/',
1226                         function ( $matches ) {
1227                                 return str_replace( ':', '&#58;', $matches[1] );
1228                         },
1229                         $encValue );
1230                 return $encValue;
1231         }
1232
1233         /**
1234          * Given a value, escape it so that it can be used in an id attribute and
1235          * return it.  This will use HTML5 validation, allowing anything but ASCII
1236          * whitespace.
1237          *
1238          * To ensure we don't have to bother escaping anything, we also strip ', ".
1239          * TODO: Is this the best tactic?
1240          *
1241          * We also strip # because it upsets IE, and % because it could be
1242          * ambiguous if it's part of something that looks like a percent escape
1243          * (which don't work reliably in fragments cross-browser).
1244          *
1245          * @deprecated since 1.30, use one of this class' escapeIdFor*() functions
1246          *
1247          * @see https://www.w3.org/TR/html401/types.html#type-name Valid characters
1248          *   in the id and name attributes
1249          * @see https://www.w3.org/TR/html401/struct/links.html#h-12.2.3 Anchors with
1250          *   the id attribute
1251          * @see https://www.w3.org/TR/html5/dom.html#the-id-attribute
1252          *   HTML5 definition of id attribute
1253          *
1254          * @param string $id Id to escape
1255          * @param string|array $options String or array of strings (default is []):
1256          *   'noninitial': This is a non-initial fragment of an id, not a full id,
1257          *       so don't pay attention if the first character isn't valid at the
1258          *       beginning of an id.
1259          * @return string
1260          */
1261         static function escapeId( $id, $options = [] ) {
1262                 $options = (array)$options;
1263
1264                 // HTML4-style escaping
1265                 static $replace = [
1266                         '%3A' => ':',
1267                         '%' => '.'
1268                 ];
1269
1270                 $id = urlencode( strtr( $id, ' ', '_' ) );
1271                 $id = strtr( $id, $replace );
1272
1273                 if ( !preg_match( '/^[a-zA-Z]/', $id ) && !in_array( 'noninitial', $options ) ) {
1274                         // Initial character must be a letter!
1275                         $id = "x$id";
1276                 }
1277                 return $id;
1278         }
1279
1280         /**
1281          * Given a section name or other user-generated or otherwise unsafe string, escapes it to be
1282          * a valid HTML id attribute.
1283          *
1284          * WARNING: unlike escapeId(), the output of this function is not guaranteed to be HTML safe,
1285          * be sure to use proper escaping.
1286          *
1287          * @param string $id String to escape
1288          * @param int $mode One of ID_* constants, specifying whether the primary or fallback encoding
1289          *     should be used.
1290          * @return string|bool Escaped ID or false if fallback encoding is requested but it's not
1291          *     configured.
1292          *
1293          * @since 1.30
1294          */
1295         public static function escapeIdForAttribute( $id, $mode = self::ID_PRIMARY ) {
1296                 global $wgFragmentMode;
1297
1298                 if ( !isset( $wgFragmentMode[$mode] ) ) {
1299                         if ( $mode === self::ID_PRIMARY ) {
1300                                 throw new UnexpectedValueException( '$wgFragmentMode is configured with no primary mode' );
1301                         }
1302                         return false;
1303                 }
1304
1305                 $internalMode = $wgFragmentMode[$mode];
1306
1307                 return self::escapeIdInternal( $id, $internalMode );
1308         }
1309
1310         /**
1311          * Given a section name or other user-generated or otherwise unsafe string, escapes it to be
1312          * a valid URL fragment.
1313          *
1314          * WARNING: unlike escapeId(), the output of this function is not guaranteed to be HTML safe,
1315          * be sure to use proper escaping.
1316          *
1317          * @param string $id String to escape
1318          * @return string Escaped ID
1319          *
1320          * @since 1.30
1321          */
1322         public static function escapeIdForLink( $id ) {
1323                 global $wgFragmentMode;
1324
1325                 if ( !isset( $wgFragmentMode[self::ID_PRIMARY] ) ) {
1326                         throw new UnexpectedValueException( '$wgFragmentMode is configured with no primary mode' );
1327                 }
1328
1329                 $mode = $wgFragmentMode[self::ID_PRIMARY];
1330
1331                 $id = self::escapeIdInternal( $id, $mode );
1332
1333                 return $id;
1334         }
1335
1336         /**
1337          * Given a section name or other user-generated or otherwise unsafe string, escapes it to be
1338          * a valid URL fragment for external interwikis.
1339          *
1340          * @param string $id String to escape
1341          * @return string Escaped ID
1342          *
1343          * @since 1.30
1344          */
1345         public static function escapeIdForExternalInterwiki( $id ) {
1346                 global $wgExternalInterwikiFragmentMode;
1347
1348                 $id = self::escapeIdInternal( $id, $wgExternalInterwikiFragmentMode );
1349
1350                 return $id;
1351         }
1352
1353         /**
1354          * Helper for escapeIdFor*() functions. Performs most of the actual escaping.
1355          *
1356          * @param string $id String to escape
1357          * @param string $mode One of modes from $wgFragmentMode
1358          * @return string
1359          */
1360         private static function escapeIdInternal( $id, $mode ) {
1361                 switch ( $mode ) {
1362                         case 'html5':
1363                                 $id = str_replace( ' ', '_', $id );
1364                                 break;
1365                         case 'legacy':
1366                                 // This corresponds to 'noninitial' mode of the old escapeId()
1367                                 static $replace = [
1368                                         '%3A' => ':',
1369                                         '%' => '.'
1370                                 ];
1371
1372                                 $id = urlencode( str_replace( ' ', '_', $id ) );
1373                                 $id = strtr( $id, $replace );
1374                                 break;
1375                         default:
1376                                 throw new InvalidArgumentException( "Invalid mode '$mode' passed to '" . __METHOD__ );
1377                 }
1378
1379                 return $id;
1380         }
1381
1382         /**
1383          * Given a string containing a space delimited list of ids, escape each id
1384          * to match ids escaped by the escapeIdForAttribute() function.
1385          *
1386          * @since 1.27
1387          *
1388          * @param string $referenceString Space delimited list of ids
1389          * @return string
1390          */
1391         public static function escapeIdReferenceList( $referenceString ) {
1392                 # Explode the space delimited list string into an array of tokens
1393                 $references = preg_split( '/\s+/', "{$referenceString}", -1, PREG_SPLIT_NO_EMPTY );
1394
1395                 # Escape each token as an id
1396                 foreach ( $references as &$ref ) {
1397                         $ref = self::escapeIdForAttribute( $ref );
1398                 }
1399
1400                 # Merge the array back to a space delimited list string
1401                 # If the array is empty, the result will be an empty string ('')
1402                 $referenceString = implode( ' ', $references );
1403
1404                 return $referenceString;
1405         }
1406
1407         /**
1408          * Given a value, escape it so that it can be used as a CSS class and
1409          * return it.
1410          *
1411          * @todo For extra validity, input should be validated UTF-8.
1412          *
1413          * @see https://www.w3.org/TR/CSS21/syndata.html Valid characters/format
1414          *
1415          * @param string $class
1416          * @return string
1417          */
1418         static function escapeClass( $class ) {
1419                 // Convert ugly stuff to underscores and kill underscores in ugly places
1420                 return rtrim( preg_replace(
1421                         [ '/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/', '/_+/' ],
1422                         '_',
1423                         $class ), '_' );
1424         }
1425
1426         /**
1427          * Given HTML input, escape with htmlspecialchars but un-escape entities.
1428          * This allows (generally harmless) entities like &#160; to survive.
1429          *
1430          * @param string $html HTML to escape
1431          * @return string Escaped input
1432          */
1433         static function escapeHtmlAllowEntities( $html ) {
1434                 $html = self::decodeCharReferences( $html );
1435                 # It seems wise to escape ' as well as ", as a matter of course.  Can't
1436                 # hurt. Use ENT_SUBSTITUTE so that incorrectly truncated multibyte characters
1437                 # don't cause the entire string to disappear.
1438                 $html = htmlspecialchars( $html, ENT_QUOTES | ENT_SUBSTITUTE );
1439                 return $html;
1440         }
1441
1442         /**
1443          * Return an associative array of attribute names and values from
1444          * a partial tag string. Attribute names are forced to lowercase,
1445          * character references are decoded to UTF-8 text.
1446          *
1447          * @param string $text
1448          * @return array
1449          */
1450         public static function decodeTagAttributes( $text ) {
1451                 if ( trim( $text ) == '' ) {
1452                         return [];
1453                 }
1454
1455                 $pairs = [];
1456                 if ( !preg_match_all(
1457                         self::getAttribsRegex(),
1458                         $text,
1459                         $pairs,
1460                         PREG_SET_ORDER ) ) {
1461                         return [];
1462                 }
1463
1464                 $attribs = [];
1465                 foreach ( $pairs as $set ) {
1466                         $attribute = strtolower( $set[1] );
1467
1468                         // Filter attribute names with unacceptable characters
1469                         if ( !preg_match( self::getAttribNameRegex(), $attribute ) ) {
1470                                 continue;
1471                         }
1472
1473                         $value = self::getTagAttributeCallback( $set );
1474
1475                         // Normalize whitespace
1476                         $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
1477                         $value = trim( $value );
1478
1479                         // Decode character references
1480                         $attribs[$attribute] = self::decodeCharReferences( $value );
1481                 }
1482                 return $attribs;
1483         }
1484
1485         /**
1486          * Build a partial tag string from an associative array of attribute
1487          * names and values as returned by decodeTagAttributes.
1488          *
1489          * @param array $assoc_array
1490          * @return string
1491          */
1492         public static function safeEncodeTagAttributes( $assoc_array ) {
1493                 $attribs = [];
1494                 foreach ( $assoc_array as $attribute => $value ) {
1495                         $encAttribute = htmlspecialchars( $attribute );
1496                         $encValue = self::safeEncodeAttribute( $value );
1497
1498                         $attribs[] = "$encAttribute=\"$encValue\"";
1499                 }
1500                 return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
1501         }
1502
1503         /**
1504          * Pick the appropriate attribute value from a match set from the
1505          * attribs regex matches.
1506          *
1507          * @param array $set
1508          * @throws MWException When tag conditions are not met.
1509          * @return string
1510          */
1511         private static function getTagAttributeCallback( $set ) {
1512                 if ( isset( $set[5] ) ) {
1513                         # No quotes.
1514                         return $set[5];
1515                 } elseif ( isset( $set[4] ) ) {
1516                         # Single-quoted
1517                         return $set[4];
1518                 } elseif ( isset( $set[3] ) ) {
1519                         # Double-quoted
1520                         return $set[3];
1521                 } elseif ( !isset( $set[2] ) ) {
1522                         # In XHTML, attributes must have a value so return an empty string.
1523                         # See "Empty attribute syntax",
1524                         # https://www.w3.org/TR/html5/syntax.html#syntax-attribute-name
1525                         return "";
1526                 } else {
1527                         throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
1528                 }
1529         }
1530
1531         /**
1532          * @param string $text
1533          * @return string
1534          */
1535         private static function normalizeWhitespace( $text ) {
1536                 return trim( preg_replace(
1537                         '/(?:\r\n|[\x20\x0d\x0a\x09])+/',
1538                         ' ',
1539                         $text ) );
1540         }
1541
1542         /**
1543          * Normalizes whitespace in a section name, such as might be returned
1544          * by Parser::stripSectionName(), for use in the id's that are used for
1545          * section links.
1546          *
1547          * @param string $section
1548          * @return string
1549          */
1550         static function normalizeSectionNameWhitespace( $section ) {
1551                 return trim( preg_replace( '/[ _]+/', ' ', $section ) );
1552         }
1553
1554         /**
1555          * Ensure that any entities and character references are legal
1556          * for XML and XHTML specifically. Any stray bits will be
1557          * &amp;-escaped to result in a valid text fragment.
1558          *
1559          * a. named char refs can only be &lt; &gt; &amp; &quot;, others are
1560          *   numericized (this way we're well-formed even without a DTD)
1561          * b. any numeric char refs must be legal chars, not invalid or forbidden
1562          * c. use lower cased "&#x", not "&#X"
1563          * d. fix or reject non-valid attributes
1564          *
1565          * @param string $text
1566          * @return string
1567          * @private
1568          */
1569         static function normalizeCharReferences( $text ) {
1570                 return preg_replace_callback(
1571                         self::CHAR_REFS_REGEX,
1572                         [ self::class, 'normalizeCharReferencesCallback' ],
1573                         $text );
1574         }
1575
1576         /**
1577          * @param string $matches
1578          * @return string
1579          */
1580         static function normalizeCharReferencesCallback( $matches ) {
1581                 $ret = null;
1582                 if ( $matches[1] != '' ) {
1583                         $ret = self::normalizeEntity( $matches[1] );
1584                 } elseif ( $matches[2] != '' ) {
1585                         $ret = self::decCharReference( $matches[2] );
1586                 } elseif ( $matches[3] != '' ) {
1587                         $ret = self::hexCharReference( $matches[3] );
1588                 }
1589                 if ( is_null( $ret ) ) {
1590                         return htmlspecialchars( $matches[0] );
1591                 } else {
1592                         return $ret;
1593                 }
1594         }
1595
1596         /**
1597          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1598          * return the equivalent numeric entity reference (except for the core &lt;
1599          * &gt; &amp; &quot;). If the entity is a MediaWiki-specific alias, returns
1600          * the HTML equivalent. Otherwise, returns HTML-escaped text of
1601          * pseudo-entity source (eg &amp;foo;)
1602          *
1603          * @param string $name
1604          * @return string
1605          */
1606         static function normalizeEntity( $name ) {
1607                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1608                         return '&' . self::$htmlEntityAliases[$name] . ';';
1609                 } elseif ( in_array( $name, [ 'lt', 'gt', 'amp', 'quot' ] ) ) {
1610                         return "&$name;";
1611                 } elseif ( isset( self::$htmlEntities[$name] ) ) {
1612                         return '&#' . self::$htmlEntities[$name] . ';';
1613                 } else {
1614                         return "&amp;$name;";
1615                 }
1616         }
1617
1618         /**
1619          * @param int $codepoint
1620          * @return null|string
1621          */
1622         static function decCharReference( $codepoint ) {
1623                 $point = intval( $codepoint );
1624                 if ( self::validateCodepoint( $point ) ) {
1625                         return sprintf( '&#%d;', $point );
1626                 } else {
1627                         return null;
1628                 }
1629         }
1630
1631         /**
1632          * @param int $codepoint
1633          * @return null|string
1634          */
1635         static function hexCharReference( $codepoint ) {
1636                 $point = hexdec( $codepoint );
1637                 if ( self::validateCodepoint( $point ) ) {
1638                         return sprintf( '&#x%x;', $point );
1639                 } else {
1640                         return null;
1641                 }
1642         }
1643
1644         /**
1645          * Returns true if a given Unicode codepoint is a valid character in
1646          * both HTML5 and XML.
1647          * @param int $codepoint
1648          * @return bool
1649          */
1650         private static function validateCodepoint( $codepoint ) {
1651                 # U+000C is valid in HTML5 but not allowed in XML.
1652                 # U+000D is valid in XML but not allowed in HTML5.
1653                 # U+007F - U+009F are disallowed in HTML5 (control characters).
1654                 return $codepoint == 0x09
1655                         || $codepoint == 0x0a
1656                         || ( $codepoint >= 0x20 && $codepoint <= 0x7e )
1657                         || ( $codepoint >= 0xa0 && $codepoint <= 0xd7ff )
1658                         || ( $codepoint >= 0xe000 && $codepoint <= 0xfffd )
1659                         || ( $codepoint >= 0x10000 && $codepoint <= 0x10ffff );
1660         }
1661
1662         /**
1663          * Decode any character references, numeric or named entities,
1664          * in the text and return a UTF-8 string.
1665          *
1666          * @param string $text
1667          * @return string
1668          */
1669         public static function decodeCharReferences( $text ) {
1670                 return preg_replace_callback(
1671                         self::CHAR_REFS_REGEX,
1672                         [ self::class, 'decodeCharReferencesCallback' ],
1673                         $text );
1674         }
1675
1676         /**
1677          * Decode any character references, numeric or named entities,
1678          * in the next and normalize the resulting string. (T16952)
1679          *
1680          * This is useful for page titles, not for text to be displayed,
1681          * MediaWiki allows HTML entities to escape normalization as a feature.
1682          *
1683          * @param string $text Already normalized, containing entities
1684          * @return string Still normalized, without entities
1685          */
1686         public static function decodeCharReferencesAndNormalize( $text ) {
1687                 $text = preg_replace_callback(
1688                         self::CHAR_REFS_REGEX,
1689                         [ self::class, 'decodeCharReferencesCallback' ],
1690                         $text,
1691                         -1, //limit
1692                         $count
1693                 );
1694
1695                 if ( $count ) {
1696                         return MediaWikiServices::getInstance()->getContentLanguage()->normalize( $text );
1697                 } else {
1698                         return $text;
1699                 }
1700         }
1701
1702         /**
1703          * @param string $matches
1704          * @return string
1705          */
1706         static function decodeCharReferencesCallback( $matches ) {
1707                 if ( $matches[1] != '' ) {
1708                         return self::decodeEntity( $matches[1] );
1709                 } elseif ( $matches[2] != '' ) {
1710                         return self::decodeChar( intval( $matches[2] ) );
1711                 } elseif ( $matches[3] != '' ) {
1712                         return self::decodeChar( hexdec( $matches[3] ) );
1713                 }
1714                 # Last case should be an ampersand by itself
1715                 return $matches[0];
1716         }
1717
1718         /**
1719          * Return UTF-8 string for a codepoint if that is a valid
1720          * character reference, otherwise U+FFFD REPLACEMENT CHARACTER.
1721          * @param int $codepoint
1722          * @return string
1723          * @private
1724          */
1725         static function decodeChar( $codepoint ) {
1726                 if ( self::validateCodepoint( $codepoint ) ) {
1727                         return UtfNormal\Utils::codepointToUtf8( $codepoint );
1728                 } else {
1729                         return UtfNormal\Constants::UTF8_REPLACEMENT;
1730                 }
1731         }
1732
1733         /**
1734          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1735          * return the UTF-8 encoding of that character. Otherwise, returns
1736          * pseudo-entity source (eg "&foo;")
1737          *
1738          * @param string $name
1739          * @return string
1740          */
1741         static function decodeEntity( $name ) {
1742                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1743                         $name = self::$htmlEntityAliases[$name];
1744                 }
1745                 if ( isset( self::$htmlEntities[$name] ) ) {
1746                         return UtfNormal\Utils::codepointToUtf8( self::$htmlEntities[$name] );
1747                 } else {
1748                         return "&$name;";
1749                 }
1750         }
1751
1752         /**
1753          * Fetch the whitelist of acceptable attributes for a given element name.
1754          *
1755          * @param string $element
1756          * @return array A sequential array of acceptable attribute names
1757          * @deprecated since 1.34; should be private
1758          */
1759         static function attributeWhitelist( $element ) {
1760                 wfDeprecated( __METHOD__, '1.34' );
1761                 $list = self::setupAttributeWhitelist();
1762                 return $list[$element] ?? [];
1763         }
1764
1765         /**
1766          * Fetch the whitelist of acceptable attributes for a given element name.
1767          *
1768          * @param string $element
1769          * @return array An associative array where keys are acceptable attribute
1770          *   names
1771          */
1772         private static function attributeWhitelistInternal( $element ) {
1773                 $list = self::setupAttributeWhitelistInternal();
1774                 return $list[$element] ?? [];
1775         }
1776
1777         /**
1778          * Foreach array key (an allowed HTML element), return an array
1779          * of allowed attributes
1780          * @return array
1781          * @deprecated since 1.34; should be private
1782          */
1783         static function setupAttributeWhitelist() {
1784                 wfDeprecated( __METHOD__, '1.34' );
1785                 $wlist = self::setupAttributeWhitelistInternal();
1786                 // This method is expected to return a sequential array as the
1787                 // value for each HTML element key.
1788                 return array_map( function ( $v ) {
1789                         return array_keys( $v );
1790                 }, $wlist );
1791         }
1792
1793         /**
1794          * Foreach array key (an allowed HTML element), return an array
1795          * of allowed attributes
1796          * @return array An associative array: keys are HTML element names;
1797          *   values are associative arrays where the keys are allowed attribute
1798          *   names.
1799          */
1800         private static function setupAttributeWhitelistInternal() {
1801                 static $whitelist;
1802
1803                 if ( $whitelist !== null ) {
1804                         return $whitelist;
1805                 }
1806
1807                 // For lookup efficiency flip each attributes array so the keys are
1808                 // the valid attributes.
1809                 $merge = function ( $a, $b, $c = [] ) {
1810                         return array_merge( $a, array_flip( $b ), array_flip( $c ) );
1811                 };
1812                 $common = $merge( [], [
1813                         # HTML
1814                         'id',
1815                         'class',
1816                         'style',
1817                         'lang',
1818                         'dir',
1819                         'title',
1820
1821                         # WAI-ARIA
1822                         'aria-describedby',
1823                         'aria-flowto',
1824                         'aria-label',
1825                         'aria-labelledby',
1826                         'aria-owns',
1827                         'role',
1828
1829                         # RDFa
1830                         # These attributes are specified in section 9 of
1831                         # https://www.w3.org/TR/2008/REC-rdfa-syntax-20081014
1832                         'about',
1833                         'property',
1834                         'resource',
1835                         'datatype',
1836                         'typeof',
1837
1838                         # Microdata. These are specified by
1839                         # https://html.spec.whatwg.org/multipage/microdata.html#the-microdata-model
1840                         'itemid',
1841                         'itemprop',
1842                         'itemref',
1843                         'itemscope',
1844                         'itemtype',
1845                 ] );
1846
1847                 $block = $merge( $common, [ 'align' ] );
1848
1849                 $tablealign = [ 'align', 'valign' ];
1850                 $tablecell = [
1851                         'abbr',
1852                         'axis',
1853                         'headers',
1854                         'scope',
1855                         'rowspan',
1856                         'colspan',
1857                         'nowrap', # deprecated
1858                         'width', # deprecated
1859                         'height', # deprecated
1860                         'bgcolor', # deprecated
1861                 ];
1862
1863                 # Numbers refer to sections in HTML 4.01 standard describing the element.
1864                 # See: https://www.w3.org/TR/html4/
1865                 $whitelist = [
1866                         # 7.5.4
1867                         'div'        => $block,
1868                         'center'     => $common, # deprecated
1869                         'span'       => $common,
1870
1871                         # 7.5.5
1872                         'h1'         => $block,
1873                         'h2'         => $block,
1874                         'h3'         => $block,
1875                         'h4'         => $block,
1876                         'h5'         => $block,
1877                         'h6'         => $block,
1878
1879                         # 7.5.6
1880                         # address
1881
1882                         # 8.2.4
1883                         'bdo'        => $common,
1884
1885                         # 9.2.1
1886                         'em'         => $common,
1887                         'strong'     => $common,
1888                         'cite'       => $common,
1889                         'dfn'        => $common,
1890                         'code'       => $common,
1891                         'samp'       => $common,
1892                         'kbd'        => $common,
1893                         'var'        => $common,
1894                         'abbr'       => $common,
1895                         # acronym
1896
1897                         # 9.2.2
1898                         'blockquote' => $merge( $common, [ 'cite' ] ),
1899                         'q'          => $merge( $common, [ 'cite' ] ),
1900
1901                         # 9.2.3
1902                         'sub'        => $common,
1903                         'sup'        => $common,
1904
1905                         # 9.3.1
1906                         'p'          => $block,
1907
1908                         # 9.3.2
1909                         'br'         => $merge( $common, [ 'clear' ] ),
1910
1911                         # https://www.w3.org/TR/html5/text-level-semantics.html#the-wbr-element
1912                         'wbr'        => $common,
1913
1914                         # 9.3.4
1915                         'pre'        => $merge( $common, [ 'width' ] ),
1916
1917                         # 9.4
1918                         'ins'        => $merge( $common, [ 'cite', 'datetime' ] ),
1919                         'del'        => $merge( $common, [ 'cite', 'datetime' ] ),
1920
1921                         # 10.2
1922                         'ul'         => $merge( $common, [ 'type' ] ),
1923                         'ol'         => $merge( $common, [ 'type', 'start', 'reversed' ] ),
1924                         'li'         => $merge( $common, [ 'type', 'value' ] ),
1925
1926                         # 10.3
1927                         'dl'         => $common,
1928                         'dd'         => $common,
1929                         'dt'         => $common,
1930
1931                         # 11.2.1
1932                         'table'      => $merge( $common,
1933                                                                 [ 'summary', 'width', 'border', 'frame',
1934                                                                                 'rules', 'cellspacing', 'cellpadding',
1935                                                                                 'align', 'bgcolor',
1936                                                                 ] ),
1937
1938                         # 11.2.2
1939                         'caption'    => $block,
1940
1941                         # 11.2.3
1942                         'thead'      => $common,
1943                         'tfoot'      => $common,
1944                         'tbody'      => $common,
1945
1946                         # 11.2.4
1947                         'colgroup'   => $merge( $common, [ 'span' ] ),
1948                         'col'        => $merge( $common, [ 'span' ] ),
1949
1950                         # 11.2.5
1951                         'tr'         => $merge( $common, [ 'bgcolor' ], $tablealign ),
1952
1953                         # 11.2.6
1954                         'td'         => $merge( $common, $tablecell, $tablealign ),
1955                         'th'         => $merge( $common, $tablecell, $tablealign ),
1956
1957                         # 12.2
1958                         # NOTE: <a> is not allowed directly, but the attrib
1959                         # whitelist is used from the Parser object
1960                         'a'          => $merge( $common, [ 'href', 'rel', 'rev' ] ), # rel/rev esp. for RDFa
1961
1962                         # 13.2
1963                         # Not usually allowed, but may be used for extension-style hooks
1964                         # such as <math> when it is rasterized, or if $wgAllowImageTag is
1965                         # true
1966                         'img'        => $merge( $common, [ 'alt', 'src', 'width', 'height', 'srcset' ] ),
1967                         # Attributes for A/V tags added in T163583 / T133673
1968                         'audio'      => $merge( $common, [ 'controls', 'preload', 'width', 'height' ] ),
1969                         'video'      => $merge( $common, [ 'poster', 'controls', 'preload', 'width', 'height' ] ),
1970                         'source'     => $merge( $common, [ 'type', 'src' ] ),
1971                         'track'      => $merge( $common, [ 'type', 'src', 'srclang', 'kind', 'label' ] ),
1972
1973                         # 15.2.1
1974                         'tt'         => $common,
1975                         'b'          => $common,
1976                         'i'          => $common,
1977                         'big'        => $common,
1978                         'small'      => $common,
1979                         'strike'     => $common,
1980                         's'          => $common,
1981                         'u'          => $common,
1982
1983                         # 15.2.2
1984                         'font'       => $merge( $common, [ 'size', 'color', 'face' ] ),
1985                         # basefont
1986
1987                         # 15.3
1988                         'hr'         => $merge( $common, [ 'width' ] ),
1989
1990                         # HTML Ruby annotation text module, simple ruby only.
1991                         # https://www.w3.org/TR/html5/text-level-semantics.html#the-ruby-element
1992                         'ruby'       => $common,
1993                         # rbc
1994                         'rb'         => $common,
1995                         'rp'         => $common,
1996                         'rt'         => $common, # $merge( $common, [ 'rbspan' ] ),
1997                         'rtc'        => $common,
1998
1999                         # MathML root element, where used for extensions
2000                         # 'title' may not be 100% valid here; it's XHTML
2001                         # https://www.w3.org/TR/REC-MathML/
2002                         'math'       => $merge( [], [ 'class', 'style', 'id', 'title' ] ),
2003
2004                         // HTML 5 section 4.5
2005                         'figure'     => $common,
2006                         'figure-inline' => $common, # T118520
2007                         'figcaption' => $common,
2008
2009                         # HTML 5 section 4.6
2010                         'bdi' => $common,
2011
2012                         # HTML5 elements, defined by:
2013                         # https://html.spec.whatwg.org/multipage/semantics.html#the-data-element
2014                         'data' => $merge( $common, [ 'value' ] ),
2015                         'time' => $merge( $common, [ 'datetime' ] ),
2016                         'mark' => $common,
2017
2018                         // meta and link are only permitted by removeHTMLtags when Microdata
2019                         // is enabled so we don't bother adding a conditional to hide these
2020                         // Also meta and link are only valid in WikiText as Microdata elements
2021                         // (ie: validateTag rejects tags missing the attributes needed for Microdata)
2022                         // So we don't bother including $common attributes that have no purpose.
2023                         'meta' => $merge( [], [ 'itemprop', 'content' ] ),
2024                         'link' => $merge( [], [ 'itemprop', 'href', 'title' ] ),
2025                 ];
2026
2027                 return $whitelist;
2028         }
2029
2030         /**
2031          * Take a fragment of (potentially invalid) HTML and return
2032          * a version with any tags removed, encoded as plain text.
2033          *
2034          * Warning: this return value must be further escaped for literal
2035          * inclusion in HTML output as of 1.10!
2036          *
2037          * @param string $html HTML fragment
2038          * @return string
2039          */
2040         static function stripAllTags( $html ) {
2041                 // Use RemexHtml to tokenize $html and extract the text
2042                 $handler = new RemexStripTagHandler;
2043                 $tokenizer = new RemexHtml\Tokenizer\Tokenizer( $handler, $html, [
2044                         'ignoreErrors' => true,
2045                         // don't ignore char refs, we want them to be decoded
2046                         'ignoreNulls' => true,
2047                         'skipPreprocess' => true,
2048                 ] );
2049                 $tokenizer->execute();
2050                 $text = $handler->getResult();
2051
2052                 $text = self::normalizeWhitespace( $text );
2053                 return $text;
2054         }
2055
2056         /**
2057          * Hack up a private DOCTYPE with HTML's standard entity declarations.
2058          * PHP 4 seemed to know these if you gave it an HTML doctype, but
2059          * PHP 5.1 doesn't.
2060          *
2061          * Use for passing XHTML fragments to PHP's XML parsing functions
2062          *
2063          * @return string
2064          */
2065         static function hackDocType() {
2066                 $out = "<!DOCTYPE html [\n";
2067                 foreach ( self::$htmlEntities as $entity => $codepoint ) {
2068                         $out .= "<!ENTITY $entity \"&#$codepoint;\">";
2069                 }
2070                 $out .= "]>\n";
2071                 return $out;
2072         }
2073
2074         /**
2075          * @param string $url
2076          * @return mixed|string
2077          */
2078         static function cleanUrl( $url ) {
2079                 # Normalize any HTML entities in input. They will be
2080                 # re-escaped by makeExternalLink().
2081                 $url = self::decodeCharReferences( $url );
2082
2083                 # Escape any control characters introduced by the above step
2084                 $url = preg_replace_callback( '/[\][<>"\\x00-\\x20\\x7F\|]/',
2085                         [ __CLASS__, 'cleanUrlCallback' ], $url );
2086
2087                 # Validate hostname portion
2088                 $matches = [];
2089                 if ( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
2090                         list( /* $whole */, $protocol, $host, $rest ) = $matches;
2091
2092                         // Characters that will be ignored in IDNs.
2093                         // https://tools.ietf.org/html/rfc3454#section-3.1
2094                         // Strip them before further processing so blacklists and such work.
2095                         $strip = "/
2096                                 \\s|          # general whitespace
2097                                 \xc2\xad|     # 00ad SOFT HYPHEN
2098                                 \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
2099                                 \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
2100                                 \xe2\x81\xa0| # 2060 WORD JOINER
2101                                 \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
2102                                 \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
2103                                 \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
2104                                 \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
2105                                 \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
2106                                 \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
2107                                 \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
2108                                 [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe0f VARIATION SELECTOR-1-16
2109                                 /xuD";
2110
2111                         $host = preg_replace( $strip, '', $host );
2112
2113                         // IPv6 host names are bracketed with [].  Url-decode these.
2114                         if ( substr_compare( "//%5B", $host, 0, 5 ) === 0 &&
2115                                 preg_match( '!^//%5B([0-9A-Fa-f:.]+)%5D((:\d+)?)$!', $host, $matches )
2116                         ) {
2117                                 $host = '//[' . $matches[1] . ']' . $matches[2];
2118                         }
2119
2120                         // @todo FIXME: Validate hostnames here
2121
2122                         return $protocol . $host . $rest;
2123                 } else {
2124                         return $url;
2125                 }
2126         }
2127
2128         /**
2129          * @param array $matches
2130          * @return string
2131          */
2132         static function cleanUrlCallback( $matches ) {
2133                 return urlencode( $matches[0] );
2134         }
2135
2136         /**
2137          * Does a string look like an e-mail address?
2138          *
2139          * This validates an email address using an HTML5 specification found at:
2140          * http://www.whatwg.org/html/states-of-the-type-attribute.html#valid-e-mail-address
2141          * Which as of 2011-01-24 says:
2142          *
2143          *   A valid e-mail address is a string that matches the ABNF production
2144          *   1*( atext / "." ) "@" ldh-str *( "." ldh-str ) where atext is defined
2145          *   in RFC 5322 section 3.2.3, and ldh-str is defined in RFC 1034 section
2146          *   3.5.
2147          *
2148          * This function is an implementation of the specification as requested in
2149          * T24449.
2150          *
2151          * Client-side forms will use the same standard validation rules via JS or
2152          * HTML 5 validation; additional restrictions can be enforced server-side
2153          * by extensions via the 'isValidEmailAddr' hook.
2154          *
2155          * Note that this validation doesn't 100% match RFC 2822, but is believed
2156          * to be liberal enough for wide use. Some invalid addresses will still
2157          * pass validation here.
2158          *
2159          * @since 1.18
2160          *
2161          * @param string $addr E-mail address
2162          * @return bool
2163          */
2164         public static function validateEmail( $addr ) {
2165                 $result = null;
2166                 if ( !Hooks::run( 'isValidEmailAddr', [ $addr, &$result ] ) ) {
2167                         return $result;
2168                 }
2169
2170                 // Please note strings below are enclosed in brackets [], this make the
2171                 // hyphen "-" a range indicator. Hence it is double backslashed below.
2172                 // See T28948
2173                 $rfc5322_atext = "a-z0-9!#$%&'*+\\-\/=?^_`{|}~";
2174                 $rfc1034_ldh_str = "a-z0-9\\-";
2175
2176                 $html5_email_regexp = "/
2177                 ^                      # start of string
2178                 [$rfc5322_atext\\.]+    # user part which is liberal :p
2179                 @                      # 'apostrophe'
2180                 [$rfc1034_ldh_str]+       # First domain part
2181                 (\\.[$rfc1034_ldh_str]+)*  # Following part prefixed with a dot
2182                 $                      # End of string
2183                 /ix"; // case Insensitive, eXtended
2184
2185                 return (bool)preg_match( $html5_email_regexp, $addr );
2186         }
2187 }