includes/utils/StringUtils.php

   1 <?php
   2 /**
   3  * Methods to play with strings.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 /**
  24  * A collection of static methods to play with strings.
  25  */
  26 class StringUtils {
  27
  28         /**
  29          * Test whether a string is valid UTF-8.
  30          *
  31          * The function check for invalid byte sequences, overlong encoding but
  32          * not for different normalisations.
  33          *
  34          * This relies internally on the mbstring function mb_check_encoding()
  35          * hardcoded to check against UTF-8. Whenever the function is not available
  36          * we fallback to a pure PHP implementation. Setting $disableMbstring to
  37          * true will skip the use of mb_check_encoding, this is mostly intended for
  38          * unit testing our internal implementation.
  39          *
  40          * @since 1.21
  41          * @note In MediaWiki 1.21, this function did not provide proper UTF-8 validation.
  42          * In particular, the pure PHP code path did not in fact check for overlong forms.
  43          * Beware of this when backporting code to that version of MediaWiki.
  44          *
  45          * @param string $value String to check
  46          * @param boolean $disableMbstring Whether to use the pure PHP
  47          * implementation instead of trying mb_check_encoding. Intended for unit
  48          * testing. Default: false
  49          *
  50          * @return boolean Whether the given $value is a valid UTF-8 encoded string
  51          */
  52         static function isUtf8( $value, $disableMbstring = false ) {
  53                 $value = (string)$value;
  54
  55                 // If the mbstring extension is loaded, use it. However, before PHP 5.4, values above
  56                 // U+10FFFF are incorrectly allowed, so we have to check for them separately.
  57                 if ( !$disableMbstring && function_exists( 'mb_check_encoding' ) ) {
  58                         static $newPHP;
  59                         if ( $newPHP === null ) {
  60                                 $newPHP = !mb_check_encoding( "\xf4\x90\x80\x80", 'UTF-8' );
  61                         }
  62
  63                         return mb_check_encoding( $value, 'UTF-8' ) &&
  64                                 ( $newPHP || preg_match( "/\xf4[\x90-\xbf]|[\xf5-\xff]/S", $value ) === 0 );
  65                 }
  66
  67                 if ( preg_match( "/[\x80-\xff]/S", $value ) === 0 ) {
  68                         // String contains only ASCII characters, has to be valid
  69                         return true;
  70                 }
  71
  72                 // PCRE implements repetition using recursion; to avoid a stack overflow (and segfault)
  73                 // for large input, we check for invalid sequences (<= 5 bytes) rather than valid
  74                 // sequences, which can be as long as the input string is. Multiple short regexes are
  75                 // used rather than a single long regex for performance.
  76                 static $regexes;
  77                 if ( $regexes === null ) {
  78                         $cont = "[\x80-\xbf]";
  79                         $after = "(?!$cont)"; // "(?:[^\x80-\xbf]|$)" would work here
  80                         $regexes = array(
  81                                 // Continuation byte at the start
  82                                 "/^$cont/",
  83
  84                                 // ASCII byte followed by a continuation byte
  85                                 "/[\\x00-\x7f]$cont/S",
  86
  87                                 // Illegal byte
  88                                 "/[\xc0\xc1\xf5-\xff]/S",
  89
  90                                 // Invalid 2-byte sequence, or valid one then an extra continuation byte
  91                                 "/[\xc2-\xdf](?!$cont$after)/S",
  92
  93                                 // Invalid 3-byte sequence, or valid one then an extra continuation byte
  94                                 "/\xe0(?![\xa0-\xbf]$cont$after)/",
  95                                 "/[\xe1-\xec\xee\xef](?!$cont{2}$after)/S",
  96                                 "/\xed(?![\x80-\x9f]$cont$after)/",
  97
  98                                 // Invalid 4-byte sequence, or valid one then an extra continuation byte
  99                                 "/\xf0(?![\x90-\xbf]$cont{2}$after)/",
 100                                 "/[\xf1-\xf3](?!$cont{3}$after)/S",
 101                                 "/\xf4(?![\x80-\x8f]$cont{2}$after)/",
 102                         );
 103                 }
 104
 105                 foreach ( $regexes as $regex ) {
 106                         if ( preg_match( $regex, $value ) !== 0 ) {
 107                                 return false;
 108                         }
 109                 }
 110                 return true;
 111         }
 112
 113         /**
 114          * Perform an operation equivalent to
 115          *
 116          *     preg_replace( "!$startDelim(.*?)$endDelim!", $replace, $subject );
 117          *
 118          * except that it's worst-case O(N) instead of O(N^2)
 119          *
 120          * Compared to delimiterReplace(), this implementation is fast but memory-
 121          * hungry and inflexible. The memory requirements are such that I don't
 122          * recommend using it on anything but guaranteed small chunks of text.
 123          *
 124          * @param $startDelim
 125          * @param $endDelim
 126          * @param $replace
 127          * @param $subject
 128          *
 129          * @return string
 130          */
 131         static function hungryDelimiterReplace( $startDelim, $endDelim, $replace, $subject ) {
 132                 $segments = explode( $startDelim, $subject );
 133                 $output = array_shift( $segments );
 134                 foreach ( $segments as $s ) {
 135                         $endDelimPos = strpos( $s, $endDelim );
 136                         if ( $endDelimPos === false ) {
 137                                 $output .= $startDelim . $s;
 138                         } else {
 139                                 $output .= $replace . substr( $s, $endDelimPos + strlen( $endDelim ) );
 140                         }
 141                 }
 142                 return $output;
 143         }
 144
 145         /**
 146          * Perform an operation equivalent to
 147          *
 148          *   preg_replace_callback( "!$startDelim(.*)$endDelim!s$flags", $callback, $subject )
 149          *
 150          * This implementation is slower than hungryDelimiterReplace but uses far less
 151          * memory. The delimiters are literal strings, not regular expressions.
 152          *
 153          * If the start delimiter ends with an initial substring of the end delimiter,
 154          * e.g. in the case of C-style comments, the behavior differs from the model
 155          * regex. In this implementation, the end must share no characters with the
 156          * start, so e.g. /*\/ is not considered to be both the start and end of a
 157          * comment. /*\/xy/*\/ is considered to be a single comment with contents /xy/.
 158          *
 159          * @param string $startDelim start delimiter
 160          * @param string $endDelim end delimiter
 161          * @param $callback Callback: function to call on each match
 162          * @param $subject String
 163          * @param string $flags regular expression flags
 164          * @throws MWException
 165          * @return string
 166          */
 167         static function delimiterReplaceCallback( $startDelim, $endDelim, $callback, $subject, $flags = '' ) {
 168                 $inputPos = 0;
 169                 $outputPos = 0;
 170                 $output = '';
 171                 $foundStart = false;
 172                 $encStart = preg_quote( $startDelim, '!' );
 173                 $encEnd = preg_quote( $endDelim, '!' );
 174                 $strcmp = strpos( $flags, 'i' ) === false ? 'strcmp' : 'strcasecmp';
 175                 $endLength = strlen( $endDelim );
 176                 $m = array();
 177
 178                 while ( $inputPos < strlen( $subject ) &&
 179                         preg_match( "!($encStart)|($encEnd)!S$flags", $subject, $m, PREG_OFFSET_CAPTURE, $inputPos ) )
 180                 {
 181                         $tokenOffset = $m[0][1];
 182                         if ( $m[1][0] != '' ) {
 183                                 if ( $foundStart &&
 184                                         $strcmp( $endDelim, substr( $subject, $tokenOffset, $endLength ) ) == 0 )
 185                                 {
 186                                         # An end match is present at the same location
 187                                         $tokenType = 'end';
 188                                         $tokenLength = $endLength;
 189                                 } else {
 190                                         $tokenType = 'start';
 191                                         $tokenLength = strlen( $m[0][0] );
 192                                 }
 193                         } elseif ( $m[2][0] != '' ) {
 194                                 $tokenType = 'end';
 195                                 $tokenLength = strlen( $m[0][0] );
 196                         } else {
 197                                 throw new MWException( 'Invalid delimiter given to ' . __METHOD__ );
 198                         }
 199
 200                         if ( $tokenType == 'start' ) {
 201                                 # Only move the start position if we haven't already found a start
 202                                 # This means that START START END matches outer pair
 203                                 if ( !$foundStart ) {
 204                                         # Found start
 205                                         $inputPos = $tokenOffset + $tokenLength;
 206                                         # Write out the non-matching section
 207                                         $output .= substr( $subject, $outputPos, $tokenOffset - $outputPos );
 208                                         $outputPos = $tokenOffset;
 209                                         $contentPos = $inputPos;
 210                                         $foundStart = true;
 211                                 } else {
 212                                         # Move the input position past the *first character* of START,
 213                                         # to protect against missing END when it overlaps with START
 214                                         $inputPos = $tokenOffset + 1;
 215                                 }
 216                         } elseif ( $tokenType == 'end' ) {
 217                                 if ( $foundStart ) {
 218                                         # Found match
 219                                         $output .= call_user_func( $callback, array(
 220                                                 substr( $subject, $outputPos, $tokenOffset + $tokenLength - $outputPos ),
 221                                                 substr( $subject, $contentPos, $tokenOffset - $contentPos )
 222                                         ));
 223                                         $foundStart = false;
 224                                 } else {
 225                                         # Non-matching end, write it out
 226                                         $output .= substr( $subject, $inputPos, $tokenOffset + $tokenLength - $outputPos );
 227                                 }
 228                                 $inputPos = $outputPos = $tokenOffset + $tokenLength;
 229                         } else {
 230                                 throw new MWException( 'Invalid delimiter given to ' . __METHOD__ );
 231                         }
 232                 }
 233                 if ( $outputPos < strlen( $subject ) ) {
 234                         $output .= substr( $subject, $outputPos );
 235                 }
 236                 return $output;
 237         }
 238
 239         /**
 240          * Perform an operation equivalent to
 241          *
 242          *   preg_replace( "!$startDelim(.*)$endDelim!$flags", $replace, $subject )
 243          *
 244          * @param string $startDelim start delimiter regular expression
 245          * @param string $endDelim end delimiter regular expression
 246          * @param string $replace replacement string. May contain $1, which will be
 247          *                 replaced by the text between the delimiters
 248          * @param string $subject to search
 249          * @param string $flags regular expression flags
 250          * @return String: The string with the matches replaced
 251          */
 252         static function delimiterReplace( $startDelim, $endDelim, $replace, $subject, $flags = '' ) {
 253                 $replacer = new RegexlikeReplacer( $replace );
 254                 return self::delimiterReplaceCallback( $startDelim, $endDelim,
 255                         $replacer->cb(), $subject, $flags );
 256         }
 257
 258         /**
 259          * More or less "markup-safe" explode()
 260          * Ignores any instances of the separator inside <...>
 261          * @param string $separator
 262          * @param string $text
 263          * @return array
 264          */
 265         static function explodeMarkup( $separator, $text ) {
 266                 $placeholder = "\x00";
 267
 268                 // Remove placeholder instances
 269                 $text = str_replace( $placeholder, '', $text );
 270
 271                 // Replace instances of the separator inside HTML-like tags with the placeholder
 272                 $replacer = new DoubleReplacer( $separator, $placeholder );
 273                 $cleaned = StringUtils::delimiterReplaceCallback( '<', '>', $replacer->cb(), $text );
 274
 275                 // Explode, then put the replaced separators back in
 276                 $items = explode( $separator, $cleaned );
 277                 foreach ( $items as $i => $str ) {
 278                         $items[$i] = str_replace( $placeholder, $separator, $str );
 279                 }
 280
 281                 return $items;
 282         }
 283
 284         /**
 285          * Escape a string to make it suitable for inclusion in a preg_replace()
 286          * replacement parameter.
 287          *
 288          * @param string $string
 289          * @return string
 290          */
 291         static function escapeRegexReplacement( $string ) {
 292                 $string = str_replace( '\\', '\\\\', $string );
 293                 $string = str_replace( '$', '\\$', $string );
 294                 return $string;
 295         }
 296
 297         /**
 298          * Workalike for explode() with limited memory usage.
 299          * Returns an Iterator
 300          * @param string $separator
 301          * @param string $subject
 302          * @return ArrayIterator|ExplodeIterator
 303          */
 304         static function explode( $separator, $subject ) {
 305                 if ( substr_count( $subject, $separator ) > 1000 ) {
 306                         return new ExplodeIterator( $separator, $subject );
 307                 } else {
 308                         return new ArrayIterator( explode( $separator, $subject ) );
 309                 }
 310         }
 311 }
 312
 313 /**
 314  * Base class for "replacers", objects used in preg_replace_callback() and
 315  * StringUtils::delimiterReplaceCallback()
 316  */
 317 class Replacer {
 318
 319         /**
 320          * @return array
 321          */
 322         function cb() {
 323                 return array( &$this, 'replace' );
 324         }
 325 }
 326
 327 /**
 328  * Class to replace regex matches with a string similar to that used in preg_replace()
 329  */
 330 class RegexlikeReplacer extends Replacer {
 331         var $r;
 332
 333         /**
 334          * @param string $r
 335          */
 336         function __construct( $r ) {
 337                 $this->r = $r;
 338         }
 339
 340         /**
 341          * @param array $matches
 342          * @return string
 343          */
 344         function replace( $matches ) {
 345                 $pairs = array();
 346                 foreach ( $matches as $i => $match ) {
 347                         $pairs["\$$i"] = $match;
 348                 }
 349                 return strtr( $this->r, $pairs );
 350         }
 351
 352 }
 353
 354 /**
 355  * Class to perform secondary replacement within each replacement string
 356  */
 357 class DoubleReplacer extends Replacer {
 358
 359         /**
 360          * @param $from
 361          * @param $to
 362          * @param int $index
 363          */
 364         function __construct( $from, $to, $index = 0 ) {
 365                 $this->from = $from;
 366                 $this->to = $to;
 367                 $this->index = $index;
 368         }
 369
 370         /**
 371          * @param array $matches
 372          * @return mixed
 373          */
 374         function replace( $matches ) {
 375                 return str_replace( $this->from, $this->to, $matches[$this->index] );
 376         }
 377 }
 378
 379 /**
 380  * Class to perform replacement based on a simple hashtable lookup
 381  */
 382 class HashtableReplacer extends Replacer {
 383         var $table, $index;
 384
 385         /**
 386          * @param $table
 387          * @param int $index
 388          */
 389         function __construct( $table, $index = 0 ) {
 390                 $this->table = $table;
 391                 $this->index = $index;
 392         }
 393
 394         /**
 395          * @param array $matches
 396          * @return mixed
 397          */
 398         function replace( $matches ) {
 399                 return $this->table[$matches[$this->index]];
 400         }
 401 }
 402
 403 /**
 404  * Replacement array for FSS with fallback to strtr()
 405  * Supports lazy initialisation of FSS resource
 406  */
 407 class ReplacementArray {
 408         /*mostly private*/ var $data = false;
 409         /*mostly private*/ var $fss = false;
 410
 411         /**
 412          * Create an object with the specified replacement array
 413          * The array should have the same form as the replacement array for strtr()
 414          * @param array $data
 415          */
 416         function __construct( $data = array() ) {
 417                 $this->data = $data;
 418         }
 419
 420         /**
 421          * @return array
 422          */
 423         function __sleep() {
 424                 return array( 'data' );
 425         }
 426
 427         function __wakeup() {
 428                 $this->fss = false;
 429         }
 430
 431         /**
 432          * Set the whole replacement array at once
 433          * @param array $data
 434          */
 435         function setArray( $data ) {
 436                 $this->data = $data;
 437                 $this->fss = false;
 438         }
 439
 440         /**
 441          * @return array|bool
 442          */
 443         function getArray() {
 444                 return $this->data;
 445         }
 446
 447         /**
 448          * Set an element of the replacement array
 449          * @param string $from
 450          * @param string $to
 451          */
 452         function setPair( $from, $to ) {
 453                 $this->data[$from] = $to;
 454                 $this->fss = false;
 455         }
 456
 457         /**
 458          * @param array $data
 459          */
 460         function mergeArray( $data ) {
 461                 $this->data = array_merge( $this->data, $data );
 462                 $this->fss = false;
 463         }
 464
 465         /**
 466          * @param ReplacementArray $other
 467          */
 468         function merge( $other ) {
 469                 $this->data = array_merge( $this->data, $other->data );
 470                 $this->fss = false;
 471         }
 472
 473         /**
 474          * @param string $from
 475          */
 476         function removePair( $from ) {
 477                 unset( $this->data[$from] );
 478                 $this->fss = false;
 479         }
 480
 481         /**
 482          * @param array $data
 483          */
 484         function removeArray( $data ) {
 485                 foreach ( $data as $from => $to ) {
 486                         $this->removePair( $from );
 487                 }
 488                 $this->fss = false;
 489         }
 490
 491         /**
 492          * @param string $subject
 493          * @return string
 494          */
 495         function replace( $subject ) {
 496                 if ( function_exists( 'fss_prep_replace' ) ) {
 497                         wfProfileIn( __METHOD__ . '-fss' );
 498                         if ( $this->fss === false ) {
 499                                 $this->fss = fss_prep_replace( $this->data );
 500                         }
 501                         $result = fss_exec_replace( $this->fss, $subject );
 502                         wfProfileOut( __METHOD__ . '-fss' );
 503                 } else {
 504                         wfProfileIn( __METHOD__ . '-strtr' );
 505                         $result = strtr( $subject, $this->data );
 506                         wfProfileOut( __METHOD__ . '-strtr' );
 507                 }
 508                 return $result;
 509         }
 510 }
 511
 512 /**
 513  * An iterator which works exactly like:
 514  *
 515  * foreach ( explode( $delim, $s ) as $element ) {
 516  *    ...
 517  * }
 518  *
 519  * Except it doesn't use 193 byte per element
 520  */
 521 class ExplodeIterator implements Iterator {
 522         // The subject string
 523         var $subject, $subjectLength;
 524
 525         // The delimiter
 526         var $delim, $delimLength;
 527
 528         // The position of the start of the line
 529         var $curPos;
 530
 531         // The position after the end of the next delimiter
 532         var $endPos;
 533
 534         // The current token
 535         var $current;
 536
 537         /**
 538          * Construct a DelimIterator
 539          * @param string $delim
 540          * @param string $subject
 541          */
 542         function __construct( $delim, $subject ) {
 543                 $this->subject = $subject;
 544                 $this->delim = $delim;
 545
 546                 // Micro-optimisation (theoretical)
 547                 $this->subjectLength = strlen( $subject );
 548                 $this->delimLength = strlen( $delim );
 549
 550                 $this->rewind();
 551         }
 552
 553         function rewind() {
 554                 $this->curPos = 0;
 555                 $this->endPos = strpos( $this->subject, $this->delim );
 556                 $this->refreshCurrent();
 557         }
 558
 559         function refreshCurrent() {
 560                 if ( $this->curPos === false ) {
 561                         $this->current = false;
 562                 } elseif ( $this->curPos >= $this->subjectLength ) {
 563                         $this->current = '';
 564                 } elseif ( $this->endPos === false ) {
 565                         $this->current = substr( $this->subject, $this->curPos );
 566                 } else {
 567                         $this->current = substr( $this->subject, $this->curPos, $this->endPos - $this->curPos );
 568                 }
 569         }
 570
 571         function current() {
 572                 return $this->current;
 573         }
 574
 575         /**
 576          * @return int|bool Current position or boolean false if invalid
 577          */
 578         function key() {
 579                 return $this->curPos;
 580         }
 581
 582         /**
 583          * @return string
 584          */
 585         function next() {
 586                 if ( $this->endPos === false ) {
 587                         $this->curPos = false;
 588                 } else {
 589                         $this->curPos = $this->endPos + $this->delimLength;
 590                         if ( $this->curPos >= $this->subjectLength ) {
 591                                 $this->endPos = false;
 592                         } else {
 593                                 $this->endPos = strpos( $this->subject, $this->delim, $this->curPos );
 594                         }
 595                 }
 596                 $this->refreshCurrent();
 597                 return $this->current;
 598         }
 599
 600         /**
 601          * @return bool
 602          */
 603         function valid() {
 604                 return $this->curPos !== false;
 605         }
 606 }