X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2FStringUtils.php;h=54a85dc0ba518349380a613a935646bd39636283;hb=75ef257c2986ffc9ddab85800367932b7fdb1fa1;hp=70d0bff1d3ab37d0752722070eeb51a56a24fadf;hpb=2dedbbdfa10f2e043a9e3d71f6b9828dd0468502;p=lhc%2Fweb%2Fwiklou.git

diff --git a/includes/StringUtils.php b/includes/StringUtils.php
index 70d0bff1d3..54a85dc0ba 100644
--- a/includes/StringUtils.php
+++ b/includes/StringUtils.php
@@ -1,8 +1,74 @@
 <?php
+/**
+ * Methods to play with strings.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
 /**
  * A collection of static methods to play with strings.
  */
 class StringUtils {
+
+	/**
+	 * Test whether a string is valid UTF-8.
+	 *
+	 * The function check for invalid byte sequences, overlong encoding but
+	 * not for different normalisations.
+	 *
+	 * This relies internally on the mbstring function mb_check_encoding()
+	 * hardcoded to check against UTF-8. Whenever the function is not available
+	 * we fallback to a pure PHP implementation. Setting $disableMbstring to
+	 * true will skip the use of mb_check_encoding, this is mostly intended for
+	 * unit testing our internal implementation.
+	 *
+	 * @since 1.21
+	 *
+	 * @param string $value String to check
+	 * @param boolean $disableMbstring Whether to use the pure PHP
+	 * implementation instead of trying mb_check_encoding. Intended for unit
+	 * testing. Default: false
+	 *
+	 * @return boolean Whether the given $value is a valid UTF-8 encoded string
+	 */
+	static function isUtf8( $value, $disableMbstring = false ) {
+
+		if ( preg_match( '/[\x80-\xff]/', $value ) === 0 ) {
+			# no high bit set, this is pure ASCII which is defacto
+			# valid UTF-8
+			return true;
+		}
+
+		if ( !$disableMbstring && function_exists( 'mb_check_encoding' ) ) {
+			return mb_check_encoding( $value, 'UTF-8' );
+		} else {
+			$hasUtf8 = preg_match( '/^(?>
+				  [\x00-\x7f]
+				| [\xc0-\xdf][\x80-\xbf]
+				| [\xe0-\xef][\x80-\xbf]{2}
+				| [\xf0-\xf7][\x80-\xbf]{3}
+				| [\xf8-\xfb][\x80-\xbf]{4}
+				| \xfc[\x84-\xbf][\x80-\xbf]{4}
+			)+$/x', $value );
+			return ($hasUtf8 > 0 );
+		}
+	}
+
 	/**
 	 * Perform an operation equivalent to
 	 *
@@ -13,6 +79,13 @@ class StringUtils {
 	 * Compared to delimiterReplace(), this implementation is fast but memory-
 	 * hungry and inflexible. The memory requirements are such that I don't
 	 * recommend using it on anything but guaranteed small chunks of text.
+	 *
+	 * @param $startDelim
+	 * @param $endDelim
+	 * @param $replace
+	 * @param $subject
+	 *
+	 * @return string
 	 */
 	static function hungryDelimiterReplace( $startDelim, $endDelim, $replace, $subject ) {
 		$segments = explode( $startDelim, $subject );
@@ -36,13 +109,20 @@ class StringUtils {
 	 * This implementation is slower than hungryDelimiterReplace but uses far less
 	 * memory. The delimiters are literal strings, not regular expressions.
 	 *
-	 * @param string $flags Regular expression flags
+	 * If the start delimiter ends with an initial substring of the end delimiter,
+	 * e.g. in the case of C-style comments, the behaviour differs from the model
+	 * regex. In this implementation, the end must share no characters with the
+	 * start, so e.g. /*\/ is not considered to be both the start and end of a
+	 * comment. /*\/xy/*\/ is considered to be a single comment with contents /xy/.
+	 *
+	 * @param $startDelim String: start delimiter
+	 * @param $endDelim String: end delimiter
+	 * @param $callback Callback: function to call on each match
+	 * @param $subject String
+	 * @param $flags String: regular expression flags
+	 * @throws MWException
+	 * @return string
 	 */
-	# If the start delimiter ends with an initial substring of the end delimiter,
-	# e.g. in the case of C-style comments, the behaviour differs from the model
-	# regex. In this implementation, the end must share no characters with the
-	# start, so e.g. /*/ is not considered to be both the start and end of a
-	# comment. /*/xy/*/ is considered to be a single comment with contents /xy/.
 	static function delimiterReplaceCallback( $startDelim, $endDelim, $callback, $subject, $flags = '' ) {
 		$inputPos = 0;
 		$outputPos = 0;
@@ -77,16 +157,20 @@ class StringUtils {
 			}
 
 			if ( $tokenType == 'start' ) {
-				$inputPos = $tokenOffset + $tokenLength;
 				# Only move the start position if we haven't already found a start
 				# This means that START START END matches outer pair
 				if ( !$foundStart ) {
 					# Found start
+					$inputPos = $tokenOffset + $tokenLength;
 					# Write out the non-matching section
 					$output .= substr( $subject, $outputPos, $tokenOffset - $outputPos );
 					$outputPos = $tokenOffset;
 					$contentPos = $inputPos;
 					$foundStart = true;
+				} else {
+					# Move the input position past the *first character* of START,
+					# to protect against missing END when it overlaps with START
+					$inputPos = $tokenOffset + 1;
 				}
 			} elseif ( $tokenType == 'end' ) {
 				if ( $foundStart ) {
@@ -111,17 +195,18 @@ class StringUtils {
 		return $output;
 	}
 
-	/*
+	/**
 	 * Perform an operation equivalent to
 	 *
 	 *   preg_replace( "!$startDelim(.*)$endDelim!$flags", $replace, $subject )
 	 *
-	 * @param string $startDelim Start delimiter regular expression
-	 * @param string $endDelim End delimiter regular expression
-	 * @param string $replace Replacement string. May contain $1, which will be
-	 *               replaced by the text between the delimiters
-	 * @param string $subject String to search
-	 * @return string The string with the matches replaced
+	 * @param $startDelim String: start delimiter regular expression
+	 * @param $endDelim String: end delimiter regular expression
+	 * @param $replace String: replacement string. May contain $1, which will be
+	 *                 replaced by the text between the delimiters
+	 * @param $subject String to search
+	 * @param $flags String: regular expression flags
+	 * @return String: The string with the matches replaced
 	 */
 	static function delimiterReplace( $startDelim, $endDelim, $replace, $subject, $flags = '' ) {
 		$replacer = new RegexlikeReplacer( $replace );
@@ -132,8 +217,8 @@ class StringUtils {
 	/**
 	 * More or less "markup-safe" explode()
 	 * Ignores any instances of the separator inside <...>
-	 * @param string $separator
-	 * @param string $text
+	 * @param $separator String
+	 * @param $text String
 	 * @return array
 	 */
 	static function explodeMarkup( $separator, $text ) {
@@ -159,14 +244,29 @@ class StringUtils {
 	 * Escape a string to make it suitable for inclusion in a preg_replace()
 	 * replacement parameter.
 	 *
-	 * @param string $string
-	 * @return string
+	 * @param $string String
+	 * @return String
 	 */
 	static function escapeRegexReplacement( $string ) {
 		$string = str_replace( '\\', '\\\\', $string );
 		$string = str_replace( '$', '\\$', $string );
 		return $string;
 	}
+
+	/**
+	 * Workalike for explode() with limited memory usage.
+	 * Returns an Iterator
+	 * @param $separator
+	 * @param $subject
+	 * @return ArrayIterator|ExplodeIterator
+	 */
+	static function explode( $separator, $subject ) {
+		if ( substr_count( $subject, $separator ) > 1000 ) {
+			return new ExplodeIterator( $separator, $subject );
+		} else {
+			return new ArrayIterator( explode( $separator, $subject ) );
+		}
+	}
 }
 
 /**
@@ -174,6 +274,10 @@ class StringUtils {
  * StringUtils::delimiterReplaceCallback()
  */
 class Replacer {
+
+	/**
+	 * @return array
+	 */
 	function cb() {
 		return array( &$this, 'replace' );
 	}
@@ -184,10 +288,18 @@ class Replacer {
  */
 class RegexlikeReplacer extends Replacer {
 	var $r;
+
+	/**
+	 * @param $r string
+	 */
 	function __construct( $r ) {
 		$this->r = $r;
 	}
 
+	/**
+	 * @param $matches array
+	 * @return string
+	 */
 	function replace( $matches ) {
 		$pairs = array();
 		foreach ( $matches as $i => $match ) {
@@ -202,12 +314,22 @@ class RegexlikeReplacer extends Replacer {
  * Class to perform secondary replacement within each replacement string
  */
 class DoubleReplacer extends Replacer {
+
+	/**
+	 * @param $from
+	 * @param $to
+	 * @param $index int
+	 */
 	function __construct( $from, $to, $index = 0 ) {
 		$this->from = $from;
 		$this->to = $to;
 		$this->index = $index;
 	}
 
+	/**
+	 * @param $matches array
+	 * @return mixed
+	 */
 	function replace( $matches ) {
 		return str_replace( $this->from, $this->to, $matches[$this->index] );
 	}
@@ -219,11 +341,19 @@ class DoubleReplacer extends Replacer {
 class HashtableReplacer extends Replacer {
 	var $table, $index;
 
+	/**
+	 * @param $table
+	 * @param $index int
+	 */
 	function __construct( $table, $index = 0 ) {
 		$this->table = $table;
 		$this->index = $index;
 	}
 
+	/**
+	 * @param $matches array
+	 * @return mixed
+	 */
 	function replace( $matches ) {
 		return $this->table[$matches[$this->index]];
 	}
@@ -240,11 +370,15 @@ class ReplacementArray {
 	/**
 	 * Create an object with the specified replacement array
 	 * The array should have the same form as the replacement array for strtr()
+	 * @param array $data
 	 */
 	function __construct( $data = array() ) {
 		$this->data = $data;
 	}
 
+	/**
+	 * @return array
+	 */
 	function __sleep() {
 		return array( 'data' );
 	}
@@ -261,39 +395,61 @@ class ReplacementArray {
 		$this->fss = false;
 	}
 
+	/**
+	 * @return array|bool
+	 */
 	function getArray() {
 		return $this->data;
 	}
 
 	/**
 	 * Set an element of the replacement array
+	 * @param $from string
+	 * @param $to stromg
 	 */
 	function setPair( $from, $to ) {
 		$this->data[$from] = $to;
 		$this->fss = false;
 	}
 
+	/**
+	 * @param $data array
+	 */
 	function mergeArray( $data ) {
 		$this->data = array_merge( $this->data, $data );
 		$this->fss = false;
 	}
 
+	/**
+	 * @param $other
+	 */
 	function merge( $other ) {
 		$this->data = array_merge( $this->data, $other->data );
 		$this->fss = false;
 	}
 
+	/**
+	 * @param $from string
+	 */
 	function removePair( $from ) {
 		unset($this->data[$from]);
 		$this->fss = false;
 	}
 
+	/**
+	 * @param $data array
+	 */
 	function removeArray( $data ) {
-		foreach( $data as $from => $to )
+		foreach( $data as $from => $to ) {
 			$this->removePair( $from );
+		}
 		$this->fss = false;
 	}
 
+	/**
+	 * @param $subject string
+	 * @return string
+	 */
 	function replace( $subject ) {
 		if ( function_exists( 'fss_prep_replace' ) ) {
 			wfProfileIn( __METHOD__.'-fss' );
@@ -310,3 +466,96 @@ class ReplacementArray {
 		return $result;
 	}
 }
+
+/**
+ * An iterator which works exactly like:
+ *
+ * foreach ( explode( $delim, $s ) as $element ) {
+ *    ...
+ * }
+ *
+ * Except it doesn't use 193 byte per element
+ */
+class ExplodeIterator implements Iterator {
+	// The subject string
+	var $subject, $subjectLength;
+
+	// The delimiter
+	var $delim, $delimLength;
+
+	// The position of the start of the line
+	var $curPos;
+
+	// The position after the end of the next delimiter
+	var $endPos;
+
+	// The current token
+	var $current;
+
+	/**
+	 * Construct a DelimIterator
+	 * @param $delim string
+	 * @param $s string
+	 */
+	function __construct( $delim, $s ) {
+		$this->subject = $s;
+		$this->delim = $delim;
+
+		// Micro-optimisation (theoretical)
+		$this->subjectLength = strlen( $s );
+		$this->delimLength = strlen( $delim );
+
+		$this->rewind();
+	}
+
+	function rewind() {
+		$this->curPos = 0;
+		$this->endPos = strpos( $this->subject, $this->delim );
+		$this->refreshCurrent();
+	}
+
+	function refreshCurrent() {
+		if ( $this->curPos === false ) {
+			$this->current = false;
+		} elseif ( $this->curPos >= $this->subjectLength ) {
+			$this->current = '';
+		} elseif ( $this->endPos === false ) {
+			$this->current = substr( $this->subject, $this->curPos );
+		} else {
+			$this->current = substr( $this->subject, $this->curPos, $this->endPos - $this->curPos );
+		}
+	}
+
+	function current() {
+		return $this->current;
+	}
+
+	function key() {
+		return $this->curPos;
+	}
+
+	/**
+	 * @return string
+	 */
+	function next() {
+		if ( $this->endPos === false ) {
+			$this->curPos = false;
+		} else {
+			$this->curPos = $this->endPos + $this->delimLength;
+			if ( $this->curPos >= $this->subjectLength ) {
+				$this->endPos = false;
+			} else {
+				$this->endPos = strpos( $this->subject, $this->delim, $this->curPos );
+			}
+		}
+		$this->refreshCurrent();
+		return $this->current;
+	}
+
+	/**
+	 * @return bool
+	 */
+	function valid() {
+		return $this->curPos !== false;
+	}
+}