includes/normal/CleanUpTest.php

   1 <?php
   2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   3 # http://www.mediawiki.org/
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 # http://www.gnu.org/copyleft/gpl.html
  19
  20 /**
  21  * Additional tests for UtfNormal::cleanUp() function, inclusion
  22  * regression checks for known problems.
  23  *
  24  * Requires PHPUnit.
  25  *
  26  * @package UtfNormal
  27  * @access private
  28  */
  29
  30 if( php_sapi_name() != 'cli' ) {
  31         die( "Run me from the command line please.\n" );
  32 }
  33
  34 /** */
  35 if( isset( $_SERVER['argv'] ) && in_array( '--icu', $_SERVER['argv'] ) ) {
  36         dl( 'php_utfnormal.so' );
  37 }
  38
  39 #ini_set( 'memory_limit', '40M' );
  40
  41 require_once( 'PHPUnit.php' );
  42 require_once( 'UtfNormal.php' );
  43
  44 /**
  45  * @package UtfNormal
  46  */
  47 class CleanUpTest extends PHPUnit_TestCase {
  48         /**
  49          * @param string $name ???
  50          */
  51         function CleanUpTest( $name ) {
  52                 $this->PHPUnit_TestCase( $name );
  53         }
  54
  55         /** @todo document */
  56         function setUp() {
  57         }
  58
  59         /** @todo document */
  60         function tearDown() {
  61         }
  62
  63         /** @todo document */
  64         function testAscii() {
  65                 $text = 'This is plain ASCII text.';
  66                 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
  67         }
  68
  69         /** @todo document */
  70         function testNull() {
  71                 $text = "a \x00 null";
  72                 $expect = "a \xef\xbf\xbd null";
  73                 $this->assertEquals(
  74                         bin2hex( $expect ),
  75                         bin2hex( UtfNormal::cleanUp( $text ) ) );
  76         }
  77
  78         /** @todo document */
  79         function testLatin() {
  80                 $text = "L'\xc3\xa9cole";
  81                 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
  82         }
  83
  84         /** @todo document */
  85         function testLatinNormal() {
  86                 $text = "L'e\xcc\x81cole";
  87                 $expect = "L'\xc3\xa9cole";
  88                 $this->assertEquals( $expect, UtfNormal::cleanUp( $text ) );
  89         }
  90
  91         /**
  92          * This test is *very* expensive!
  93          * @todo document
  94          */
  95         function XtestAllChars() {
  96                 $rep = UTF8_REPLACEMENT;
  97                 global $utfCanonicalComp, $utfCanonicalDecomp;
  98                 for( $i = 0x0; $i < UNICODE_MAX; $i++ ) {
  99                         $char = codepointToUtf8( $i );
 100                         $clean = UtfNormal::cleanUp( $char );
 101                         $x = sprintf( "%04X", $i );
 102                         if( $i % 0x1000 == 0 ) echo "U+$x\n";
 103                         if( $i == 0x0009 ||
 104                             $i == 0x000a ||
 105                             $i == 0x000d ||
 106                             ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST) ||
 107                             ($i > UNICODE_SURROGATE_LAST && $i < 0xfffe ) ||
 108                             ($i > 0xffff && $i <= UNICODE_MAX ) ) {
 109                                 if( isset( $utfCanonicalComp[$char] ) || isset( $utfCanonicalDecomp[$char] ) ) {
 110                                     $comp = UtfNormal::NFC( $char );
 111                                         $this->assertEquals(
 112                                                 bin2hex( $comp ),
 113                                                 bin2hex( $clean ),
 114                                                 "U+$x should be decomposed" );
 115                                 } else {
 116                                         $this->assertEquals(
 117                                                 bin2hex( $char ),
 118                                                 bin2hex( $clean ),
 119                                                 "U+$x should be intact" );
 120                                 }
 121                         } else {
 122                                 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
 123                         }
 124                 }
 125         }
 126
 127         /** @todo document */
 128         function testAllBytes() {
 129                 $this->doTestBytes( '', '' );
 130                 $this->doTestBytes( 'x', '' );
 131                 $this->doTestBytes( '', 'x' );
 132                 $this->doTestBytes( 'x', 'x' );
 133         }
 134
 135         /** @todo document */
 136         function doTestBytes( $head, $tail ) {
 137                 for( $i = 0x0; $i < 256; $i++ ) {
 138                         $char = $head . chr( $i ) . $tail;
 139                         $clean = UtfNormal::cleanUp( $char );
 140                         $x = sprintf( "%02X", $i );
 141                         if( $i == 0x0009 ||
 142                             $i == 0x000a ||
 143                             $i == 0x000d ||
 144                             ($i > 0x001f && $i < 0x80) ) {
 145                                 $this->assertEquals(
 146                                         bin2hex( $char ),
 147                                         bin2hex( $clean ),
 148                                         "ASCII byte $x should be intact" );
 149                                 if( $char != $clean ) return;
 150                         } else {
 151                                 $norm = $head . UTF8_REPLACEMENT . $tail;
 152                                 $this->assertEquals(
 153                                         bin2hex( $norm ),
 154                                         bin2hex( $clean ),
 155                                         "Forbidden byte $x should be rejected" );
 156                                 if( $norm != $clean ) return;
 157                         }
 158                 }
 159         }
 160
 161         /** @todo document */
 162         function testDoubleBytes() {
 163                 $this->doTestDoubleBytes( '', '' );
 164                 $this->doTestDoubleBytes( 'x', '' );
 165                 $this->doTestDoubleBytes( '', 'x' );
 166                 $this->doTestDoubleBytes( 'x', 'x' );
 167         }
 168
 169         /**
 170          * @todo document
 171          */
 172         function doTestDoubleBytes( $head, $tail ) {
 173                 for( $first = 0xc0; $first < 0x100; $first++ ) {
 174                         for( $second = 0x80; $second < 0x100; $second++ ) {
 175                                 $char = $head . chr( $first ) . chr( $second ) . $tail;
 176                                 $clean = UtfNormal::cleanUp( $char );
 177                                 $x = sprintf( "%02X,%02X", $first, $second );
 178                                 if( $first > 0xc1 &&
 179                                     $first < 0xe0 &&
 180                                     $second < 0xc0 ) {
 181                                     $norm = UtfNormal::NFC( $char );
 182                                         $this->assertEquals(
 183                                                 bin2hex( $norm ),
 184                                                 bin2hex( $clean ),
 185                                                 "Pair $x should be intact" );
 186                                     if( $norm != $clean ) return;
 187                                 } elseif( $first > 0xfd || $second > 0xbf ) {
 188                                         # fe and ff are not legal head bytes -- expect two replacement chars
 189                                         $norm = $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail;
 190                                         $this->assertEquals(
 191                                                 bin2hex( $norm ),
 192                                                 bin2hex( $clean ),
 193                                                 "Forbidden pair $x should be rejected" );
 194                                         if( $norm != $clean ) return;
 195                                 } else {
 196                                         $norm = $head . UTF8_REPLACEMENT . $tail;
 197                                         $this->assertEquals(
 198                                                 bin2hex( $norm ),
 199                                                 bin2hex( $clean ),
 200                                                 "Forbidden pair $x should be rejected" );
 201                                         if( $norm != $clean ) return;
 202                                 }
 203                         }
 204                 }
 205         }
 206
 207         /** @todo document */
 208         function testTripleBytes() {
 209                 $this->doTestTripleBytes( '', '' );
 210                 $this->doTestTripleBytes( 'x', '' );
 211                 $this->doTestTripleBytes( '', 'x' );
 212                 $this->doTestTripleBytes( 'x', 'x' );
 213         }
 214
 215         /** @todo document */
 216         function doTestTripleBytes( $head, $tail ) {
 217                 for( $first = 0xc0; $first < 0x100; $first++ ) {
 218                         for( $second = 0x80; $second < 0x100; $second++ ) {
 219                                 #for( $third = 0x80; $third < 0x100; $third++ ) {
 220                                 for( $third = 0x80; $third < 0x81; $third++ ) {
 221                                         $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
 222                                         $clean = UtfNormal::cleanUp( $char );
 223                                         $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
 224                                         if( $first >= 0xe0 &&
 225                                                 $first < 0xf0 &&
 226                                                 $second < 0xc0 &&
 227                                                 $third < 0xc0 ) {
 228                                                 if( $first == 0xe0 && $second < 0xa0 ) {
 229                                                         $this->assertEquals(
 230                                                                 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 231                                                                 bin2hex( $clean ),
 232                                                                 "Overlong triplet $x should be rejected" );
 233                                                 } elseif( $first == 0xed &&
 234                                                         ( chr( $first ) . chr( $second ) . chr( $third ))  >= UTF8_SURROGATE_FIRST ) {
 235                                                         $this->assertEquals(
 236                                                                 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 237                                                                 bin2hex( $clean ),
 238                                                                 "Surrogate triplet $x should be rejected" );
 239                                                 } else {
 240                                                         $this->assertEquals(
 241                                                                 bin2hex( UtfNormal::NFC( $char ) ),
 242                                                                 bin2hex( $clean ),
 243                                                                 "Triplet $x should be intact" );
 244                                                 }
 245                                         } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
 246                                                 $this->assertEquals(
 247                                                         bin2hex( UtfNormal::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT . $tail ),
 248                                                         bin2hex( $clean ),
 249                                                         "Valid 2-byte $x + broken tail" );
 250                                         } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
 251                                                 $this->assertEquals(
 252                                                         bin2hex( $head . UTF8_REPLACEMENT . UtfNormal::NFC( chr( $second ) . chr( $third ) . $tail ) ),
 253                                                         bin2hex( $clean ),
 254                                                         "Broken head + valid 2-byte $x" );
 255                                         } elseif( ( $first > 0xfd || $second > 0xfd ) &&
 256                                                     ( ( $second > 0xbf && $third > 0xbf ) ||
 257                                                       ( $second < 0xc0 && $third < 0xc0 ) ||
 258                                                       ( $second > 0xfd ) ||
 259                                                       ( $third > 0xfd ) ) ) {
 260                                                 # fe and ff are not legal head bytes -- expect three replacement chars
 261                                                 $this->assertEquals(
 262                                                         bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
 263                                                         bin2hex( $clean ),
 264                                                         "Forbidden triplet $x should be rejected" );
 265                                         } elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
 266                                                 $this->assertEquals(
 267                                                         bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 268                                                         bin2hex( $clean ),
 269                                                         "Forbidden triplet $x should be rejected" );
 270                                         } else {
 271                                                 $this->assertEquals(
 272                                                         bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
 273                                                         bin2hex( $clean ),
 274                                                         "Forbidden triplet $x should be rejected" );
 275                                         }
 276                                 }
 277                         }
 278                 }
 279         }
 280
 281         /** @todo document */
 282         function testChunkRegression() {
 283                 # Check for regression against a chunking bug
 284                 $text   = "\x46\x55\xb8" .
 285                           "\xdc\x96" .
 286                           "\xee" .
 287                           "\xe7" .
 288                           "\x44" .
 289                           "\xaa" .
 290                           "\x2f\x25";
 291                 $expect = "\x46\x55\xef\xbf\xbd" .
 292                           "\xdc\x96" .
 293                           "\xef\xbf\xbd" .
 294                           "\xef\xbf\xbd" .
 295                           "\x44" .
 296                           "\xef\xbf\xbd" .
 297                           "\x2f\x25";
 298
 299                 $this->assertEquals(
 300                         bin2hex( $expect ),
 301                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 302         }
 303
 304         /** @todo document */
 305         function testInterposeRegression() {
 306                 $text   = "\x4e\x30" .
 307                           "\xb1" .              # bad tail
 308                           "\x3a" .
 309                           "\x92" .              # bad tail
 310                           "\x62\x3a" .
 311                           "\x84" .              # bad tail
 312                           "\x43" .
 313                           "\xc6" .              # bad head
 314                           "\x3f" .
 315                           "\x92" .              # bad tail
 316                           "\xad" .              # bad tail
 317                           "\x7d" .
 318                           "\xd9\x95";
 319
 320                 $expect = "\x4e\x30" .
 321                           "\xef\xbf\xbd" .
 322                           "\x3a" .
 323                           "\xef\xbf\xbd" .
 324                           "\x62\x3a" .
 325                           "\xef\xbf\xbd" .
 326                           "\x43" .
 327                           "\xef\xbf\xbd" .
 328                           "\x3f" .
 329                           "\xef\xbf\xbd" .
 330                           "\xef\xbf\xbd" .
 331                           "\x7d" .
 332                           "\xd9\x95";
 333
 334                 $this->assertEquals(
 335                         bin2hex( $expect ),
 336                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 337         }
 338
 339         /** @todo document */
 340         function testOverlongRegression() {
 341                 $text   = "\x67" .
 342                           "\x1a" . # forbidden ascii
 343                           "\xea" . # bad head
 344                           "\xc1\xa6" . # overlong sequence
 345                           "\xad" . # bad tail
 346                           "\x1c" . # forbidden ascii
 347                           "\xb0" . # bad tail
 348                           "\x3c" .
 349                           "\x9e";  # bad tail
 350                 $expect = "\x67" .
 351                           "\xef\xbf\xbd" .
 352                           "\xef\xbf\xbd" .
 353                           "\xef\xbf\xbd" .
 354                           "\xef\xbf\xbd" .
 355                           "\xef\xbf\xbd" .
 356                           "\xef\xbf\xbd" .
 357                           "\x3c" .
 358                           "\xef\xbf\xbd";
 359                 $this->assertEquals(
 360                         bin2hex( $expect ),
 361                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 362         }
 363
 364         /** @todo document */
 365         function testSurrogateRegression() {
 366                 $text   = "\xed\xb4\x96" . # surrogate 0xDD16
 367                           "\x83" . # bad tail
 368                           "\xb4" . # bad tail
 369                           "\xac";  # bad head
 370                 $expect = "\xef\xbf\xbd" .
 371                           "\xef\xbf\xbd" .
 372                           "\xef\xbf\xbd" .
 373                           "\xef\xbf\xbd";
 374                 $this->assertEquals(
 375                         bin2hex( $expect ),
 376                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 377         }
 378
 379         /** @todo document */
 380         function testBomRegression() {
 381                 $text   = "\xef\xbf\xbe" . # U+FFFE, illegal char
 382                           "\xb2" . # bad tail
 383                           "\xef" . # bad head
 384                           "\x59";
 385                 $expect = "\xef\xbf\xbd" .
 386                           "\xef\xbf\xbd" .
 387                           "\xef\xbf\xbd" .
 388                           "\x59";
 389                 $this->assertEquals(
 390                         bin2hex( $expect ),
 391                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 392         }
 393
 394         /** @todo document */
 395         function testForbiddenRegression() {
 396                 $text   = "\xef\xbf\xbf"; # U+FFFF, illegal char
 397                 $expect = "\xef\xbf\xbd";
 398                 $this->assertEquals(
 399                         bin2hex( $expect ),
 400                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 401         }
 402
 403         /** @todo document */
 404         function testHangulRegression() {
 405                 $text = "\xed\x9c\xaf" . # Hangul char
 406                                 "\xe1\x87\x81";  # followed by another final jamo
 407                 $expect = $text;         # Should *not* change.
 408                 $this->assertEquals(
 409                         bin2hex( $expect ),
 410                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 411         }
 412 }
 413
 414
 415 $suite =& new PHPUnit_TestSuite( 'CleanUpTest' );
 416 $result = PHPUnit::run( $suite );
 417 echo $result->toString();
 418
 419 if( !$result->wasSuccessful() ) {
 420         exit( -1 );
 421 }
 422 exit( 0 );
 423 ?>