4 * Other tests for the unicode normalization module
10 if( php_sapi_name() != 'cli' ) {
11 die( "Run me from the command line please.\n" );
14 // From http://unicode.org/Public/UNIDATA/NormalizationTest.txt
15 $file = "NormalizationTest.txt";
18 $f = fopen($file, "r");
21 * The following section will be used for testing different normalization methods.
26 * - php_utfnormal.so or intl extension: both are wrappers around
27 libicu so we list the version of libicu when making the
30 * - libicu Ubuntu 3.8.1-3ubuntu1.1 php 5.2.6-3ubuntu4.5
31 ~ 2200 assertion errors
33 ~ output: http://paste2.org/p/921566
35 * - libicu Ubuntu 4.2.1-3 php 5.3.2-1ubuntu4.2
36 ~ 1384 assertion errors
38 ~ output: http://paste2.org/p/921435
40 * - libicu Debian 4.4.1-5 php 5.3.2-1ubuntu4.2
44 * - Tests comparing pure PHP output with libicu output were added
45 later and slow down the runtime.
48 require_once("./UtfNormal.php");
49 function normalize_form_c($c) { return UtfNormal
::toNFC($c); }
50 function normalize_form_d($c) { return UtfNormal
::toNFD($c); }
51 function normalize_form_kc($c) { return UtfNormal
::toNFKC($c); }
52 function normalize_form_kd($c) { return UtfNormal
::toNFKD($c); }
55 * This set of functions is only useful if youve added a param to the
56 * following functions to force pure PHP usage. I decided not to
57 * commit that code since might produce a slowdown in the UTF
58 * normalization code just for the sake of these tests. -- hexmode
60 function normalize_form_c_php($c) { return UtfNormal
::toNFC($c, "php"); }
61 function normalize_form_d_php($c) { return UtfNormal
::toNFD($c, "php"); }
62 function normalize_form_kc_php($c) { return UtfNormal
::toNFKC($c, "php"); }
63 function normalize_form_kd_php($c) { return UtfNormal
::toNFKD($c, "php"); }
65 assert_options(ASSERT_ACTIVE
, 1);
66 assert_options(ASSERT_WARNING
, 0);
67 assert_options(ASSERT_QUIET_EVAL
, 1);
68 assert_options(ASSERT_CALLBACK
, 'my_assert');
70 function my_assert( $file, $line, $code ) {
72 echo "Assertion that '$code' failed on line $lineNo ($col[5])\n";
78 while( ( $col = getRow( $f ) ) !== false ) {
81 if(count($col) == 6) {
83 if( $count %
100 === 0 ) echo "Count: $count\n";
88 # verify that the pure PHP version is correct
89 $NFCc1 = normalize_form_c($col[0]);
90 $NFCc1p = normalize_form_c_php($col[0]);
91 assert('$NFCc1 === $NFCc1p');
92 $NFCc2 = normalize_form_c($col[1]);
93 $NFCc2p = normalize_form_c_php($col[1]);
94 assert('$NFCc2 === $NFCc2p');
95 $NFCc3 = normalize_form_c($col[2]);
96 $NFCc3p = normalize_form_c_php($col[2]);
97 assert('$NFCc3 === $NFCc3p');
98 $NFCc4 = normalize_form_c($col[3]);
99 $NFCc4p = normalize_form_c_php($col[3]);
100 assert('$NFCc4 === $NFCc4p');
101 $NFCc5 = normalize_form_c($col[4]);
102 $NFCc5p = normalize_form_c_php($col[4]);
103 assert('$NFCc5 === $NFCc5p');
105 $NFDc1 = normalize_form_d($col[0]);
106 $NFDc1p = normalize_form_d_php($col[0]);
107 assert('$NFDc1 === $NFDc1p');
108 $NFDc2 = normalize_form_d($col[1]);
109 $NFDc2p = normalize_form_d_php($col[1]);
110 assert('$NFDc2 === $NFDc2p');
111 $NFDc3 = normalize_form_d($col[2]);
112 $NFDc3p = normalize_form_d_php($col[2]);
113 assert('$NFDc3 === $NFDc3p');
114 $NFDc4 = normalize_form_d($col[3]);
115 $NFDc4p = normalize_form_d_php($col[3]);
116 assert('$NFDc4 === $NFDc4p');
117 $NFDc5 = normalize_form_d($col[4]);
118 $NFDc5p = normalize_form_d_php($col[4]);
119 assert('$NFDc5 === $NFDc5p');
121 $NFKDc1 = normalize_form_kd($col[0]);
122 $NFKDc1p = normalize_form_kd_php($col[0]);
123 assert('$NFKDc1 === $NFKDc1p');
124 $NFKDc2 = normalize_form_kd($col[1]);
125 $NFKDc2p = normalize_form_kd_php($col[1]);
126 assert('$NFKDc2 === $NFKDc2p');
127 $NFKDc3 = normalize_form_kd($col[2]);
128 $NFKDc3p = normalize_form_kd_php($col[2]);
129 assert('$NFKDc3 === $NFKDc3p');
130 $NFKDc4 = normalize_form_kd($col[3]);
131 $NFKDc4p = normalize_form_kd_php($col[3]);
132 assert('$NFKDc4 === $NFKDc4p');
133 $NFKDc5 = normalize_form_kd($col[4]);
134 $NFKDc5p = normalize_form_kd_php($col[4]);
135 assert('$NFKDc5 === $NFKDc5p');
137 $NFKCc1 = normalize_form_kc($col[0]);
138 $NFKCc1p = normalize_form_kc_php($col[0]);
139 assert('$NFKCc1 === $NFKCc1p');
140 $NFKCc2 = normalize_form_kc($col[1]);
141 $NFKCc2p = normalize_form_kc_php($col[1]);
142 assert('$NFKCc2 === $NFKCc2p');
143 $NFKCc3 = normalize_form_kc($col[2]);
144 $NFKCc3p = normalize_form_kc_php($col[2]);
145 assert('$NFKCc3 === $NFKCc3p');
146 $NFKCc4 = normalize_form_kc($col[3]);
147 $NFKCc4p = normalize_form_kc_php($col[3]);
148 assert('$NFKCc4 === $NFKCc4p');
149 $NFKCc5 = normalize_form_kc($col[4]);
150 $NFKCc5p = normalize_form_kc_php($col[4]);
151 assert('$NFKCc5 === $NFKCc5p');
153 # c2 == NFC(c1) == NFC(c2) == NFC(c3)
154 assert('$col[1] === $NFCc1');
155 assert('$col[1] === $NFCc2');
156 assert('$col[1] === $NFCc3');
158 # c4 == NFC(c4) == NFC(c5)
159 assert('$col[3] === $NFCc4');
160 assert('$col[3] === $NFCc5');
162 # c3 == NFD(c1) == NFD(c2) == NFD(c3)
163 assert('$col[2] === $NFDc1');
164 assert('$col[2] === $NFDc2');
165 assert('$col[2] === $NFDc3');
167 # c5 == NFD(c4) == NFD(c5)
168 assert('$col[4] === $NFDc4');
169 assert('$col[4] === $NFDc5');
171 # c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
172 assert('$col[3] === $NFKCc1');
173 assert('$col[3] === $NFKCc2');
174 assert('$col[3] === $NFKCc3');
175 assert('$col[3] === $NFKCc4');
176 assert('$col[3] === $NFKCc5');
178 # c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
179 assert('$col[4] === $NFKDc1');
180 assert('$col[4] === $NFKDc2');
181 assert('$col[4] === $NFKDc3');
182 assert('$col[4] === $NFKDc4');
183 assert('$col[4] === $NFKDc5');
188 // Compare against http://en.wikipedia.org/wiki/UTF-8#Description
189 function unichr($c) {
192 } else if ($c <= 0x7FF) {
193 return chr(0xC0 |
$c >> 6) . chr(0x80 |
$c & 0x3F);
194 } else if ($c <= 0xFFFF) {
195 return chr(0xE0 |
$c >> 12) . chr(0x80 |
$c >> 6 & 0x3F)
196 . chr(0x80 |
$c & 0x3F);
197 } else if ($c <= 0x10FFFF) {
198 return chr(0xF0 |
$c >> 18) . chr(0x80 |
$c >> 12 & 0x3F)
199 . chr(0x80 |
$c >> 6 & 0x3F)
200 . chr(0x80 |
$c & 0x3F);
206 function unistr($c) {
207 return implode("", array_map("unichr", array_map("hexdec", explode(" ", $c))));
210 function getRow( $f ) {
211 global $comment, $sep;
214 if( $row === false ) return false;
216 $pos = strpos( $row, $comment );
217 $pos2 = strpos( $row, ")" );
218 if( $pos === 0 ) return array($row);
222 if($pos2) $c = substr( $row, $pos2 +
2 );
223 else $c = substr( $row, $pos );
224 $row = substr( $row, 0, $pos );
228 foreach(explode( $sep, $row ) as $ent) {
229 if(trim($ent) !== "") {
230 $ret[] = unistr($ent);