includes/normal/UtfNormalGenerate.php

   1 <?php
   2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   3 # http://www.mediawiki.org/
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 # http://www.gnu.org/copyleft/gpl.html
  19
  20 /**
  21  * This script generates UniNormalData.inc from the Unicode Character Database
  22  * and supplementary files.
  23  *
  24  * @package UtfNormal
  25  * @access private
  26  */
  27
  28 /** */
  29
  30 if( php_sapi_name() != 'cli' ) {
  31         die( "Run me from the command line please.\n" );
  32 }
  33
  34 require_once 'UtfNormalUtil.php';
  35
  36 $in = fopen("DerivedNormalizationProps.txt", "rt" );
  37 if( !$in ) {
  38         print "Can't open DerivedNormalizationProps.txt for reading.\n";
  39         print "If necessary, fetch this file from the internet:\n";
  40         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  41         exit(-1);
  42 }
  43 print "Initializing normalization quick check tables...\n";
  44 $checkNFC = array();
  45 while( false !== ($line = fgets( $in ) ) ) {
  46         if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
  47                 list( $junk, $first, $last, $prop, $value ) = $matches;
  48                 #print "$first $last $prop $value\n";
  49                 if( !$last ) $last = $first;
  50                 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
  51                         $char = codepointToUtf8( $i );
  52                         $checkNFC[$char] = $value;
  53                 }
  54         }
  55 }
  56 fclose( $in );
  57
  58 $in = fopen("CompositionExclusions.txt", "rt" );
  59 if( !$in ) {
  60         print "Can't open CompositionExclusions.txt for reading.\n";
  61         print "If necessary, fetch this file from the internet:\n";
  62         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  63         exit(-1);
  64 }
  65 $exclude = array();
  66 while( false !== ($line = fgets( $in ) ) ) {
  67         if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
  68                 $codepoint = $matches[1];
  69                 $source = codepointToUtf8( hexdec( $codepoint ) );
  70                 $exclude[$source] = true;
  71         }
  72 }
  73 fclose($in);
  74
  75 $in = fopen("UnicodeData.txt", "rt" );
  76 if( !$in ) {
  77         print "Can't open UnicodeData.txt for reading.\n";
  78         print "If necessary, fetch this file from the internet:\n";
  79         print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
  80         exit(-1);
  81 }
  82
  83 $compatibilityDecomp = array();
  84 $canonicalDecomp = array();
  85 $canonicalComp = array();
  86 $combiningClass = array();
  87 $total = 0;
  88 $compat = 0;
  89 $canon = 0;
  90
  91 print "Reading character definitions...\n";
  92 while( false !== ($line = fgets( $in ) ) ) {
  93         $columns = split(';', $line);
  94         $codepoint = $columns[0];
  95         $name = $columns[1];
  96         $canonicalCombiningClass = $columns[3];
  97         $decompositionMapping = $columns[5];
  98
  99         $source = codepointToUtf8( hexdec( $codepoint ) );
 100
 101         if( $canonicalCombiningClass != 0 ) {
 102                 $combiningClass[$source] = intval( $canonicalCombiningClass );
 103         }
 104
 105         if( $decompositionMapping === '' ) continue;
 106         if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
 107                 # Compatibility decomposition
 108                 $canonical = false;
 109                 $decompositionMapping = $matches[2];
 110                 $compat++;
 111         } else {
 112                 $canonical = true;
 113                 $canon++;
 114         }
 115         $total++;
 116         $dest = hexSequenceToUtf8( $decompositionMapping );
 117
 118         $compatibilityDecomp[$source] = $dest;
 119         if( $canonical ) {
 120                 $canonicalDecomp[$source] = $dest;
 121                 if( empty( $exclude[$source] ) ) {
 122                         $canonicalComp[$dest] = $source;
 123                 }
 124         }
 125         #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
 126 }
 127 fclose( $in );
 128
 129 print "Recursively expanding canonical mappings...\n";
 130 $changed = 42;
 131 $pass = 1;
 132 while( $changed > 0 ) {
 133         print "pass $pass\n";
 134         $changed = 0;
 135         foreach( $canonicalDecomp as $source => $dest ) {
 136                 $newDest = preg_replace_callback(
 137                         '/([\xc0-\xff][\x80-\xbf]+)/',
 138                         'callbackCanonical',
 139                         $dest);
 140                 if( $newDest === $dest ) continue;
 141                 $changed++;
 142                 $canonicalDecomp[$source] = $newDest;
 143         }
 144         $pass++;
 145 }
 146
 147 print "Recursively expanding compatibility mappings...\n";
 148 $changed = 42;
 149 $pass = 1;
 150 while( $changed > 0 ) {
 151         print "pass $pass\n";
 152         $changed = 0;
 153         foreach( $compatibilityDecomp as $source => $dest ) {
 154                 $newDest = preg_replace_callback(
 155                         '/([\xc0-\xff][\x80-\xbf]+)/',
 156                         'callbackCompat',
 157                         $dest);
 158                 if( $newDest === $dest ) continue;
 159                 $changed++;
 160                 $compatibilityDecomp[$source] = $newDest;
 161         }
 162         $pass++;
 163 }
 164
 165 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
 166
 167 $out = fopen("UtfNormalData.inc", "wt");
 168 if( $out ) {
 169         $serCombining = escapeSingleString( serialize( $combiningClass ) );
 170         $serComp = escapeSingleString( serialize( $canonicalComp ) );
 171         $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
 172         $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
 173         $outdata = "<" . "?php
 174 /**
 175  * This file was automatically generated -- do not edit!
 176  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 177  * @package MediaWiki
 178  */
 179 /** */
 180 global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp, \$utfCheckNFC;
 181 \$utfCombiningClass = unserialize( '$serCombining' );
 182 \$utfCanonicalComp = unserialize( '$serComp' );
 183 \$utfCanonicalDecomp = unserialize( '$serCanon' );
 184 \$utfCheckNFC = unserialize( '$serCheckNFC' );
 185 ?" . ">\n";
 186         fputs( $out, $outdata );
 187         fclose( $out );
 188         print "Wrote out UtfNormalData.inc\n";
 189 } else {
 190         print "Can't create file UtfNormalData.inc\n";
 191         exit(-1);
 192 }
 193
 194
 195 $out = fopen("UtfNormalDataK.inc", "wt");
 196 if( $out ) {
 197         $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
 198         $outdata = "<" . "?php
 199 /**
 200  * This file was automatically generated -- do not edit!
 201  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 202  * @package MediaWiki
 203  */
 204 /** */
 205 global \$utfCompatibilityDecomp;
 206 \$utfCompatibilityDecomp = unserialize( '$serCompat' );
 207 ?" . ">\n";
 208         fputs( $out, $outdata );
 209         fclose( $out );
 210         print "Wrote out UtfNormalDataK.inc\n";
 211         exit(0);
 212 } else {
 213         print "Can't create file UtfNormalDataK.inc\n";
 214         exit(-1);
 215 }
 216
 217 # ---------------
 218
 219 function callbackCanonical( $matches ) {
 220         global $canonicalDecomp;
 221         if( isset( $canonicalDecomp[$matches[1]] ) ) {
 222                 return $canonicalDecomp[$matches[1]];
 223         }
 224         return $matches[1];
 225 }
 226
 227 function callbackCompat( $matches ) {
 228         global $compatibilityDecomp;
 229         if( isset( $compatibilityDecomp[$matches[1]] ) ) {
 230                 return $compatibilityDecomp[$matches[1]];
 231         }
 232         return $matches[1];
 233 }
 234
 235 ?>