includes/normal/UtfNormalGenerate.php

   1 <?php
   2 /**
   3  * This script generates UniNormalData.inc from the Unicode Character Database
   4  * and supplementary files.
   5  *
   6  * Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   7  * https://www.mediawiki.org/
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write to the Free Software Foundation, Inc.,
  21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  22  * http://www.gnu.org/copyleft/gpl.html
  23  *
  24  * @file
  25  * @ingroup UtfNormal
  26  */
  27
  28 if( PHP_SAPI != 'cli' ) {
  29         die( "Run me from the command line please.\n" );
  30 }
  31
  32 require_once 'UtfNormalDefines.php';
  33 require_once 'UtfNormalUtil.php';
  34
  35 $in = fopen("DerivedNormalizationProps.txt", "rt" );
  36 if( !$in ) {
  37         print "Can't open DerivedNormalizationProps.txt for reading.\n";
  38         print "If necessary, fetch this file from the internet:\n";
  39         print "http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt\n";
  40         exit(-1);
  41 }
  42 print "Initializing normalization quick check tables...\n";
  43 $checkNFC = array();
  44 while( false !== ($line = fgets( $in ) ) ) {
  45         $matches = array();
  46         if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
  47                 list( $junk, $first, $last, $prop, $value ) = $matches;
  48                 #print "$first $last $prop $value\n";
  49                 if( !$last ) $last = $first;
  50                 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
  51                         $char = codepointToUtf8( $i );
  52                         $checkNFC[$char] = $value;
  53                 }
  54         }
  55 }
  56 fclose( $in );
  57
  58 $in = fopen("CompositionExclusions.txt", "rt" );
  59 if( !$in ) {
  60         print "Can't open CompositionExclusions.txt for reading.\n";
  61         print "If necessary, fetch this file from the internet:\n";
  62         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  63         exit(-1);
  64 }
  65 $exclude = array();
  66 while( false !== ($line = fgets( $in ) ) ) {
  67         if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
  68                 $codepoint = $matches[1];
  69                 $source = codepointToUtf8( hexdec( $codepoint ) );
  70                 $exclude[$source] = true;
  71         }
  72 }
  73 fclose($in);
  74
  75 $in = fopen("UnicodeData.txt", "rt" );
  76 if( !$in ) {
  77         print "Can't open UnicodeData.txt for reading.\n";
  78         print "If necessary, fetch this file from the internet:\n";
  79         print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
  80         exit(-1);
  81 }
  82
  83 $compatibilityDecomp = array();
  84 $canonicalDecomp = array();
  85 $canonicalComp = array();
  86 $combiningClass = array();
  87 $total = 0;
  88 $compat = 0;
  89 $canon = 0;
  90
  91 print "Reading character definitions...\n";
  92 while( false !== ($line = fgets( $in ) ) ) {
  93         $columns = explode(';', $line);
  94         $codepoint = $columns[0];
  95         $name = $columns[1];
  96         $canonicalCombiningClass = $columns[3];
  97         $decompositionMapping = $columns[5];
  98
  99         $source = codepointToUtf8( hexdec( $codepoint ) );
 100
 101         if( $canonicalCombiningClass != 0 ) {
 102                 $combiningClass[$source] = intval( $canonicalCombiningClass );
 103         }
 104
 105         if( $decompositionMapping === '' ) continue;
 106         if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
 107                 # Compatibility decomposition
 108                 $canonical = false;
 109                 $decompositionMapping = $matches[2];
 110                 $compat++;
 111         } else {
 112                 $canonical = true;
 113                 $canon++;
 114         }
 115         $total++;
 116         $dest = hexSequenceToUtf8( $decompositionMapping );
 117
 118         $compatibilityDecomp[$source] = $dest;
 119         if( $canonical ) {
 120                 $canonicalDecomp[$source] = $dest;
 121                 if( empty( $exclude[$source] ) ) {
 122                         $canonicalComp[$dest] = $source;
 123                 }
 124         }
 125         #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
 126 }
 127 fclose( $in );
 128
 129 print "Recursively expanding canonical mappings...\n";
 130 $changed = 42;
 131 $pass = 1;
 132 while( $changed > 0 ) {
 133         print "pass $pass\n";
 134         $changed = 0;
 135         foreach( $canonicalDecomp as $source => $dest ) {
 136                 $newDest = preg_replace_callback(
 137                         '/([\xc0-\xff][\x80-\xbf]+)/',
 138                         'callbackCanonical',
 139                         $dest);
 140                 if( $newDest === $dest ) continue;
 141                 $changed++;
 142                 $canonicalDecomp[$source] = $newDest;
 143         }
 144         $pass++;
 145 }
 146
 147 print "Recursively expanding compatibility mappings...\n";
 148 $changed = 42;
 149 $pass = 1;
 150 while( $changed > 0 ) {
 151         print "pass $pass\n";
 152         $changed = 0;
 153         foreach( $compatibilityDecomp as $source => $dest ) {
 154                 $newDest = preg_replace_callback(
 155                         '/([\xc0-\xff][\x80-\xbf]+)/',
 156                         'callbackCompat',
 157                         $dest);
 158                 if( $newDest === $dest ) continue;
 159                 $changed++;
 160                 $compatibilityDecomp[$source] = $newDest;
 161         }
 162         $pass++;
 163 }
 164
 165 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
 166
 167 $out = fopen("UtfNormalData.inc", "wt");
 168 if( $out ) {
 169         $serCombining = escapeSingleString( serialize( $combiningClass ) );
 170         $serComp = escapeSingleString( serialize( $canonicalComp ) );
 171         $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
 172         $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
 173         $outdata = "<" . "?php
 174 /**
 175  * This file was automatically generated -- do not edit!
 176  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 177  *
 178  * @file
 179  */
 180
 181 UtfNormal::\$utfCombiningClass = unserialize( '$serCombining' );
 182 UtfNormal::\$utfCanonicalComp = unserialize( '$serComp' );
 183 UtfNormal::\$utfCanonicalDecomp = unserialize( '$serCanon' );
 184 UtfNormal::\$utfCheckNFC = unserialize( '$serCheckNFC' );
 185 \n";
 186         fputs( $out, $outdata );
 187         fclose( $out );
 188         print "Wrote out UtfNormalData.inc\n";
 189 } else {
 190         print "Can't create file UtfNormalData.inc\n";
 191         exit(-1);
 192 }
 193
 194
 195 $out = fopen("UtfNormalDataK.inc", "wt");
 196 if( $out ) {
 197         $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
 198         $outdata = "<" . "?php
 199 /**
 200  * This file was automatically generated -- do not edit!
 201  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 202  *
 203  * @file
 204  */
 205
 206 UtfNormal::\$utfCompatibilityDecomp = unserialize( '$serCompat' );
 207 \n";
 208         fputs( $out, $outdata );
 209         fclose( $out );
 210         print "Wrote out UtfNormalDataK.inc\n";
 211         exit(0);
 212 } else {
 213         print "Can't create file UtfNormalDataK.inc\n";
 214         exit(-1);
 215 }
 216
 217 # ---------------
 218
 219 function callbackCanonical( $matches ) {
 220         global $canonicalDecomp;
 221         if( isset( $canonicalDecomp[$matches[1]] ) ) {
 222                 return $canonicalDecomp[$matches[1]];
 223         }
 224         return $matches[1];
 225 }
 226
 227 function callbackCompat( $matches ) {
 228         global $compatibilityDecomp;
 229         if( isset( $compatibilityDecomp[$matches[1]] ) ) {
 230                 return $compatibilityDecomp[$matches[1]];
 231         }
 232         return $matches[1];
 233 }