Merge "PostgreSQL: Fix text search on moved pages"

[lhc/web/wiklou.git] / maintenance / language / zhtable / Makefile.py

diff --git a/maintenance/language/zhtable/Makefile.py b/maintenance/language/zhtable/Makefile.py
index f902e58..4ab57d4 100755 (executable)
--- a/maintenance/language/zhtable/Makefile.py
+++ b/maintenance/language/zhtable/Makefile.py
@@ -152,14 +152,12 @@ def applyExcludes( mlist, path ):
     return mlist
 
 def charManualTable( path ):
-    fp = open( path, 'r', encoding = 'U8' )
-    ret = {}
-    for line in fp:
-        elems = line.split( '#' )[0].split( '|' )
-        elems = unichr3( *elems )
-        if len( elems ) > 1:
-            ret[elems[0]] = elems[1:]
-    return ret
+   fp = open( path, 'r', encoding = 'U8' )
+   for line in fp:
+       elems = line.split( '#' )[0].split( '|' )
+       elems = unichr3( *elems )
+       if len( elems ) > 1:
+           yield elems[0], elems[1:]
         
 def toManyRules( src_table ):
     tomany = set()
@@ -205,13 +203,16 @@ def customRules( path ):
     fp = open( path, 'r', encoding = 'U8' )
     ret = dict()
     for line in fp:
-        elems = line.split( '#' )[0].split()
+        line = line.rstrip( '\r\n' )
+        if '#' in line:
+            line = line.split( '#' )[0].rstrip()
+        elems = line.split( '\t' )
         if len( elems ) > 1:
             ret[elems[0]] = elems[1]
     return ret
 
 def dictToSortedList( src_table, pos ):
-    return sorted( src_table.items(), key = lambda m: m[pos] )
+    return sorted( src_table.items(), key = lambda m: ( m[pos], m[1 - pos] ) )
 
 def translate( text, conv_table ):
     i = 0
@@ -231,7 +232,7 @@ def manualWordsTable( path, conv_table, reconv_table ):
     reconv_table = {}
     wordlist = [line.split( '#' )[0].strip() for line in fp]
     wordlist = list( set( wordlist ) )
-    wordlist.sort( key = len, reverse = True )
+    wordlist.sort( key = lambda w: ( len(w), w ), reverse = True )
     while wordlist:
         word = wordlist.pop()
         new_word = translate( word, conv_table )
@@ -243,7 +244,7 @@ def manualWordsTable( path, conv_table, reconv_table ):
 
 def defaultWordsTable( src_wordlist, src_tomany, char_conv_table, char_reconv_table ):
     wordlist = list( src_wordlist )
-    wordlist.sort( key = len, reverse = True )
+    wordlist.sort( key = lambda w: ( len(w), w ), reverse = True )
     word_conv_table = {}
     word_reconv_table = {}
     conv_table = char_conv_table.copy()
@@ -278,7 +279,7 @@ def PHPArray( table ):
 def main():
     #Get Unihan.zip:
     url = 'http://www.unicode.org/Public/%s/ucd/Unihan.zip' % UNIHAN_VER
-    han_dest = 'Unihan.zip'
+    han_dest = 'Unihan-%s.zip' % UNIHAN_VER
     download( url, han_dest )
     
     # Get scim-tables-$(SCIM_TABLES_VER).tar.gz:
@@ -299,7 +300,9 @@ def main():
     # Unihan.txt
     ( t2s_1tomany, s2t_1tomany ) = unihanParser( han_dest )
 
+    t2s_1tomany.update( charManualTable( 'symme_supp.manual' ) )
     t2s_1tomany.update( charManualTable( 'trad2simp.manual' ) )
+    s2t_1tomany.update( ( t[0], [f] ) for ( f, t ) in charManualTable( 'symme_supp.manual' ) )
     s2t_1tomany.update( charManualTable( 'simp2trad.manual' ) )
     
     if pyversion[:1] in ['2']:
@@ -369,8 +372,6 @@ def main():
     toCN = dictToSortedList( customRules( 'toCN.manual' ), 1 )
     # sorted list toHK
     toHK = dictToSortedList( customRules( 'toHK.manual' ), 1 )
-    # sorted list toSG
-    toSG = dictToSortedList( customRules( 'toSG.manual' ), 1 )
     # sorted list toTW
     toTW = dictToSortedList( customRules( 'toTW.manual' ), 1 )
     
@@ -395,8 +396,6 @@ $zh2Hant = array(\n'''
         +  PHPArray( toHK ) \
         +  '\n);\n\n$zh2CN = array(\n' \
         +  PHPArray( toCN ) \
-        +  '\n);\n\n$zh2SG = array(\n' \
-        +  PHPArray( toSG ) \
         +  '\n);\n'
     
     if pyversion[:1] in ['2']: