[glib] unicode: Simplify width table generation

From: Matthias Clasen <matthiasc src gnome org>
To: commits-list gnome org
Cc:
Subject: [glib] unicode: Simplify width table generation
Date: Sat, 28 Jun 2014 16:51:27 +0000 (UTC)
commit 33c8a894907cf557c50849d8a3a2447324750414
Author: Christian Persch <chpe gnome org>
Date:   Sat May 3 17:56:07 2014 +0200

    unicode: Simplify width table generation
    
    Move width table generation into the gen-unicode-tables.pl script. This makes
    updating the tables automatic without the previously required manual editing
    required to insert the tables in the right place of the source code.

 glib/gen-iswide-table.py   |   58 ------------
 glib/gen-unicode-tables.pl |   97 +++++++++++++++++++-
 glib/gunichartables.h      |  220 ++++++++++++++++++++++++++++++++++++++++++++
 glib/guniprop.c            |   90 ++----------------
 4 files changed, 322 insertions(+), 143 deletions(-)
---
diff --git a/glib/gen-unicode-tables.pl b/glib/gen-unicode-tables.pl
index 37d9de9..e6520b8 100755
--- a/glib/gen-unicode-tables.pl
+++ b/glib/gen-unicode-tables.pl
@@ -161,6 +161,10 @@ my @special_cases;
 my @special_case_offsets;
 my $special_case_offset = 0;
 
+# East asian widths
+
+my @eawidths;
+
 $do_decomp = 0;
 $do_props = 1;
 if (@ARGV && $ARGV[0] eq '-decomp')
@@ -177,10 +181,11 @@ elsif (@ARGV && $ARGV[0] eq '-both')
 
 if (@ARGV != 2) {
     $0 =~ s  */@@;
-    die "\nUsage: $0 [-decomp | -both] UNICODE-VERSION DIRECTORY\n\n       DIRECTORY should contain the 
following Unicode data files:\n       UnicodeData.txt, LineBreak.txt, SpecialCasing.txt, CaseFolding.txt,\n   
    CompositionExclusions.txt\n\n";
+    die "\nUsage: $0 [-decomp | -both] UNICODE-VERSION DIRECTORY\n\n       DIRECTORY should contain the 
following Unicode data files:\n       UnicodeData.txt, LineBreak.txt, SpecialCasing.txt, CaseFolding.txt,\n   
    CompositionExclusions.txt extracted/DerivedEastAsianWidth.txt \n\n";
 }
 
-my ($unicodedatatxt, $linebreaktxt, $specialcasingtxt, $casefoldingtxt, $compositionexclusionstxt);
+my ($unicodedatatxt, $linebreaktxt, $specialcasingtxt, $casefoldingtxt, $compositionexclusionstxt,
+    $derivedeastasianwidth);
 
 my $d = $ARGV[1];
 opendir (my $dir, $d) or die "Cannot open Unicode data dir $d: $!\n";
@@ -193,11 +198,19 @@ for my $f (readdir ($dir))
     $compositionexclusionstxt = "$d/$f" if ($f =~ /^CompositionExclusions.*\.txt/);
 }
 
+my $extd = $ARGV[1] . "/extracted";
+opendir (my $extdir, $extd) or die "Cannot open Unicode/extracted data dir $extd: $!\n";
+for my $f (readdir ($extdir))
+{
+    $derivedeastasianwidthtxt = "$extd/$f" if ($f =~ /^DerivedEastAsianWidth.*\.txt/);
+}
+
 defined $unicodedatatxt or die "Did not find UnicodeData file";
 defined $linebreaktxt or die "Did not find LineBreak file";
 defined $specialcasingtxt or die "Did not find SpecialCasing file";
 defined $casefoldingtxt or die "Did not find CaseFolding file";
 defined $compositionexclusionstxt or die "Did not find CompositionExclusions file";
+defined $derivedeastasianwidthtxt or die "Did not find DerivedEastAsianWidth file";
 
 print "Creating decomp table\n" if ($do_decomp);
 print "Creating property table\n" if ($do_props);
@@ -489,6 +502,31 @@ while (<INPUT>)
 
 close INPUT;
 
+print "Reading derived east asian widths\n";
+
+open (INPUT, "< $derivedeastasianwidthtxt") || exit 1;
+
+while (<INPUT>)
+{
+    my ($start_code, $end_code);
+    
+    chop;
+
+    s/#.*//;
+    next if /^\s*$/;
+    if (!/^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([A-Za-z_]+)\s*$/) {
+       die "Cannot parse line: '$_'\n";
+    }
+
+    if (defined $2) {
+       push @eawidths, [ hex $1, hex $2, $3 ];
+    } else {
+       push @eawidths, [ hex $1, hex $1, $3 ];
+    }
+}
+
+close INPUT;
+
 if ($do_props) {
     &print_tables ($last_code)
 }
@@ -664,6 +702,11 @@ sub print_tables
     &output_special_case_table (\*OUT);
     &output_casefold_table (\*OUT);
 
+    #
+    # And the widths tables
+    #
+    &output_width_tables (\*OUT);
+
     print OUT "#endif /* CHARTABLES_H */\n";
 
     close (OUT);
@@ -1334,5 +1377,53 @@ EOT
    printf "Generated %d bytes for casefold table\n", $recordlen * @casefold;
 }
 
-                            
+sub output_one_width_table
+{
+    my ($out, $name, $wpe) = @_;
+    my $start;
+    my $end;
+    my $wp;
+    my $rex;
+
+    print $out "static const struct Interval g_unicode_width_table_${name}[] = {\n";
+
+    $rex = qr/$wpe/;
+
+    for (my $i = 0; $i <= $#eawidths; $i++) {
+        $start = $eawidths[$i]->[0];
+        $end = $eawidths[$i]->[1];
+        $wp = $eawidths[$i]->[2];
+
+        next if ($wp !~ $rex);
 
+        while ($i <= $#eawidths - 1 &&
+               $eawidths[$i + 1]->[0] == $end + 1 &&
+               ($eawidths[$i + 1]->[2] =~ $rex)) {
+            $i++;
+            $end = $eawidths[$i]->[1];
+        }
+        
+       printf $out "{0x%04X, 0x%04X},\n", $start, $end;
+    }
+
+    printf $out "};\n\n";
+}
+
+sub output_width_tables
+{
+    my $out = shift;
+
+    @eawidths = sort { $a->[0] <=> $b->[0] } @eawidths;
+
+    print $out <<EOT;
+
+struct Interval
+{
+  gunichar start, end;
+};
+
+EOT
+
+    &output_one_width_table ($out,"wide", "[FW]");
+    &output_one_width_table ($out, "ambiguous", "[A]");
+}
diff --git a/glib/gunichartables.h b/glib/gunichartables.h
index b9c693c..2f6ad3d 100644
--- a/glib/gunichartables.h
+++ b/glib/gunichartables.h
@@ -15146,4 +15146,224 @@ static const struct {
   { 0xfb17, "\xd5\xb4\xd5\xad" },
 };
 
+
+struct Interval
+{
+  gunichar start, end;
+};
+
+static const struct Interval g_unicode_width_table_wide[] = {
+{0x1100, 0x115F},
+{0x2329, 0x232A},
+{0x2E80, 0x2E99},
+{0x2E9B, 0x2EF3},
+{0x2F00, 0x2FD5},
+{0x2FF0, 0x2FFB},
+{0x3000, 0x303E},
+{0x3041, 0x3096},
+{0x3099, 0x30FF},
+{0x3105, 0x312D},
+{0x3131, 0x318E},
+{0x3190, 0x31BA},
+{0x31C0, 0x31E3},
+{0x31F0, 0x321E},
+{0x3220, 0x3247},
+{0x3250, 0x32FE},
+{0x3300, 0x4DBF},
+{0x4E00, 0xA48C},
+{0xA490, 0xA4C6},
+{0xA960, 0xA97C},
+{0xAC00, 0xD7A3},
+{0xF900, 0xFAFF},
+{0xFE10, 0xFE19},
+{0xFE30, 0xFE52},
+{0xFE54, 0xFE66},
+{0xFE68, 0xFE6B},
+{0xFF01, 0xFF60},
+{0xFFE0, 0xFFE6},
+{0x1B000, 0x1B001},
+{0x1F200, 0x1F202},
+{0x1F210, 0x1F23A},
+{0x1F240, 0x1F248},
+{0x1F250, 0x1F251},
+{0x20000, 0x2FFFD},
+{0x30000, 0x3FFFD},
+};
+
+static const struct Interval g_unicode_width_table_ambiguous[] = {
+{0x00A1, 0x00A1},
+{0x00A4, 0x00A4},
+{0x00A7, 0x00A8},
+{0x00AA, 0x00AA},
+{0x00AD, 0x00AE},
+{0x00B0, 0x00B4},
+{0x00B6, 0x00BA},
+{0x00BC, 0x00BF},
+{0x00C6, 0x00C6},
+{0x00D0, 0x00D0},
+{0x00D7, 0x00D8},
+{0x00DE, 0x00E1},
+{0x00E6, 0x00E6},
+{0x00E8, 0x00EA},
+{0x00EC, 0x00ED},
+{0x00F0, 0x00F0},
+{0x00F2, 0x00F3},
+{0x00F7, 0x00FA},
+{0x00FC, 0x00FC},
+{0x00FE, 0x00FE},
+{0x0101, 0x0101},
+{0x0111, 0x0111},
+{0x0113, 0x0113},
+{0x011B, 0x011B},
+{0x0126, 0x0127},
+{0x012B, 0x012B},
+{0x0131, 0x0133},
+{0x0138, 0x0138},
+{0x013F, 0x0142},
+{0x0144, 0x0144},
+{0x0148, 0x014B},
+{0x014D, 0x014D},
+{0x0152, 0x0153},
+{0x0166, 0x0167},
+{0x016B, 0x016B},
+{0x01CE, 0x01CE},
+{0x01D0, 0x01D0},
+{0x01D2, 0x01D2},
+{0x01D4, 0x01D4},
+{0x01D6, 0x01D6},
+{0x01D8, 0x01D8},
+{0x01DA, 0x01DA},
+{0x01DC, 0x01DC},
+{0x0251, 0x0251},
+{0x0261, 0x0261},
+{0x02C4, 0x02C4},
+{0x02C7, 0x02C7},
+{0x02C9, 0x02CB},
+{0x02CD, 0x02CD},
+{0x02D0, 0x02D0},
+{0x02D8, 0x02DB},
+{0x02DD, 0x02DD},
+{0x02DF, 0x02DF},
+{0x0300, 0x036F},
+{0x0391, 0x03A1},
+{0x03A3, 0x03A9},
+{0x03B1, 0x03C1},
+{0x03C3, 0x03C9},
+{0x0401, 0x0401},
+{0x0410, 0x044F},
+{0x0451, 0x0451},
+{0x2010, 0x2010},
+{0x2013, 0x2016},
+{0x2018, 0x2019},
+{0x201C, 0x201D},
+{0x2020, 0x2022},
+{0x2024, 0x2027},
+{0x2030, 0x2030},
+{0x2032, 0x2033},
+{0x2035, 0x2035},
+{0x203B, 0x203B},
+{0x203E, 0x203E},
+{0x2074, 0x2074},
+{0x207F, 0x207F},
+{0x2081, 0x2084},
+{0x20AC, 0x20AC},
+{0x2103, 0x2103},
+{0x2105, 0x2105},
+{0x2109, 0x2109},
+{0x2113, 0x2113},
+{0x2116, 0x2116},
+{0x2121, 0x2122},
+{0x2126, 0x2126},
+{0x212B, 0x212B},
+{0x2153, 0x2154},
+{0x215B, 0x215E},
+{0x2160, 0x216B},
+{0x2170, 0x2179},
+{0x2189, 0x2189},
+{0x2190, 0x2199},
+{0x21B8, 0x21B9},
+{0x21D2, 0x21D2},
+{0x21D4, 0x21D4},
+{0x21E7, 0x21E7},
+{0x2200, 0x2200},
+{0x2202, 0x2203},
+{0x2207, 0x2208},
+{0x220B, 0x220B},
+{0x220F, 0x220F},
+{0x2211, 0x2211},
+{0x2215, 0x2215},
+{0x221A, 0x221A},
+{0x221D, 0x2220},
+{0x2223, 0x2223},
+{0x2225, 0x2225},
+{0x2227, 0x222C},
+{0x222E, 0x222E},
+{0x2234, 0x2237},
+{0x223C, 0x223D},
+{0x2248, 0x2248},
+{0x224C, 0x224C},
+{0x2252, 0x2252},
+{0x2260, 0x2261},
+{0x2264, 0x2267},
+{0x226A, 0x226B},
+{0x226E, 0x226F},
+{0x2282, 0x2283},
+{0x2286, 0x2287},
+{0x2295, 0x2295},
+{0x2299, 0x2299},
+{0x22A5, 0x22A5},
+{0x22BF, 0x22BF},
+{0x2312, 0x2312},
+{0x2460, 0x24E9},
+{0x24EB, 0x254B},
+{0x2550, 0x2573},
+{0x2580, 0x258F},
+{0x2592, 0x2595},
+{0x25A0, 0x25A1},
+{0x25A3, 0x25A9},
+{0x25B2, 0x25B3},
+{0x25B6, 0x25B7},
+{0x25BC, 0x25BD},
+{0x25C0, 0x25C1},
+{0x25C6, 0x25C8},
+{0x25CB, 0x25CB},
+{0x25CE, 0x25D1},
+{0x25E2, 0x25E5},
+{0x25EF, 0x25EF},
+{0x2605, 0x2606},
+{0x2609, 0x2609},
+{0x260E, 0x260F},
+{0x2614, 0x2615},
+{0x261C, 0x261C},
+{0x261E, 0x261E},
+{0x2640, 0x2640},
+{0x2642, 0x2642},
+{0x2660, 0x2661},
+{0x2663, 0x2665},
+{0x2667, 0x266A},
+{0x266C, 0x266D},
+{0x266F, 0x266F},
+{0x269E, 0x269F},
+{0x26BE, 0x26BF},
+{0x26C4, 0x26CD},
+{0x26CF, 0x26E1},
+{0x26E3, 0x26E3},
+{0x26E8, 0x26FF},
+{0x273D, 0x273D},
+{0x2757, 0x2757},
+{0x2776, 0x277F},
+{0x2B55, 0x2B59},
+{0x3248, 0x324F},
+{0xE000, 0xF8FF},
+{0xFE00, 0xFE0F},
+{0xFFFD, 0xFFFD},
+{0x1F100, 0x1F10A},
+{0x1F110, 0x1F12D},
+{0x1F130, 0x1F169},
+{0x1F170, 0x1F19A},
+{0xE0100, 0xE01EF},
+{0xF0000, 0xFFFFD},
+{0x100000, 0x10FFFD},
+};
+
 #endif /* CHARTABLES_H */
diff --git a/glib/guniprop.c b/glib/guniprop.c
index 82ba889..15c28e1 100644
--- a/glib/guniprop.c
+++ b/glib/guniprop.c
@@ -419,11 +419,6 @@ g_unichar_iszerowidth (gunichar c)
   return FALSE;
 }
 
-struct Interval
-{
-  gunichar start, end;
-};
-
 static int
 interval_compare (const void *key, const void *elt)
 {
@@ -438,20 +433,6 @@ interval_compare (const void *key, const void *elt)
   return 0;
 }
 
-
-/*
- * NOTE:
- *
- * The tables for g_unichar_iswide() and g_unichar_iswide_cjk() are
- * generated from the Unicode Character Database's file
- * extracted/DerivedEastAsianWidth.txt using the gen-iswide-table.py
- * in this way:
- *
- *   ./gen-iswide-table.py < path/to/ucd/extracted/DerivedEastAsianWidth.txt | fmt
- *
- * Last update for Unicode 6.0.
- */
-
 /**
  * g_unichar_iswide:
  * @c: a Unicode character
@@ -464,20 +445,10 @@ interval_compare (const void *key, const void *elt)
 gboolean
 g_unichar_iswide (gunichar c)
 {
-  /* See NOTE earlier for how to update this table. */
-  static const struct Interval wide[] = {
-    {0x1100, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3},
-    {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, {0x3000, 0x303E}, {0x3041, 0x3096},
-    {0x3099, 0x30FF}, {0x3105, 0x312D}, {0x3131, 0x318E}, {0x3190, 0x31BA},
-    {0x31C0, 0x31E3}, {0x31F0, 0x321E}, {0x3220, 0x3247}, {0x3250, 0x32FE},
-    {0x3300, 0x4DBF}, {0x4E00, 0xA48C}, {0xA490, 0xA4C6}, {0xA960, 0xA97C},
-    {0xAC00, 0xD7A3}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52},
-    {0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, {0xFF01, 0xFF60}, {0xFFE0, 0xFFE6},
-    {0x1B000, 0x1B001}, {0x1F200, 0x1F202}, {0x1F210, 0x1F23A}, {0x1F240,
-    0x1F248}, {0x1F250, 0x1F251}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}
-  };
-
-  if (bsearch (GUINT_TO_POINTER (c), wide, G_N_ELEMENTS (wide), sizeof wide[0],
+  if (bsearch (GUINT_TO_POINTER (c),
+               g_unicode_width_table_wide,
+               G_N_ELEMENTS (g_unicode_width_table_wide),
+               sizeof g_unicode_width_table_wide[0],
               interval_compare))
     return TRUE;
 
@@ -507,58 +478,13 @@ g_unichar_iswide (gunichar c)
 gboolean
 g_unichar_iswide_cjk (gunichar c)
 {
-  /* See NOTE earlier for how to update this table. */
-  static const struct Interval ambiguous[] = {
-    {0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8}, {0x00AA, 0x00AA},
-    {0x00AD, 0x00AE}, {0x00B0, 0x00B4}, {0x00B6, 0x00BA}, {0x00BC, 0x00BF},
-    {0x00C6, 0x00C6}, {0x00D0, 0x00D0}, {0x00D7, 0x00D8}, {0x00DE, 0x00E1},
-    {0x00E6, 0x00E6}, {0x00E8, 0x00EA}, {0x00EC, 0x00ED}, {0x00F0, 0x00F0},
-    {0x00F2, 0x00F3}, {0x00F7, 0x00FA}, {0x00FC, 0x00FC}, {0x00FE, 0x00FE},
-    {0x0101, 0x0101}, {0x0111, 0x0111}, {0x0113, 0x0113}, {0x011B, 0x011B},
-    {0x0126, 0x0127}, {0x012B, 0x012B}, {0x0131, 0x0133}, {0x0138, 0x0138},
-    {0x013F, 0x0142}, {0x0144, 0x0144}, {0x0148, 0x014B}, {0x014D, 0x014D},
-    {0x0152, 0x0153}, {0x0166, 0x0167}, {0x016B, 0x016B}, {0x01CE, 0x01CE},
-    {0x01D0, 0x01D0}, {0x01D2, 0x01D2}, {0x01D4, 0x01D4}, {0x01D6, 0x01D6},
-    {0x01D8, 0x01D8}, {0x01DA, 0x01DA}, {0x01DC, 0x01DC}, {0x0251, 0x0251},
-    {0x0261, 0x0261}, {0x02C4, 0x02C4}, {0x02C7, 0x02C7}, {0x02C9, 0x02CB},
-    {0x02CD, 0x02CD}, {0x02D0, 0x02D0}, {0x02D8, 0x02DB}, {0x02DD, 0x02DD},
-    {0x02DF, 0x02DF}, {0x0300, 0x036F}, {0x0391, 0x03A1}, {0x03A3, 0x03A9},
-    {0x03B1, 0x03C1}, {0x03C3, 0x03C9}, {0x0401, 0x0401}, {0x0410, 0x044F},
-    {0x0451, 0x0451}, {0x2010, 0x2010}, {0x2013, 0x2016}, {0x2018, 0x2019},
-    {0x201C, 0x201D}, {0x2020, 0x2022}, {0x2024, 0x2027}, {0x2030, 0x2030},
-    {0x2032, 0x2033}, {0x2035, 0x2035}, {0x203B, 0x203B}, {0x203E, 0x203E},
-    {0x2074, 0x2074}, {0x207F, 0x207F}, {0x2081, 0x2084}, {0x20AC, 0x20AC},
-    {0x2103, 0x2103}, {0x2105, 0x2105}, {0x2109, 0x2109}, {0x2113, 0x2113},
-    {0x2116, 0x2116}, {0x2121, 0x2122}, {0x2126, 0x2126}, {0x212B, 0x212B},
-    {0x2153, 0x2154}, {0x215B, 0x215E}, {0x2160, 0x216B}, {0x2170, 0x2179},
-    {0x2189, 0x2189}, {0x2190, 0x2199}, {0x21B8, 0x21B9}, {0x21D2, 0x21D2},
-    {0x21D4, 0x21D4}, {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203},
-    {0x2207, 0x2208}, {0x220B, 0x220B}, {0x220F, 0x220F}, {0x2211, 0x2211},
-    {0x2215, 0x2215}, {0x221A, 0x221A}, {0x221D, 0x2220}, {0x2223, 0x2223},
-    {0x2225, 0x2225}, {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237},
-    {0x223C, 0x223D}, {0x2248, 0x2248}, {0x224C, 0x224C}, {0x2252, 0x2252},
-    {0x2260, 0x2261}, {0x2264, 0x2267}, {0x226A, 0x226B}, {0x226E, 0x226F},
-    {0x2282, 0x2283}, {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299},
-    {0x22A5, 0x22A5}, {0x22BF, 0x22BF}, {0x2312, 0x2312}, {0x2460, 0x24E9},
-    {0x24EB, 0x254B}, {0x2550, 0x2573}, {0x2580, 0x258F}, {0x2592, 0x2595},
-    {0x25A0, 0x25A1}, {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7},
-    {0x25BC, 0x25BD}, {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, {0x25CB, 0x25CB},
-    {0x25CE, 0x25D1}, {0x25E2, 0x25E5}, {0x25EF, 0x25EF}, {0x2605, 0x2606},
-    {0x2609, 0x2609}, {0x260E, 0x260F}, {0x2614, 0x2615}, {0x261C, 0x261C},
-    {0x261E, 0x261E}, {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661},
-    {0x2663, 0x2665}, {0x2667, 0x266A}, {0x266C, 0x266D}, {0x266F, 0x266F},
-    {0x269E, 0x269F}, {0x26BE, 0x26BF}, {0x26C4, 0x26CD}, {0x26CF, 0x26E1},
-    {0x26E3, 0x26E3}, {0x26E8, 0x26FF}, {0x273D, 0x273D}, {0x2757, 0x2757},
-    {0x2776, 0x277F}, {0x2B55, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF},
-    {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, {0x1F110,
-    0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F19A}, {0xE0100, 0xE01EF},
-    {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}
-  };
-
   if (g_unichar_iswide (c))
     return TRUE;
 
-  if (bsearch (GUINT_TO_POINTER (c), ambiguous, G_N_ELEMENTS (ambiguous), sizeof ambiguous[0],
+  if (bsearch (GUINT_TO_POINTER (c), 
+               g_unicode_width_table_ambiguous,
+               G_N_ELEMENTS (g_unicode_width_table_ambiguous),
+               sizeof g_unicode_width_table_ambiguous[0],
               interval_compare))
     return TRUE;
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]