---

yaml
---
r: 153463
b: refs/heads/try2
c: 154ca08
h: refs/heads/master
i:
  153461: aeef75e
  153459: 91c38a6
  153455: 09f6c44
v: v3

Author: jbclements

[refs]

@@ -5,7 +5,7 @@ refs/heads/snap-stage3: 78a7676898d9f80ab540c6df5d4c9ce35bb50463
 refs/heads/try: 519addf6277dbafccbb4159db4b710c37eaa2ec5
 refs/tags/release-0.1: 1f5c5126e96c79d22cb7862f75304136e204f105
 refs/heads/ndm: f3868061cd7988080c30d6d5bf352a5a5fe2460b
-refs/heads/try2: 6c35d513cea468b30759b4f78becf28f11a123c0
+refs/heads/try2: 154ca0838868ca08a8aae20f6af245e2b970a3de
 refs/heads/dist-snap: ba4081a5a8573875fed17545846f6f6902c8ba8d
 refs/tags/release-0.2: c870d2dffb391e14efb05aa27898f1f6333a9596
 refs/tags/release-0.3: b5f0d0f648d9a6153664837026ba1be43d3e2503

branches/try2/src/etc/unicode.py

@@ -51,30 +51,6 @@
     'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'],
 }
 
-
-# Grapheme cluster data
-# taken from UAX29, http://www.unicode.org/reports/tr29/
-# these code points are excluded from the Control category
-# NOTE: CR and LF are also technically excluded, but for
-# the sake of convenience we leave them in the Control group
-# and manually check them in the appropriate place. This is
-# still compliant with the implementation requirements.
-grapheme_control_exceptions = set([0x200c, 0x200d])
-
-# the Regional_Indicator category
-grapheme_regional_indicator = [(0x1f1e6, 0x1f1ff)]
-
-# "The following ... are specifically excluded" from the SpacingMark category
-# http://www.unicode.org/reports/tr29/#SpacingMark
-grapheme_spacingmark_exceptions = [(0x102b, 0x102c), (0x1038, 0x1038),
-    (0x1062, 0x1064), (0x1067, 0x106d), (0x1083, 0x1083), (0x1087, 0x108c),
-    (0x108f, 0x108f), (0x109a, 0x109c), (0x19b0, 0x19b4), (0x19b8, 0x19b9),
-    (0x19bb, 0x19c0), (0x19c8, 0x19c9), (0x1a61, 0x1a61), (0x1a63, 0x1a64),
-    (0xaa7b, 0xaa7b), (0xaa7d, 0xaa7d)]
-
-# these are included in the SpacingMark category
-grapheme_spacingmark_extra = set([0xe33, 0xeb3])
-
 def fetch(f):
     if not os.path.exists(f):
         os.system("curl -O http://www.unicode.org/Public/UNIDATA/%s"
@@ -133,7 +109,7 @@ def load_unicode_data(f):
                 canon_decomp[code] = seq
 
         # place letter in categories as appropriate
-        for cat in [gencat, "Assigned"] + expanded_categories.get(gencat, []):
+        for cat in [gencat] + expanded_categories.get(gencat, []):
             if cat not in gencats:
                 gencats[cat] = []
             gencats[cat].append(code)
@@ -144,12 +120,6 @@ def load_unicode_data(f):
                 combines[combine] = []
             combines[combine].append(code)
 
-    # generate Not_Assigned from Assigned
-    gencats["Cn"] = gen_unassigned(gencats["Assigned"])
-    # Assigned is not a real category
-    del(gencats["Assigned"])
-    # Other contains Not_Assigned
-    gencats["C"].extend(gencats["Cn"])
     gencats = group_cats(gencats)
     combines = to_combines(group_cats(combines))
 
@@ -185,11 +155,6 @@ def ungroup_cat(cat):
             lo += 1
     return cat_out
 
-def gen_unassigned(assigned):
-    assigned = set(assigned)
-    return ([i for i in range(0, 0xd800) if i not in assigned] +
-            [i for i in range(0xe000, 0x110000) if i not in assigned])
-
 def to_combines(combs):
     combs_out = []
     for comb in combs:
@@ -385,45 +350,6 @@ def emit_conversions_module(f, lowerupper, upperlower):
         sorted(lowerupper.iteritems(), key=operator.itemgetter(0)), is_pub=False)
     f.write("}\n\n")
 
-def emit_grapheme_module(f, grapheme_table, grapheme_cats):
-    f.write("""pub mod grapheme {
-    use core::option::{Some, None};
-    use core::slice::ImmutableVector;
-
-    #[allow(non_camel_case_types)]
-    #[deriving(Clone)]
-    pub enum GraphemeCat {
-""")
-    for cat in grapheme_cats + ["Any"]:
-        f.write("        GC_" + cat + ",\n")
-    f.write("""    }
-
-    fn bsearch_range_value_table(c: char, r: &'static [(char, char, GraphemeCat)]) -> GraphemeCat {
-        use core::cmp::{Equal, Less, Greater};
-        match r.bsearch(|&(lo, hi, _)| {
-            if lo <= c && c <= hi { Equal }
-            else if hi < c { Less }
-            else { Greater }
-        }) {
-            Some(idx) => {
-                let (_, _, cat) = r[idx];
-                cat
-            }
-            None => GC_Any
-        }
-    }
-
-    pub fn grapheme_category(c: char) -> GraphemeCat {
-        bsearch_range_value_table(c, grapheme_cat_table)
-    }
-
-""")
-
-    emit_table(f, "grapheme_cat_table", grapheme_table, "&'static [(char, char, GraphemeCat)]",
-        pfun=lambda x: "(%s,%s,GC_%s)" % (escape_char(x[0]), escape_char(x[1]), x[2]),
-        is_pub=False)
-    f.write("}\n")
-
 def emit_charwidth_module(f, width_table):
     f.write("pub mod charwidth {\n")
     f.write("    use core::option::{Option, Some, None};\n")
@@ -462,7 +388,7 @@ def emit_charwidth_module(f, width_table):
     f.write("    //     http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c\n")
     emit_table(f, "charwidth_table", width_table, "&'static [(char, char, u8, u8)]", is_pub=False,
             pfun=lambda x: "(%s,%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), x[2], x[3]))
-    f.write("}\n\n")
+    f.write("}\n")
 
 def emit_norm_module(f, canon, compat, combine):
     canon_keys = canon.keys()
@@ -547,8 +473,6 @@ def remove_from_wtable(wtable, val):
         wtable_out.extend(wtable)
     return wtable_out
 
-
-
 def optimize_width_table(wtable):
     wtable_out = []
     w_this = wtable.pop(0)
@@ -563,7 +487,7 @@ def optimize_width_table(wtable):
     return wtable_out
 
 if __name__ == "__main__":
-    r = "tables.rs"
+    r = "unicode.rs"
     if os.path.exists(r):
         os.remove(r)
     with open(r, "w") as rf:
@@ -574,18 +498,12 @@ def optimize_width_table(wtable):
         (canon_decomp, compat_decomp, gencats, combines,
                 lowerupper, upperlower) = load_unicode_data("UnicodeData.txt")
         want_derived = ["XID_Start", "XID_Continue", "Alphabetic", "Lowercase", "Uppercase"]
-        other_derived = ["Default_Ignorable_Code_Point", "Grapheme_Extend"]
+        other_derived = ["Default_Ignorable_Code_Point"]
         derived = load_properties("DerivedCoreProperties.txt", want_derived + other_derived)
         scripts = load_properties("Scripts.txt", [])
         props = load_properties("PropList.txt",
                 ["White_Space", "Join_Control", "Noncharacter_Code_Point"])
 
-        # grapheme cluster category from DerivedCoreProperties
-        # the rest are defined below
-        grapheme_cats = {}
-        grapheme_cats["Extend"] = derived["Grapheme_Extend"]
-        del(derived["Grapheme_Extend"])
-
         # bsearch_range_table is used in all the property modules below
         emit_bsearch_range_table(rf)
 
@@ -615,7 +533,7 @@ def optimize_width_table(wtable):
         emit_norm_module(rf, canon_decomp, compat_decomp, combines)
         emit_conversions_module(rf, lowerupper, upperlower)
 
-        ### character width module
+        # character width module
         width_table = []
         for zwcat in ["Me", "Mn", "Cf"]:
             width_table.extend(map(lambda (lo, hi): (lo, hi, 0, 0), gencats[zwcat]))
@@ -637,40 +555,3 @@ def optimize_width_table(wtable):
         # optimize the width table by collapsing adjacent entities when possible
         width_table = optimize_width_table(width_table)
         emit_charwidth_module(rf, width_table)
-
-        ### grapheme cluster module
-        # from http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Break_Property_Values
-        # Hangul syllable categories
-        want_hangul = ["L", "V", "T", "LV", "LVT"]
-        grapheme_cats.update(load_properties("HangulSyllableType.txt", want_hangul))
-
-        # Control
-        # This category also includes Cs (surrogate codepoints), but Rust's `char`s are
-        # Unicode Scalar Values only, and surrogates are thus invalid `char`s.
-        grapheme_cats["Control"] = set()
-        for cat in ["Zl", "Zp", "Cc", "Cf"]:
-            grapheme_cats["Control"] |= set(ungroup_cat(gencats[cat]))
-        grapheme_cats["Control"] = group_cat(list(
-            grapheme_cats["Control"]
-            - grapheme_control_exceptions
-            | (set(ungroup_cat(gencats["Cn"]))
-               & set(ungroup_cat(derived["Default_Ignorable_Code_Point"])))))
-
-        # Regional Indicator
-        grapheme_cats["RegionalIndicator"] = grapheme_regional_indicator
-
-        # Prepend - "Currently there are no characters with this value"
-        # (from UAX#29, Unicode 7.0)
-
-        # SpacingMark
-        grapheme_cats["SpacingMark"] = group_cat(list(
-            set(ungroup_cat(gencats["Mc"]))
-            - set(ungroup_cat(grapheme_cats["Extend"]))
-            | grapheme_spacingmark_extra
-            - set(ungroup_cat(grapheme_spacingmark_exceptions))))
-
-        grapheme_table = []
-        for cat in grapheme_cats:
-            grapheme_table.extend([(x, y, cat) for (x, y) in grapheme_cats[cat]])
-        grapheme_table.sort(key=lambda w: w[0])
-        emit_grapheme_module(rf, grapheme_table, grapheme_cats.keys())