bpo-30736: upgrade to Unicode 10.0 (#2344)

Straightforward. While we're at it, though, strip trailing whitespace from generated tables.

Author: benjaminp

Doc/library/stdtypes.rst

@@ -354,7 +354,7 @@ Notes:
    The numeric literals accepted include the digits ``0`` to ``9`` or any
    Unicode equivalent (code points with the ``Nd`` property).
 
-   See http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedNumericType.txt
+   See http://www.unicode.org/Public/10.0.0/ucd/extracted/DerivedNumericType.txt
    for a complete list of code points with the ``Nd`` property.
 
 

Doc/library/unicodedata.rst

@@ -17,8 +17,8 @@
 
 This module provides access to the Unicode Character Database (UCD) which
 defines character properties for all Unicode characters. The data contained in
-this database is compiled from the `UCD version 9.0.0
-<http://www.unicode.org/Public/9.0.0/ucd>`_.
+this database is compiled from the `UCD version 10.0.0
+<http://www.unicode.org/Public/10.0.0/ucd>`_.
 
 The module uses the same names and symbols as defined by Unicode
 Standard Annex #44, `"Unicode Character Database"
@@ -168,6 +168,6 @@ Examples:
 
 .. rubric:: Footnotes
 
-.. [#] http://www.unicode.org/Public/9.0.0/ucd/NameAliases.txt
+.. [#] http://www.unicode.org/Public/10.0.0/ucd/NameAliases.txt
 
-.. [#] http://www.unicode.org/Public/9.0.0/ucd/NamedSequences.txt
+.. [#] http://www.unicode.org/Public/10.0.0/ucd/NamedSequences.txt

Doc/reference/lexical_analysis.rst

@@ -313,7 +313,7 @@ The Unicode category codes mentioned above stand for:
 * *Nd* - decimal numbers
 * *Pc* - connector punctuations
 * *Other_ID_Start* - explicit list of characters in `PropList.txt
-  <http://www.unicode.org/Public/8.0.0/ucd/PropList.txt>`_ to support backwards
+  <http://www.unicode.org/Public/10.0.0/ucd/PropList.txt>`_ to support backwards
   compatibility
 * *Other_ID_Continue* - likewise
 
@@ -875,4 +875,4 @@ occurrence outside string literals and comments is an unconditional error:
 
 .. rubric:: Footnotes
 
-.. [#] http://www.unicode.org/Public/8.0.0/ucd/NameAliases.txt
+.. [#] http://www.unicode.org/Public/10.0.0/ucd/NameAliases.txt

Doc/whatsnew/3.7.rst

@@ -237,6 +237,13 @@ xmlrpc.server
 its subclasses can be used as a decorator. (Contributed by Xiang Zhang in
 :issue:`7769`.)
 
+unicodedata
+-----------
+
+The internal :mod:`unicodedata` database has been upgraded to use `Unicode 10
+<http://www.unicode.org/versions/Unicode10.0.0/>`_. (Contributed by Benjamin
+Peterson.)
+
 urllib.parse
 ------------
 

Lib/test/test_unicodedata.py

@@ -20,7 +20,7 @@
 class UnicodeMethodsTest(unittest.TestCase):
 
     # update this, if the database changes
-    expectedchecksum = 'c1fa98674a683aa8a8d8dee0c84494f8d36346e6'
+    expectedchecksum = '727091e0fd5807eb41c72912ae95cdd74c795e27'
 
     def test_method_checksum(self):
         h = hashlib.sha1()
@@ -80,7 +80,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
 
     # Update this if the database changes. Make sure to do a full rebuild
     # (e.g. 'make distclean && make') to get the correct checksum.
-    expectedchecksum = 'f891b1e6430c712531b9bc935a38e22d78ba1bf3'
+    expectedchecksum = 'db6f92bb5010f8e85000634b08e77233355ab37a'
     def test_function_checksum(self):
         data = []
         h = hashlib.sha1()

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.7.0 alpha 1?
 Core and Builtins
 -----------------
 
+- bpo-30736: The internal unicodedata database has been upgraded to Unicode
+  10.0.
+
 - bpo-30604:  Move co_extra_freefuncs from per-thread to per-interpreter to
   avoid crashes.
 

Modules/unicodedata.c

@@ -921,11 +921,12 @@ is_unified_ideograph(Py_UCS4 code)
 {
     return
         (0x3400 <= code && code <= 0x4DB5)   || /* CJK Ideograph Extension A */
-        (0x4E00 <= code && code <= 0x9FD5)   || /* CJK Ideograph */
+        (0x4E00 <= code && code <= 0x9FEA)   || /* CJK Ideograph */
         (0x20000 <= code && code <= 0x2A6D6) || /* CJK Ideograph Extension B */
         (0x2A700 <= code && code <= 0x2B734) || /* CJK Ideograph Extension C */
         (0x2B740 <= code && code <= 0x2B81D) || /* CJK Ideograph Extension D */
-        (0x2B820 <= code && code <= 0x2CEA1);   /* CJK Ideograph Extension E */
+        (0x2B820 <= code && code <= 0x2CEA1) || /* CJK Ideograph Extension E */
+        (0x2CEB0 <= code && code <= 0x2EBEF);   /* CJK Ideograph Extension F */
 }
 
 /* macros used to determine if the given code point is in the PUA range that

Modules/unicodedata_db.h

@@ -42,7 +42,7 @@
 #   * Doc/library/stdtypes.rst, and
 #   * Doc/library/unicodedata.rst
 #   * Doc/reference/lexical_analysis.rst (two occurrences)
-UNIDATA_VERSION = "9.0.0"
+UNIDATA_VERSION = "10.0.0"
 UNICODE_DATA = "UnicodeData%s.txt"
 COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt"
 EASTASIAN_WIDTH = "EastAsianWidth%s.txt"
@@ -99,11 +99,12 @@
 # these ranges need to match unicodedata.c:is_unified_ideograph
 cjk_ranges = [
     ('3400', '4DB5'),
-    ('4E00', '9FD5'),
+    ('4E00', '9FEA'),
     ('20000', '2A6D6'),
     ('2A700', '2B734'),
     ('2B740', '2B81D'),
     ('2B820', '2CEA1'),
+    ('2CEB0', '2EBE0'),
 ]
 
 def maketables(trace=0):
@@ -1262,12 +1263,12 @@ def dump(self, file, trace=0):
             for item in self.data:
                 i = str(item) + ", "
                 if len(s) + len(i) > 78:
-                    file.write(s + "\n")
+                    file.write(s.rstrip() + "\n")
                     s = "    " + i
                 else:
                     s = s + i
             if s.strip():
-                file.write(s + "\n")
+                file.write(s.rstrip() + "\n")
         file.write("};\n\n")
 
 def getsize(data):