Skip to content

Commit 2b2a568

Browse files
AzoyAzoy
authored
Merge pull request #40758 from Azoy/5.6-patch-1
[5.6] [stdlib] Use Apple's PUA Unicode data
2 parents 00f7685 + 2b76cf7 commit 2b2a568

25 files changed

+98605
-25190
lines changed

‎stdlib/public/stubs/CMakeLists.txt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@ set(swift_stubs_sources
66
Stubs.cpp
77
ThreadLocalStorage.cpp
88
MathStubs.cpp
9-
UnicodeData.cpp
10-
UnicodeGrapheme.cpp
11-
UnicodeNormalization.cpp
12-
UnicodeScalarProps.cpp
9+
Unicode/UnicodeData.cpp
10+
Unicode/UnicodeGrapheme.cpp
11+
Unicode/UnicodeNormalization.cpp
12+
Unicode/UnicodeScalarProps.cpp
1313
)
1414
set(swift_stubs_objc_sources
1515
Availability.mm

‎stdlib/public/stubs/Unicode/Apple/NormalizationData.h

Lines changed: 1627 additions & 0 deletions
Large diffs are not rendered by default.

‎stdlib/public/stubs/Unicode/Apple/ScalarPropsData.h

Lines changed: 24744 additions & 0 deletions
Large diffs are not rendered by default.

‎stdlib/public/stubs/Unicode/Common/GraphemeData.h

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2021 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
// This was auto-generated by utils/gen-unicode-data/GenGraphemeBreakProperty,
14+
// please do not edit this file yourself!
15+
16+
#ifndef GRAPHEME_DATA_H
17+
#define GRAPHEME_DATA_H
18+
19+
#include "../SwiftShims/SwiftStdint.h"
20+
21+
#define GRAPHEME_BREAK_DATA_COUNT 621
22+
23+
static __swift_uint32_t _swift_stdlib_graphemeBreakProperties[621] = {
24+
0x3E00000, 0x400007F, 0x800000A9, 0xAD, 0x800000AE, 0x2DE00300, 0x20C00483, 0x25800591,
25+
0x200005BF, 0x202005C1, 0x202005C4, 0x200005C7, 0x40A00600, 0x21400610, 0x61C, 0x2280064B,
26+
0x20000670, 0x20C006D6, 0x400006DD, 0x20A006DF, 0x202006E7, 0x206006EA, 0x4000070F, 0x20000711,
27+
0x23400730, 0x214007A6, 0x210007EB, 0x200007FD, 0x20600816, 0x2100081B, 0x20400825, 0x20800829,
28+
0x20400859, 0x40200890, 0x20E00898, 0x22E008CA, 0x400008E2, 0x23E008E3, 0x60000903, 0x2000093A,
29+
0x6000093B, 0x2000093C, 0x6040093E, 0x20E00941, 0x60600949, 0x2000094D, 0x6020094E, 0x20C00951,
30+
0x20200962, 0x20000981, 0x60200982, 0x200009BC, 0x200009BE, 0x602009BF, 0x206009C1, 0x602009C7,
31+
0x602009CB, 0x200009CD, 0x200009D7, 0x202009E2, 0x200009FE, 0x20200A01, 0x60000A03, 0x20000A3C,
32+
0x60400A3E, 0x20200A41, 0x20200A47, 0x20400A4B, 0x20000A51, 0x20200A70, 0x20000A75, 0x20200A81,
33+
0x60000A83, 0x20000ABC, 0x60400ABE, 0x20800AC1, 0x20200AC7, 0x60000AC9, 0x60200ACB, 0x20000ACD,
34+
0x20200AE2, 0x20A00AFA, 0x20000B01, 0x60200B02, 0x20000B3C, 0x20200B3E, 0x60000B40, 0x20600B41,
35+
0x60200B47, 0x60200B4B, 0x20000B4D, 0x20400B55, 0x20200B62, 0x20000B82, 0x20000BBE, 0x60000BBF,
36+
0x20000BC0, 0x60200BC1, 0x60400BC6, 0x60400BCA, 0x20000BCD, 0x20000BD7, 0x20000C00, 0x60400C01,
37+
0x20000C04, 0x20000C3C, 0x20400C3E, 0x60600C41, 0x20400C46, 0x20600C4A, 0x20200C55, 0x20200C62,
38+
0x20000C81, 0x60200C82, 0x20000CBC, 0x60000CBE, 0x20000CBF, 0x60200CC0, 0x20000CC2, 0x60200CC3,
39+
0x20000CC6, 0x60200CC7, 0x60200CCA, 0x20200CCC, 0x20200CD5, 0x20200CE2, 0x20200D00, 0x60200D02,
40+
0x20200D3B, 0x20000D3E, 0x60200D3F, 0x20600D41, 0x60400D46, 0x60400D4A, 0x20000D4D, 0x40000D4E,
41+
0x20000D57, 0x20200D62, 0x20000D81, 0x60200D82, 0x20000DCA, 0x20000DCF, 0x60200DD0, 0x20400DD2,
42+
0x20000DD6, 0x60C00DD8, 0x20000DDF, 0x60200DF2, 0x20000E31, 0x60000E33, 0x20C00E34, 0x20E00E47,
43+
0x20000EB1, 0x60000EB3, 0x21000EB4, 0x20A00EC8, 0x20200F18, 0x20000F35, 0x20000F37, 0x20000F39,
44+
0x60200F3E, 0x21A00F71, 0x60000F7F, 0x20800F80, 0x20200F86, 0x21400F8D, 0x24600F99, 0x20000FC6,
45+
0x2060102D, 0x60001031, 0x20A01032, 0x20201039, 0x6020103B, 0x2020103D, 0x60201056, 0x20201058,
46+
0x2040105E, 0x20601071, 0x20001082, 0x60001084, 0x20201085, 0x2000108D, 0x2000109D, 0x2040135D,
47+
0x20401712, 0x60001715, 0x20201732, 0x60001734, 0x20201752, 0x20201772, 0x202017B4, 0x600017B6,
48+
0x20C017B7, 0x60E017BE, 0x200017C6, 0x602017C7, 0x214017C9, 0x200017DD, 0x2040180B, 0x180E,
49+
0x2000180F, 0x20201885, 0x200018A9, 0x20401920, 0x60601923, 0x20201927, 0x60401929, 0x60201930,
50+
0x20001932, 0x60A01933, 0x20401939, 0x20201A17, 0x60201A19, 0x20001A1B, 0x60001A55, 0x20001A56,
51+
0x60001A57, 0x20C01A58, 0x20001A60, 0x20001A62, 0x20E01A65, 0x60A01A6D, 0x21201A73, 0x20001A7F,
52+
0x23C01AB0, 0x20601B00, 0x60001B04, 0x20C01B34, 0x60001B3B, 0x20001B3C, 0x60801B3D, 0x20001B42,
53+
0x60201B43, 0x21001B6B, 0x20201B80, 0x60001B82, 0x60001BA1, 0x20601BA2, 0x60201BA6, 0x20201BA8,
54+
0x60001BAA, 0x20401BAB, 0x20001BE6, 0x60001BE7, 0x20201BE8, 0x60401BEA, 0x20001BED, 0x60001BEE,
55+
0x20401BEF, 0x60201BF2, 0x60E01C24, 0x20E01C2C, 0x60201C34, 0x20201C36, 0x20401CD0, 0x21801CD4,
56+
0x60001CE1, 0x20C01CE2, 0x20001CED, 0x20001CF4, 0x60001CF7, 0x20201CF8, 0x27E01DC0, 0x200B,
57+
0x2000200C, 0x20200E, 0xC02028, 0x8000203C, 0x80002049, 0x1E02060, 0x240020D0, 0x80002122,
58+
0x80002139, 0x80A02194, 0x802021A9, 0x8020231A, 0x80002328, 0x80002388, 0x800023CF, 0x814023E9,
59+
0x804023F8, 0x800024C2, 0x802025AA, 0x800025B6, 0x800025C0, 0x806025FB, 0x80A02600, 0x81602607,
60+
0x8E202614, 0x8EA02690, 0x81402708, 0x80002714, 0x80002716, 0x8000271D, 0x80002721, 0x80002728,
61+
0x80202733, 0x80002744, 0x80002747, 0x8000274C, 0x8000274E, 0x80402753, 0x80002757, 0x80802763,
62+
0x80402795, 0x800027A1, 0x800027B0, 0x800027BF, 0x80202934, 0x80402B05, 0x80202B1B, 0x80002B50,
63+
0x80002B55, 0x20402CEF, 0x20002D7F, 0x23E02DE0, 0x20A0302A, 0x80003030, 0x8000303D, 0x20203099,
64+
0x80003297, 0x80003299, 0x2060A66F, 0x2120A674, 0x2020A69E, 0x2020A6F0, 0x2000A802, 0x2000A806,
65+
0x2000A80B, 0x6020A823, 0x2020A825, 0x6000A827, 0x2000A82C, 0x6020A880, 0x61E0A8B4, 0x2020A8C4,
66+
0x2220A8E0, 0x2000A8FF, 0x20E0A926, 0x2140A947, 0x6020A952, 0x2040A980, 0x6000A983, 0x2000A9B3,
67+
0x6020A9B4, 0x2060A9B6, 0x6020A9BA, 0x2020A9BC, 0x6040A9BE, 0x2000A9E5, 0x20A0AA29, 0x6020AA2F,
68+
0x2020AA31, 0x6020AA33, 0x2020AA35, 0x2000AA43, 0x2000AA4C, 0x6000AA4D, 0x2000AA7C, 0x2000AAB0,
69+
0x2040AAB2, 0x2020AAB7, 0x2020AABE, 0x2000AAC1, 0x6000AAEB, 0x2020AAEC, 0x6020AAEE, 0x6000AAF5,
70+
0x2000AAF6, 0x6020ABE3, 0x2000ABE5, 0x6020ABE6, 0x2000ABE8, 0x6020ABE9, 0x6000ABEC, 0x2000ABED,
71+
0x2000FB1E, 0x21E0FE00, 0x21E0FE20, 0xFEFF, 0x2020FF9E, 0x160FFF0, 0x200101FD, 0x200102E0,
72+
0x20810376, 0x20410A01, 0x20210A05, 0x20610A0C, 0x20410A38, 0x20010A3F, 0x20210AE5, 0x20610D24,
73+
0x20210EAB, 0x21410F46, 0x20610F82, 0x60011000, 0x20011001, 0x60011002, 0x21C11038, 0x20011070,
74+
0x20211073, 0x2041107F, 0x60011082, 0x604110B0, 0x206110B3, 0x602110B7, 0x202110B9, 0x400110BD,
75+
0x200110C2, 0x400110CD, 0x20411100, 0x20811127, 0x6001112C, 0x20E1112D, 0x60211145, 0x20011173,
76+
0x20211180, 0x60011182, 0x604111B3, 0x210111B6, 0x602111BF, 0x402111C2, 0x206111C9, 0x600111CE,
77+
0x200111CF, 0x6041122C, 0x2041122F, 0x60211232, 0x20011234, 0x60011235, 0x20211236, 0x2001123E,
78+
0x200112DF, 0x604112E0, 0x20E112E3, 0x20211300, 0x60211302, 0x2021133B, 0x2001133E, 0x6001133F,
79+
0x20011340, 0x60611341, 0x60211347, 0x6041134B, 0x20011357, 0x60211362, 0x20C11366, 0x20811370,
80+
0x60411435, 0x20E11438, 0x60211440, 0x20411442, 0x60011445, 0x20011446, 0x2001145E, 0x200114B0,
81+
0x602114B1, 0x20A114B3, 0x600114B9, 0x200114BA, 0x602114BB, 0x200114BD, 0x600114BE, 0x202114BF,
82+
0x600114C1, 0x202114C2, 0x200115AF, 0x602115B0, 0x206115B2, 0x606115B8, 0x202115BC, 0x600115BE,
83+
0x202115BF, 0x202115DC, 0x60411630, 0x20E11633, 0x6021163B, 0x2001163D, 0x6001163E, 0x2021163F,
84+
0x200116AB, 0x600116AC, 0x200116AD, 0x602116AE, 0x20A116B0, 0x600116B6, 0x200116B7, 0x2041171D,
85+
0x20611722, 0x60011726, 0x20811727, 0x6041182C, 0x2101182F, 0x60011838, 0x20211839, 0x20011930,
86+
0x60811931, 0x60211937, 0x2021193B, 0x6001193D, 0x2001193E, 0x4001193F, 0x60011940, 0x40011941,
87+
0x60011942, 0x20011943, 0x604119D1, 0x206119D4, 0x202119DA, 0x606119DC, 0x200119E0, 0x600119E4,
88+
0x21211A01, 0x20A11A33, 0x60011A39, 0x40011A3A, 0x20611A3B, 0x20011A47, 0x20A11A51, 0x60211A57,
89+
0x20411A59, 0x40A11A84, 0x21811A8A, 0x60011A97, 0x20211A98, 0x60011C2F, 0x20C11C30, 0x20A11C38,
90+
0x60011C3E, 0x20011C3F, 0x22A11C92, 0x60011CA9, 0x20C11CAA, 0x60011CB1, 0x20211CB2, 0x60011CB4,
91+
0x20211CB5, 0x20A11D31, 0x20011D3A, 0x20211D3C, 0x20C11D3F, 0x40011D46, 0x20011D47, 0x60811D8A,
92+
0x20211D90, 0x60211D93, 0x20011D95, 0x60011D96, 0x20011D97, 0x20211EF3, 0x60211EF5, 0x1013430,
93+
0x20816AF0, 0x20C16B30, 0x20016F4F, 0x66C16F51, 0x20616F8F, 0x20016FE4, 0x60216FF0, 0x2021BC9D,
94+
0x61BCA0, 0x25A1CF00, 0x22C1CF30, 0x2001D165, 0x6001D166, 0x2041D167, 0x6001D16D, 0x2081D16E,
95+
0xE1D173, 0x20E1D17B, 0x20C1D185, 0x2061D1AA, 0x2041D242, 0x26C1DA00, 0x2621DA3B, 0x2001DA75,
96+
0x2001DA84, 0x2081DA9B, 0x21C1DAA1, 0x20C1E000, 0x2201E008, 0x20C1E01B, 0x2021E023, 0x2081E026,
97+
0x20C1E130, 0x2001E2AE, 0x2061E2EC, 0x20C1E8D0, 0x20C1E944, 0x9FE1F000, 0x8041F10D, 0x8001F12F,
98+
0x80A1F16C, 0x8021F17E, 0x8001F18E, 0x8121F191, 0x8701F1AD, 0x81C1F201, 0x8001F21A, 0x8001F22F,
99+
0x8101F232, 0x8061F23C, 0xB621F249, 0x2081F3FB, 0xA7A1F400, 0xA121F546, 0x8FE1F680, 0x8161F774,
100+
0x8541F7D5, 0x8061F80C, 0x80E1F848, 0x80A1F85A, 0x80E1F888, 0x8A21F8AE, 0x85C1F90C, 0x8121F93C,
101+
0xB701F947, 0x3EE0000, 0x2BEE0020, 0xFEE0080, 0x3DEE0100,
102+
};
103+
104+
#endif // #ifndef GRAPHEME_DATA_H

‎stdlib/public/stubs/UnicodeNormalization.cpp renamed to ‎stdlib/public/stubs/Unicode/Common/NormalizationData.h

Lines changed: 7 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@
1313
// This was auto-generated by utils/gen-unicode-data/GenNormalization,
1414
// please do not edit this file yourself!
1515

16-
#include "../SwiftShims/UnicodeData.h"
17-
#include <limits>
16+
#ifndef NORMALIZATION_DATA_H
17+
#define NORMALIZATION_DATA_H
18+
19+
#include "../SwiftShims/SwiftStdint.h"
1820

1921
static const __swift_uint16_t _swift_stdlib_normData_data[66] = {
2022
0x110, 0x70, 0x3B0, 0x8, 0x750, 0x78, 0x6C4, 0x410, 0x6E0, 0x60, 0x734, 0x784,
@@ -343,27 +345,7 @@ static const __swift_uint64_t _swift_stdlib_normData[403] = {
343345
0xFFFFFFFFFFFFFFFF, 0x3FFF, 0x0, 0x0, 0xB930000,
344346
};
345347

346-
SWIFT_RUNTIME_STDLIB_INTERNAL
347-
__swift_uint16_t _swift_stdlib_getNormData(__swift_uint32_t scalar) {
348-
// Fast Path: ASCII and some latiny scalars are very basic and have no
349-
// normalization properties.
350-
if (scalar < 0xC0) {
351-
return 0;
352-
}
353-
354-
auto dataIdx = _swift_stdlib_getScalarBitArrayIdx(scalar,
355-
_swift_stdlib_normData,
356-
_swift_stdlib_normData_ranks);
357-
358-
// If we don't have an index into the data indices, then this scalar has no
359-
// normalization information.
360-
if (dataIdx == std::numeric_limits<__swift_intptr_t>::max()) {
361-
return 0;
362-
}
363-
364-
auto scalarDataIdx = _swift_stdlib_normData_data_indices[dataIdx];
365-
return _swift_stdlib_normData_data[scalarDataIdx];
366-
}
348+
#define NFD_DECOMP_LEVEL_COUNT 11
367349

368350
static const __swift_uint16_t _swift_stdlib_nfd_decomp_sizes[11] = {
369351
0x80D, 0x4F8, 0x32E, 0x1EF, 0x137, 0xC3, 0x7B, 0x54, 0x40, 0x40, 0x40,
@@ -1565,18 +1547,7 @@ static const __swift_uint32_t _swift_stdlib_nfd_decomp_indices[2061] = {
15651547
0x16681F7D,
15661548
};
15671549

1568-
SWIFT_RUNTIME_STDLIB_INTERNAL
1569-
const __swift_uint8_t * const _swift_stdlib_nfd_decompositions = _swift_stdlib_nfd_decomp;
1570-
1571-
SWIFT_RUNTIME_STDLIB_INTERNAL
1572-
__swift_uint32_t _swift_stdlib_getDecompositionEntry(__swift_uint32_t scalar) {
1573-
__swift_intptr_t decompIdx = _swift_stdlib_getMphIdx(scalar, 11,
1574-
_swift_stdlib_nfd_decomp_keys,
1575-
_swift_stdlib_nfd_decomp_ranks,
1576-
_swift_stdlib_nfd_decomp_sizes);
1577-
1578-
return _swift_stdlib_nfd_decomp_indices[decompIdx];
1579-
}
1550+
#define NFC_COMP_LEVEL_COUNT 3
15801551

15811552
static const __swift_uint16_t _swift_stdlib_nfc_comp_sizes[3] = {
15821553
0x40, 0x40, 0x40,
@@ -2011,61 +1982,4 @@ static const __swift_uint32_t * const _swift_stdlib_nfc_comp_indices[63] = {
20111982
_swift_stdlib_nfc_comp60, _swift_stdlib_nfc_comp61, _swift_stdlib_nfc_comp62,
20121983
};
20131984

2014-
SWIFT_RUNTIME_STDLIB_INTERNAL
2015-
__swift_uint32_t _swift_stdlib_getComposition(__swift_uint32_t x,
2016-
__swift_uint32_t y) {
2017-
__swift_intptr_t compIdx = _swift_stdlib_getMphIdx(y, 3,
2018-
_swift_stdlib_nfc_comp_keys,
2019-
_swift_stdlib_nfc_comp_ranks,
2020-
_swift_stdlib_nfc_comp_sizes);
2021-
auto array = _swift_stdlib_nfc_comp_indices[compIdx];
2022-
2023-
// Ensure that the first element in this array is equal to our y scalar.
2024-
auto realY = (array[0] << 11) >> 11;
2025-
2026-
if (y != realY) {
2027-
return std::numeric_limits<__swift_uint32_t>::max();
2028-
}
2029-
2030-
auto count = array[0] >> 21;
2031-
2032-
__swift_uint32_t low = 1;
2033-
__swift_uint32_t high = count - 1;
2034-
2035-
while (high >= low) {
2036-
auto idx = low + (high - low) / 2;
2037-
2038-
auto entry = array[idx];
2039-
2040-
// Shift the range count out of the scalar.
2041-
auto lower = (entry << 15) >> 15;
2042-
2043-
bool isNegative = entry >> 31;
2044-
auto rangeCount = (entry << 1) >> 18;
2045-
2046-
if (isNegative) {
2047-
rangeCount = -rangeCount;
2048-
}
2049-
2050-
auto composed = lower + rangeCount;
2051-
2052-
if (x == lower) {
2053-
return composed;
2054-
}
2055-
2056-
if (x > lower) {
2057-
low = idx + 1;
2058-
continue;
2059-
}
2060-
2061-
if (x < lower) {
2062-
high = idx - 1;
2063-
continue;
2064-
}
2065-
}
2066-
2067-
// If we made it out here, then our scalar was not found in the composition
2068-
// array.
2069-
// Return the max here to indicate that we couldn't find one.
2070-
return std::numeric_limits<__swift_uint32_t>::max();
2071-
}
1985+
#endif // #ifndef NORMALIZATION_DATA_H

0 commit comments

Comments
 (0)