Skip to content

Commit a26215d

Browse files
bpo-42318: Fix support of non-BMP characters in Tkinter on macOS (GH-23281)
1 parent 7a27c7e commit a26215d

File tree

3 files changed

+94
-7
lines changed

3 files changed

+94
-7
lines changed

Lib/test/test_tcl.py

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import unittest
2+
import locale
23
import re
34
import subprocess
45
import sys
@@ -61,6 +62,10 @@ def test_eval_null_in_result(self):
6162
tcl = self.interp
6263
self.assertEqual(tcl.eval('set a "a\\0b"'), 'a\x00b')
6364

65+
def test_eval_surrogates_in_result(self):
66+
tcl = self.interp
67+
self.assertIn(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>')
68+
6469
def testEvalException(self):
6570
tcl = self.interp
6671
self.assertRaises(TclError,tcl.eval,'set a')
@@ -193,29 +198,48 @@ def test_getboolean(self):
193198

194199
def testEvalFile(self):
195200
tcl = self.interp
196-
with open(os_helper.TESTFN, 'w') as f:
197-
self.addCleanup(os_helper.unlink, os_helper.TESTFN)
201+
filename = os_helper.TESTFN_ASCII
202+
self.addCleanup(os_helper.unlink, filename)
203+
with open(filename, 'w') as f:
198204
f.write("""set a 1
199205
set b 2
200206
set c [ expr $a + $b ]
201207
""")
202-
tcl.evalfile(os_helper.TESTFN)
208+
tcl.evalfile(filename)
203209
self.assertEqual(tcl.eval('set a'),'1')
204210
self.assertEqual(tcl.eval('set b'),'2')
205211
self.assertEqual(tcl.eval('set c'),'3')
206212

207213
def test_evalfile_null_in_result(self):
208214
tcl = self.interp
209-
with open(os_helper.TESTFN, 'w') as f:
210-
self.addCleanup(os_helper.unlink, os_helper.TESTFN)
215+
filename = os_helper.TESTFN_ASCII
216+
self.addCleanup(os_helper.unlink, filename)
217+
with open(filename, 'w') as f:
211218
f.write("""
212219
set a "a\0b"
213220
set b "a\\0b"
214221
""")
215-
tcl.evalfile(os_helper.TESTFN)
222+
tcl.evalfile(filename)
216223
self.assertEqual(tcl.eval('set a'), 'a\x00b')
217224
self.assertEqual(tcl.eval('set b'), 'a\x00b')
218225

226+
def test_evalfile_surrogates_in_result(self):
227+
tcl = self.interp
228+
encoding = tcl.call('encoding', 'system')
229+
self.addCleanup(tcl.call, 'encoding', 'system', encoding)
230+
tcl.call('encoding', 'system', 'utf-8')
231+
232+
filename = os_helper.TESTFN_ASCII
233+
self.addCleanup(os_helper.unlink, filename)
234+
with open(filename, 'wb') as f:
235+
f.write(b"""
236+
set a "<\xed\xa0\xbd\xed\xb2\xbb>"
237+
set b "<\\ud83d\\udcbb>"
238+
""")
239+
tcl.evalfile(filename)
240+
self.assertEqual(tcl.eval('set a'), '<\U0001f4bb>')
241+
self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>')
242+
219243
def testEvalFileException(self):
220244
tcl = self.interp
221245
filename = "doesnotexists"
@@ -438,6 +462,11 @@ def passValue(value):
438462
self.assertEqual(passValue('str\x00ing\u20ac'), 'str\x00ing\u20ac')
439463
self.assertEqual(passValue('str\x00ing\U0001f4bb'),
440464
'str\x00ing\U0001f4bb')
465+
if sys.platform != 'win32':
466+
self.assertEqual(passValue('<\udce2\udc82\udcac>'),
467+
'<\u20ac>')
468+
self.assertEqual(passValue('<\udced\udca0\udcbd\udced\udcb2\udcbb>'),
469+
'<\U0001f4bb>')
441470
self.assertEqual(passValue(b'str\x00ing'),
442471
b'str\x00ing' if self.wantobjects else 'str\x00ing')
443472
self.assertEqual(passValue(b'str\xc0\x80ing'),
@@ -497,6 +526,9 @@ def float_eq(actual, expected):
497526
check('string\xbd')
498527
check('string\u20ac')
499528
check('string\U0001f4bb')
529+
if sys.platform != 'win32':
530+
check('<\udce2\udc82\udcac>', '<\u20ac>')
531+
check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>')
500532
check('')
501533
check(b'string', 'string')
502534
check(b'string\xe2\x82\xac', 'string\xe2\x82\xac')
@@ -540,6 +572,8 @@ def test_splitlist(self):
540572
('a \u20ac', ('a', '\u20ac')),
541573
('a \U0001f4bb', ('a', '\U0001f4bb')),
542574
(b'a \xe2\x82\xac', ('a', '\u20ac')),
575+
(b'a \xf0\x9f\x92\xbb', ('a', '\U0001f4bb')),
576+
(b'a \xed\xa0\xbd\xed\xb2\xbb', ('a', '\U0001f4bb')),
543577
(b'a\xc0\x80b c\xc0\x80d', ('a\x00b', 'c\x00d')),
544578
('a {b c}', ('a', 'b c')),
545579
(r'a b\ c', ('a', 'b c')),
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed support of non-BMP characters in :mod:`tkinter` on macOS.

Modules/_tkinter.c

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,8 @@ unicodeFromTclStringAndSize(const char *s, Py_ssize_t size)
395395

396396
char *buf = NULL;
397397
PyErr_Clear();
398-
/* Tcl encodes null character as \xc0\x80 */
398+
/* Tcl encodes null character as \xc0\x80.
399+
https://en.wikipedia.org/wiki/UTF-8#Modified_UTF-8 */
399400
if (memchr(s, '\xc0', size)) {
400401
char *q;
401402
const char *e = s + size;
@@ -419,6 +420,57 @@ unicodeFromTclStringAndSize(const char *s, Py_ssize_t size)
419420
if (buf != NULL) {
420421
PyMem_Free(buf);
421422
}
423+
if (r == NULL || PyUnicode_KIND(r) == PyUnicode_1BYTE_KIND) {
424+
return r;
425+
}
426+
427+
/* In CESU-8 non-BMP characters are represented as a surrogate pair,
428+
like in UTF-16, and then each surrogate code point is encoded in UTF-8.
429+
https://en.wikipedia.org/wiki/CESU-8 */
430+
Py_ssize_t len = PyUnicode_GET_LENGTH(r);
431+
Py_ssize_t i, j;
432+
/* All encoded surrogate characters start with \xED. */
433+
i = PyUnicode_FindChar(r, 0xdcED, 0, len, 1);
434+
if (i == -2) {
435+
Py_DECREF(r);
436+
return NULL;
437+
}
438+
if (i == -1) {
439+
return r;
440+
}
441+
Py_UCS4 *u = PyUnicode_AsUCS4Copy(r);
442+
Py_DECREF(r);
443+
if (u == NULL) {
444+
return NULL;
445+
}
446+
Py_UCS4 ch;
447+
for (j = i; i < len; i++, u[j++] = ch) {
448+
Py_UCS4 ch1, ch2, ch3, high, low;
449+
/* Low surrogates U+D800 - U+DBFF are encoded as
450+
\xED\xA0\x80 - \xED\xAF\xBF. */
451+
ch1 = ch = u[i];
452+
if (ch1 != 0xdcED) continue;
453+
ch2 = u[i + 1];
454+
if (!(0xdcA0 <= ch2 && ch2 <= 0xdcAF)) continue;
455+
ch3 = u[i + 2];
456+
if (!(0xdc80 <= ch3 && ch3 <= 0xdcBF)) continue;
457+
high = 0xD000 | ((ch2 & 0x3F) << 6) | (ch3 & 0x3F);
458+
assert(Py_UNICODE_IS_HIGH_SURROGATE(high));
459+
/* High surrogates U+DC00 - U+DFFF are encoded as
460+
\xED\xB0\x80 - \xED\xBF\xBF. */
461+
ch1 = u[i + 3];
462+
if (ch1 != 0xdcED) continue;
463+
ch2 = u[i + 4];
464+
if (!(0xdcB0 <= ch2 && ch2 <= 0xdcBF)) continue;
465+
ch3 = u[i + 5];
466+
if (!(0xdc80 <= ch3 && ch3 <= 0xdcBF)) continue;
467+
low = 0xD000 | ((ch2 & 0x3F) << 6) | (ch3 & 0x3F);
468+
assert(Py_UNICODE_IS_HIGH_SURROGATE(high));
469+
ch = Py_UNICODE_JOIN_SURROGATES(high, low);
470+
i += 5;
471+
}
472+
r = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, u, j);
473+
PyMem_Free(u);
422474
return r;
423475
}
424476

0 commit comments

Comments
 (0)