recipes: add patches for python3 recipes, for reproducible .pyc generation

SomberNight · SomberNight · commit 0ab0d872e6c6 · 2024-10-15T23:28:45.000Z
patches from: python/cpython#27926 python/cpython#8226 Both of these are included in python 3.11 upstream.
diff --git a/pythonforandroid/recipes/hostpython3/__init__.py b/pythonforandroid/recipes/hostpython3/__init__.py
@@ -6,6 +6,7 @@
 from os.path import join
 
 from pythonforandroid.logger import shprint
+from pythonforandroid.patching import version_starts_with
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.util import (
     BuildInterruptingException,
@@ -46,7 +47,11 @@ class HostPython3Recipe(Recipe):
     '''The default url to download our host python recipe. This url will
     change depending on the python version set in attribute :attr:`version`.'''
 
-    patches = ['patches/pyconfig_detection.patch']
+    patches = [
+        'patches/pyconfig_detection.patch',
+        ('patches/py3.10_reproducible-pyc.diff', version_starts_with("3.10")),
+        ('patches/py3.10_reproducible-marshal_flagref.patch', version_starts_with("3.10")),
+    ]
 
     @property
     def _exe_name(self):
diff --git a/pythonforandroid/recipes/hostpython3/patches/py3.10_reproducible-marshal_flagref.patch b/pythonforandroid/recipes/hostpython3/patches/py3.10_reproducible-marshal_flagref.patch
@@ -0,0 +1,30 @@
+From 6c8ea7c1dacd42f3ba00440231ec0e6b1a38300d Mon Sep 17 00:00:00 2001
+From: Inada Naoki <songofacandy@gmail.com>
+Date: Sat, 14 Jul 2018 00:46:11 +0900
+Subject: [PATCH] Use FLAG_REF always for interned strings
+
+---
+ Python/marshal.c | 9 +++++++--
+ 1 file changed, 7 insertions(+), 2 deletions(-)
+
+diff --git a/Python/marshal.c b/Python/marshal.c
+index 6d06266c6a8e2e..51db2e3b2e29a2 100644
+--- a/Python/marshal.c
++++ b/Python/marshal.c
+@@ -275,9 +275,14 @@ w_ref(PyObject *v, char *flag, WFILE *p)
+     if (p->version < 3 || p->hashtable == NULL)
+         return 0; /* not writing object references */
+ 
+-    /* if it has only one reference, it definitely isn't shared */
+-    if (Py_REFCNT(v) == 1)
++    /* If it has only one reference, it definitely isn't shared.
++     * But we use TYPE_REF always for interned string, to PYC file stable
++     * as possible.
++     */
++    if (Py_REFCNT(v) == 1 &&
++            !(PyUnicode_CheckExact(v) && PyUnicode_CHECK_INTERNED(v))) {
+         return 0;
++    }
+ 
+     entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v);
+     if (entry != NULL) {
diff --git a/pythonforandroid/recipes/hostpython3/patches/py3.10_reproducible-pyc.diff b/pythonforandroid/recipes/hostpython3/patches/py3.10_reproducible-pyc.diff
@@ -0,0 +1,97 @@
+From 36ae9beb04763d498df2114657bfbbcfe58bf913 Mon Sep 17 00:00:00 2001
+From: Brandt Bucher <brandt@python.org>
+Date: Mon, 23 Aug 2021 18:34:17 -0700
+Subject: [PATCH] Serialize frozenset elements deterministically
+
+---
+ Lib/test/test_marshal.py                      | 25 +++++++++++++++
+ .../2021-08-23-21-39-59.bpo-37596.ojRcwB.rst  |  2 ++
+ Python/marshal.c                              | 32 +++++++++++++++++++
+ 3 files changed, 59 insertions(+)
+ create mode 100644 Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst
+
+--- a/Lib/test/test_marshal.py
++++ b/Lib/test/test_marshal.py
+@@ -1,5 +1,6 @@
+ from test import support
+ from test.support import os_helper
++from test.support.script_helper import assert_python_ok
+ import array
+ import io
+ import marshal
+@@ -318,6 +319,31 @@ class BugsTestCase(unittest.TestCase):
+         for i in range(len(data)):
+             self.assertRaises(EOFError, marshal.loads, data[0: i])
+ 
++    def test_deterministic_sets(self):
++        # bpo-37596: To support reproducible builds, sets and frozensets need to
++        # have their elements serialized in a consistent order (even when they
++        # have been scrambled by hash randomization):
++        for kind in ("set", "frozenset"):
++            for elements in (
++                "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'",
++                # Also test for bad interactions with backreferencing:
++                "('string', 1), ('string', 2), ('string', 3)",
++            ):
++                s = f"{kind}([{elements}])"
++                with self.subTest(s):
++                    # First, make sure that our test case still has different
++                    # orders under hash seeds 0 and 1. If this check fails, we
++                    # need to update this test with different elements:
++                    args = ["-c", f"print({s})"]
++                    _, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
++                    _, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
++                    self.assertNotEqual(repr_0, repr_1)
++                    # Then, perform the actual test:
++                    args = ["-c", f"import marshal; print(marshal.dumps({s}))"]
++                    _, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
++                    _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
++                    self.assertEqual(dump_0, dump_1)
++
+ LARGE_SIZE = 2**31
+ pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
+ 
+--- a/Python/marshal.c
++++ b/Python/marshal.c
+@@ -502,9 +502,41 @@ w_complex_object(PyObject *v, char flag,
+             W_TYPE(TYPE_SET, p);
+         n = PySet_GET_SIZE(v);
+         W_SIZE(n, p);
++        // bpo-37596: To support reproducible builds, sets and frozensets need
++        // to have their elements serialized in a consistent order (even when
++        // they have been scrambled by hash randomization). To ensure this, we
++        // use an order equivalent to sorted(v, key=marshal.dumps):
++        PyObject *pairs = PyList_New(0);
++        if (pairs == NULL) {
++            p->error = WFERR_NOMEMORY;
++            return;
++        }
+         while (_PySet_NextEntry(v, &pos, &value, &hash)) {
++            PyObject *dump = PyMarshal_WriteObjectToString(value, p->version);
++            if (dump == NULL) {
++                p->error = WFERR_UNMARSHALLABLE;
++                goto anyset_done;
++            }
++            PyObject *pair = PyTuple_Pack(2, dump, value);
++            Py_DECREF(dump);
++            if (pair == NULL || PyList_Append(pairs, pair)) {
++                p->error = WFERR_NOMEMORY;
++                Py_XDECREF(pair);
++                goto anyset_done;
++            }
++            Py_DECREF(pair);
++        }
++        if (PyList_Sort(pairs)) {
++            p->error = WFERR_NOMEMORY;
++            goto anyset_done;
++        }
++        for (Py_ssize_t i = 0; i < n; i++) {
++            PyObject *pair = PyList_GET_ITEM(pairs, i);
++            value = PyTuple_GET_ITEM(pair, 1);
+             w_object(value, p);
+         }
++    anyset_done:
++        Py_DECREF(pairs);
+     }
+     else if (PyCode_Check(v)) {
+         PyCodeObject *co = (PyCodeObject *)v;
diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py
@@ -71,7 +71,10 @@ class Python3Recipe(TargetPythonRecipe):
         # Python 3.8.1 & 3.9.X
         ('patches/py3.8.1.patch', version_starts_with("3.8")),
         ('patches/py3.8.1.patch', version_starts_with("3.9")),
-        ('patches/py3.8.1.patch', version_starts_with("3.10"))
+        ('patches/py3.8.1.patch', version_starts_with("3.10")),
+
+        ('patches/py3.10_reproducible-pyc.diff', version_starts_with("3.10")),
+        ('patches/py3.10_reproducible-marshal_flagref.patch', version_starts_with("3.10")),
     ]
 
     if shutil.which('lld') is not None:
diff --git a/pythonforandroid/recipes/python3/patches/py3.10_reproducible-marshal_flagref.patch b/pythonforandroid/recipes/python3/patches/py3.10_reproducible-marshal_flagref.patch
@@ -0,0 +1,30 @@
+From 6c8ea7c1dacd42f3ba00440231ec0e6b1a38300d Mon Sep 17 00:00:00 2001
+From: Inada Naoki <songofacandy@gmail.com>
+Date: Sat, 14 Jul 2018 00:46:11 +0900
+Subject: [PATCH] Use FLAG_REF always for interned strings
+
+---
+ Python/marshal.c | 9 +++++++--
+ 1 file changed, 7 insertions(+), 2 deletions(-)
+
+diff --git a/Python/marshal.c b/Python/marshal.c
+index 6d06266c6a8e2e..51db2e3b2e29a2 100644
+--- a/Python/marshal.c
++++ b/Python/marshal.c
+@@ -275,9 +275,14 @@ w_ref(PyObject *v, char *flag, WFILE *p)
+     if (p->version < 3 || p->hashtable == NULL)
+         return 0; /* not writing object references */
+ 
+-    /* if it has only one reference, it definitely isn't shared */
+-    if (Py_REFCNT(v) == 1)
++    /* If it has only one reference, it definitely isn't shared.
++     * But we use TYPE_REF always for interned string, to PYC file stable
++     * as possible.
++     */
++    if (Py_REFCNT(v) == 1 &&
++            !(PyUnicode_CheckExact(v) && PyUnicode_CHECK_INTERNED(v))) {
+         return 0;
++    }
+ 
+     entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v);
+     if (entry != NULL) {
diff --git a/pythonforandroid/recipes/python3/patches/py3.10_reproducible-pyc.diff b/pythonforandroid/recipes/python3/patches/py3.10_reproducible-pyc.diff
@@ -0,0 +1,97 @@
+From 36ae9beb04763d498df2114657bfbbcfe58bf913 Mon Sep 17 00:00:00 2001
+From: Brandt Bucher <brandt@python.org>
+Date: Mon, 23 Aug 2021 18:34:17 -0700
+Subject: [PATCH] Serialize frozenset elements deterministically
+
+---
+ Lib/test/test_marshal.py                      | 25 +++++++++++++++
+ .../2021-08-23-21-39-59.bpo-37596.ojRcwB.rst  |  2 ++
+ Python/marshal.c                              | 32 +++++++++++++++++++
+ 3 files changed, 59 insertions(+)
+ create mode 100644 Misc/NEWS.d/next/Library/2021-08-23-21-39-59.bpo-37596.ojRcwB.rst
+
+--- a/Lib/test/test_marshal.py
++++ b/Lib/test/test_marshal.py
+@@ -1,5 +1,6 @@
+ from test import support
+ from test.support import os_helper
++from test.support.script_helper import assert_python_ok
+ import array
+ import io
+ import marshal
+@@ -318,6 +319,31 @@ class BugsTestCase(unittest.TestCase):
+         for i in range(len(data)):
+             self.assertRaises(EOFError, marshal.loads, data[0: i])
+ 
++    def test_deterministic_sets(self):
++        # bpo-37596: To support reproducible builds, sets and frozensets need to
++        # have their elements serialized in a consistent order (even when they
++        # have been scrambled by hash randomization):
++        for kind in ("set", "frozenset"):
++            for elements in (
++                "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'",
++                # Also test for bad interactions with backreferencing:
++                "('string', 1), ('string', 2), ('string', 3)",
++            ):
++                s = f"{kind}([{elements}])"
++                with self.subTest(s):
++                    # First, make sure that our test case still has different
++                    # orders under hash seeds 0 and 1. If this check fails, we
++                    # need to update this test with different elements:
++                    args = ["-c", f"print({s})"]
++                    _, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
++                    _, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
++                    self.assertNotEqual(repr_0, repr_1)
++                    # Then, perform the actual test:
++                    args = ["-c", f"import marshal; print(marshal.dumps({s}))"]
++                    _, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
++                    _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
++                    self.assertEqual(dump_0, dump_1)
++
+ LARGE_SIZE = 2**31
+ pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
+ 
+--- a/Python/marshal.c
++++ b/Python/marshal.c
+@@ -502,9 +502,41 @@ w_complex_object(PyObject *v, char flag,
+             W_TYPE(TYPE_SET, p);
+         n = PySet_GET_SIZE(v);
+         W_SIZE(n, p);
++        // bpo-37596: To support reproducible builds, sets and frozensets need
++        // to have their elements serialized in a consistent order (even when
++        // they have been scrambled by hash randomization). To ensure this, we
++        // use an order equivalent to sorted(v, key=marshal.dumps):
++        PyObject *pairs = PyList_New(0);
++        if (pairs == NULL) {
++            p->error = WFERR_NOMEMORY;
++            return;
++        }
+         while (_PySet_NextEntry(v, &pos, &value, &hash)) {
++            PyObject *dump = PyMarshal_WriteObjectToString(value, p->version);
++            if (dump == NULL) {
++                p->error = WFERR_UNMARSHALLABLE;
++                goto anyset_done;
++            }
++            PyObject *pair = PyTuple_Pack(2, dump, value);
++            Py_DECREF(dump);
++            if (pair == NULL || PyList_Append(pairs, pair)) {
++                p->error = WFERR_NOMEMORY;
++                Py_XDECREF(pair);
++                goto anyset_done;
++            }
++            Py_DECREF(pair);
++        }
++        if (PyList_Sort(pairs)) {
++            p->error = WFERR_NOMEMORY;
++            goto anyset_done;
++        }
++        for (Py_ssize_t i = 0; i < n; i++) {
++            PyObject *pair = PyList_GET_ITEM(pairs, i);
++            value = PyTuple_GET_ITEM(pair, 1);
+             w_object(value, p);
+         }
++    anyset_done:
++        Py_DECREF(pairs);
+     }
+     else if (PyCode_Check(v)) {
+         PyCodeObject *co = (PyCodeObject *)v;
