Add _PyInterpreterConfig.check_multi_interp_extensions and Py_RTFLAGS_MULTI_INTERP_EXTENSIONS.

Author: ericsnowcurrently

Include/cpython/initconfig.h

@@ -248,6 +248,7 @@ typedef struct {
     int allow_exec;
     int allow_threads;
     int allow_daemon_threads;
+    int check_multi_interp_extensions;
 } _PyInterpreterConfig;
 
 #define _PyInterpreterConfig_INIT \
@@ -256,6 +257,7 @@ typedef struct {
         .allow_exec = 0, \
         .allow_threads = 1, \
         .allow_daemon_threads = 0, \
+        .check_multi_interp_extensions = 1, \
     }
 
 #define _PyInterpreterConfig_LEGACY_INIT \
@@ -264,6 +266,7 @@ typedef struct {
         .allow_exec = 1, \
         .allow_threads = 1, \
         .allow_daemon_threads = 1, \
+        .check_multi_interp_extensions = 0, \
     }
 
 /* --- Helper functions --------------------------------------- */

Include/cpython/pystate.h

@@ -11,6 +11,9 @@ is available in a given context.  For example, forking the process
 might not be allowed in the current interpreter (i.e. os.fork() would fail).
 */
 
+/* Set if import should check a module for subinterpreter support. */
+#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8)
+
 /* Set if threads are allowed. */
 #define Py_RTFLAGS_THREADS (1UL << 10)
 

Lib/test/test_capi.py

@@ -1147,17 +1147,20 @@ def test_configured_settings(self):
         """
         import json
 
+        EXTENSIONS = 1<<8
         THREADS = 1<<10
         DAEMON_THREADS = 1<<11
         FORK = 1<<15
         EXEC = 1<<16
 
-        features = ['fork', 'exec', 'threads', 'daemon_threads']
+        features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions']
         kwlist = [f'allow_{n}' for n in features]
+        kwlist[-1] = 'check_multi_interp_extensions'
         for config, expected in {
-            (True, True, True, True): FORK | EXEC | THREADS | DAEMON_THREADS,
-            (False, False, False, False): 0,
-            (False, False, True, False): THREADS,
+            (True, True, True, True, True):
+                FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
+            (False, False, False, False, False): 0,
+            (False, False, True, False, True): THREADS | EXTENSIONS,
         }.items():
             kwargs = dict(zip(kwlist, config))
             expected = {

Lib/test/test_embed.py

@@ -1654,13 +1654,15 @@ def test_init_use_frozen_modules(self):
                                        api=API_PYTHON, env=env)
 
     def test_init_main_interpreter_settings(self):
+        EXTENSIONS = 1<<8
         THREADS = 1<<10
         DAEMON_THREADS = 1<<11
         FORK = 1<<15
         EXEC = 1<<16
         expected = {
             # All optional features should be enabled.
-            'feature_flags': FORK | EXEC | THREADS | DAEMON_THREADS,
+            'feature_flags':
+                FORK | EXEC | THREADS | DAEMON_THREADS,
         }
         out, err = self.run_embedded_interpreter(
             'test_init_main_interpreter_settings',

Lib/test/test_import/__init__.py

@@ -1433,13 +1433,17 @@ def import_script(self, name, fd):
             os.write({fd}, text.encode('utf-8'))
             ''')
 
-    def check_compatible_shared(self, name):
+    def check_compatible_shared(self, name, *, strict=False):
         # Verify that the named module may be imported in a subinterpreter.
         #
         # The subinterpreter will be in the current process.
         # The module will have already been imported in the main interpreter.
         # Thus, for extension/builtin modules, the module definition will
         # have been loaded already and cached globally.
+        #
+        # "strict" determines whether or not the interpreter will be
+        # configured to check for modules that are not compatible
+        # with use in multiple interpreters.
 
         # This check should always pass for all modules if not strict.
 
@@ -1449,12 +1453,13 @@ def check_compatible_shared(self, name):
         ret = run_in_subinterp_with_config(
             self.import_script(name, w),
             **self.RUN_KWARGS,
+            check_multi_interp_extensions=strict,
         )
         self.assertEqual(ret, 0)
         out = os.read(r, 100)
         self.assertEqual(out, b'okay')
 
-    def check_compatible_isolated(self, name):
+    def check_compatible_isolated(self, name, *, strict=False):
         # Differences from check_compatible_shared():
         #  * subinterpreter in a new process
         #  * module has never been imported before in that process
@@ -1468,67 +1473,60 @@ def check_compatible_isolated(self, name):
             ret = _testcapi.run_in_subinterp_with_config(
                 {self.import_script(name, "sys.stdout.fileno()")!r},
                 **{self.RUN_KWARGS},
+                check_multi_interp_extensions={strict},
             )
             assert ret == 0, ret
             '''))
         self.assertEqual(err, b'')
         self.assertEqual(out, b'okay')
 
-    def check_incompatible_isolated(self, name):
-        # Differences from check_compatible_isolated():
-        #  * verify that import fails
-        _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
-            import _testcapi, sys
-            assert {name!r} not in sys.modules, {name!r}
-            ret = _testcapi.run_in_subinterp_with_config(
-                {self.import_script(name, "sys.stdout.fileno()")!r},
-                **{self.RUN_KWARGS},
-            )
-            assert ret == 0, ret
-            '''))
-        self.assertEqual(err, b'')
-        self.assertEqual(
-            out.decode('utf-8'),
-            f'ImportError: module {name} does not support loading in subinterpreters',
-        )
-
     def test_builtin_compat(self):
         module = 'sys'
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
 
     @cpython_only
     def test_frozen_compat(self):
         module = '_frozen_importlib'
         if __import__(module).__spec__.origin != 'frozen':
             raise unittest.SkipTest(f'{module} is unexpectedly not frozen')
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
 
-    @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
+    @unittest.skipIf(_testsinglephase is None, "test requires _testsinglphase module")
     def test_single_init_extension_compat(self):
         module = '_testsinglephase'
-        with self.subTest(f'{module}: shared'):
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
             self.check_compatible_shared(module)
-        with self.subTest(f'{module}: isolated'):
+        with self.subTest(f'{module}: strict, isolated'):
             self.check_compatible_isolated(module)
 
     @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
     def test_multi_init_extension_compat(self):
         module = '_testmultiphase'
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
-        with self.subTest(f'{module}: isolated'):
-            self.check_compatible_isolated(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
+        with self.subTest(f'{module}: strict, isolated'):
+            self.check_compatible_isolated(module, strict=True)
 
     def test_python_compat(self):
         module = 'threading'
         if __import__(module).__spec__.origin == 'frozen':
             raise unittest.SkipTest(f'{module} is unexpectedly frozen')
-        with self.subTest(f'{module}: shared'):
-            self.check_compatible_shared(module)
-        with self.subTest(f'{module}: isolated'):
-            self.check_compatible_isolated(module)
+        with self.subTest(f'{module}: not strict'):
+            self.check_compatible_shared(module, strict=False)
+        with self.subTest(f'{module}: strict, shared'):
+            self.check_compatible_shared(module, strict=True)
+        with self.subTest(f'{module}: strict, isolated'):
+            self.check_compatible_isolated(module, strict=True)
 
 
 if __name__ == '__main__':

Lib/test/test_threading.py

@@ -1327,6 +1327,7 @@ def func():
                 allow_exec=True,
                 allow_threads={allowed},
                 allow_daemon_threads={daemon_allowed},
+                check_multi_interp_extensions=False,
             )
             """)
         with test.support.SuppressCrashReport():

Modules/_testcapimodule.c

@@ -3234,6 +3234,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
     int allow_exec = -1;
     int allow_threads = -1;
     int allow_daemon_threads = -1;
+    int check_multi_interp_extensions = -1;
     int r;
     PyThreadState *substate, *mainstate;
     /* only initialise 'cflags.cf_flags' to test backwards compatibility */
@@ -3244,11 +3245,13 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
                              "allow_exec",
                              "allow_threads",
                              "allow_daemon_threads",
+                             "check_multi_interp_extensions",
                              NULL};
     if (!PyArg_ParseTupleAndKeywords(args, kwargs,
-                    "s$pppp:run_in_subinterp_with_config", kwlist,
+                    "s$ppppp:run_in_subinterp_with_config", kwlist,
                     &code, &allow_fork, &allow_exec,
-                    &allow_threads, &allow_daemon_threads)) {
+                    &allow_threads, &allow_daemon_threads,
+                    &check_multi_interp_extensions)) {
         return NULL;
     }
     if (allow_fork < 0) {
@@ -3267,6 +3270,10 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
         PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads");
         return NULL;
     }
+    if (check_multi_interp_extensions < 0) {
+        PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions");
+        return NULL;
+    }
 
     mainstate = PyThreadState_Get();
 
@@ -3277,6 +3284,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
         .allow_exec = allow_exec,
         .allow_threads = allow_threads,
         .allow_daemon_threads = allow_daemon_threads,
+        .check_multi_interp_extensions = check_multi_interp_extensions,
     };
     substate = _Py_NewInterpreterFromConfig(&config);
     if (substate == NULL) {

Python/pylifecycle.c

@@ -630,6 +630,10 @@ init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *con
     if (config->allow_daemon_threads) {
         interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS;
     }
+
+    if (config->check_multi_interp_extensions) {
+        interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS;
+    }
 }