bpo-22257: Small changes for PEP 432. (#1728)

PEP 432 specifies a number of large changes to interpreter startup code, including exposing a cleaner C-API. The major changes depend on a number of smaller changes. This patch includes all those smaller changes.

Author: ericsnowcurrently

Include/Python.h

@@ -112,6 +112,7 @@
 
 #include "pyarena.h"
 #include "modsupport.h"
+#include "compile.h"
 #include "pythonrun.h"
 #include "pylifecycle.h"
 #include "ceval.h"
@@ -123,7 +124,6 @@
 #include "abstract.h"
 #include "bltinmodule.h"
 
-#include "compile.h"
 #include "eval.h"
 
 #include "pyctype.h"

Include/compile.h

@@ -11,6 +11,23 @@ extern "C" {
 /* Public interface */
 struct _node; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
+/* XXX (ncoghlan): Unprefixed type name in a public API! */
+
+#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
+                   CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
+                   CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
+                   CO_FUTURE_GENERATOR_STOP)
+#define PyCF_MASK_OBSOLETE (CO_NESTED)
+#define PyCF_SOURCE_IS_UTF8  0x0100
+#define PyCF_DONT_IMPLY_DEDENT 0x0200
+#define PyCF_ONLY_AST 0x0400
+#define PyCF_IGNORE_COOKIE 0x0800
+
+#ifndef Py_LIMITED_API
+typedef struct {
+    int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
+} PyCompilerFlags;
+#endif
 
 /* Future feature support */
 

Include/pylifecycle.h

@@ -77,14 +77,15 @@ PyAPI_FUNC(const char *) _Py_gitversion(void);
 /* Internal -- various one-time initializations */
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
-PyAPI_FUNC(PyObject *) _PySys_Init(void);
+PyAPI_FUNC(PyObject *) _PySys_BeginInit(void);
+PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict);
 PyAPI_FUNC(void) _PyImport_Init(void);
 PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
 PyAPI_FUNC(void) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(int) _PyFloat_Init(void);
 PyAPI_FUNC(int) PyByteArray_Init(void);
-PyAPI_FUNC(void) _PyRandom_Init(void);
+PyAPI_FUNC(void) _Py_HashRandomization_Init(void);
 #endif
 
 /* Various internal finalizers */
@@ -106,7 +107,7 @@ PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void);
 PyAPI_FUNC(void) _PyGC_Fini(void);
 PyAPI_FUNC(void) PySlice_Fini(void);
 PyAPI_FUNC(void) _PyType_Fini(void);
-PyAPI_FUNC(void) _PyRandom_Fini(void);
+PyAPI_FUNC(void) _Py_HashRandomization_Fini(void);
 PyAPI_FUNC(void) PyAsyncGen_Fini(void);
 
 PyAPI_DATA(PyThreadState *) _Py_Finalizing;

Include/pystate.h

@@ -302,6 +302,7 @@ PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void);
 /* Routines for advanced debuggers, requested by David Beazley.
    Don't use unless you know what you are doing! */
 #ifndef Py_LIMITED_API
+PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void);
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Head(void);
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Next(PyInterpreterState *);
 PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *);

Include/pythonrun.h

@@ -7,22 +7,6 @@
 extern "C" {
 #endif
 
-#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
-                   CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
-                   CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
-                   CO_FUTURE_GENERATOR_STOP)
-#define PyCF_MASK_OBSOLETE (CO_NESTED)
-#define PyCF_SOURCE_IS_UTF8  0x0100
-#define PyCF_DONT_IMPLY_DEDENT 0x0200
-#define PyCF_ONLY_AST 0x0400
-#define PyCF_IGNORE_COOKIE 0x0800
-
-#ifndef Py_LIMITED_API
-typedef struct {
-    int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
-} PyCompilerFlags;
-#endif
-
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
 PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);

Lib/importlib/_bootstrap.py

@@ -1134,12 +1134,15 @@ def _setup(sys_module, _imp_module):
 
 
 def _install(sys_module, _imp_module):
-    """Install importlib as the implementation of import."""
+    """Install importers for builtin and frozen modules"""
     _setup(sys_module, _imp_module)
 
     sys.meta_path.append(BuiltinImporter)
     sys.meta_path.append(FrozenImporter)
 
+
+def _install_external_importers():
+    """Install importers that require external filesystem access"""
     global _bootstrap_external
     import _frozen_importlib_external
     _bootstrap_external = _frozen_importlib_external

Lib/test/coding20731.py

@@ -1,4 +1,4 @@
-#coding:latin1
-
-
-
+#coding:latin1
+
+
+

Lib/test/test_cmd_line.py

@@ -485,8 +485,29 @@ def test_isolatedmode(self):
                                           cwd=tmpdir)
             self.assertEqual(out.strip(), b"ok")
 
+
+class IgnoreEnvironmentTest(unittest.TestCase):
+
+    def run_ignoring_vars(self, predicate, **env_vars):
+        # Runs a subprocess with -E set, even though we're passing
+        # specific environment variables
+        # Logical inversion to match predicate check to a zero return
+        # code indicating success
+        code = "import sys; sys.stderr.write(str(sys.flags)); sys.exit(not ({}))".format(predicate)
+        return assert_python_ok('-E', '-c', code, **env_vars)
+
+    def test_ignore_PYTHONPATH(self):
+        path = "should_be_ignored"
+        self.run_ignoring_vars("'{}' not in sys.path".format(path),
+                               PYTHONPATH=path)
+
+    def test_ignore_PYTHONHASHSEED(self):
+        self.run_ignoring_vars("sys.flags.hash_randomization == 1",
+                               PYTHONHASHSEED="0")
+
+
 def test_main():
-    test.support.run_unittest(CmdLineTest)
+    test.support.run_unittest(CmdLineTest, IgnoreEnvironmentTest)
     test.support.reap_children()
 
 if __name__ == "__main__":

Lib/test/test_site.py

@@ -183,6 +183,7 @@ def test_addsitedir(self):
     @unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 "
                           "user-site (site.ENABLE_USER_SITE)")
     def test_s_option(self):
+        # (ncoghlan) Change this to use script_helper...
         usersite = site.USER_SITE
         self.assertIn(usersite, sys.path)
 
@@ -199,7 +200,7 @@ def test_s_option(self):
         if usersite == site.getsitepackages()[0]:
             self.assertEqual(rc, 1)
         else:
-            self.assertEqual(rc, 0)
+            self.assertEqual(rc, 0, "User site still added to path with -s")
 
         env = os.environ.copy()
         env["PYTHONNOUSERSITE"] = "1"
@@ -209,14 +210,16 @@ def test_s_option(self):
         if usersite == site.getsitepackages()[0]:
             self.assertEqual(rc, 1)
         else:
-            self.assertEqual(rc, 0)
+            self.assertEqual(rc, 0,
+                        "User site still added to path with PYTHONNOUSERSITE")
 
         env = os.environ.copy()
         env["PYTHONUSERBASE"] = "/tmp"
         rc = subprocess.call([sys.executable, '-c',
             'import sys, site; sys.exit(site.USER_BASE.startswith("/tmp"))'],
             env=env)
-        self.assertEqual(rc, 1)
+        self.assertEqual(rc, 1,
+                        "User base not set by PYTHONUSERBASE")
 
     def test_getuserbase(self):
         site.USER_BASE = None

Makefile.pre.in

@@ -349,7 +349,7 @@ PYTHON_OBJS=	\
 		Python/pystate.o \
 		Python/pythonrun.o \
 		Python/pytime.o \
-		Python/random.o \
+		Python/bootstrap_hash.o \
 		Python/structmember.o \
 		Python/symtable.o \
 		Python/sysmodule.o \

Misc/NEWS

@@ -154,6 +154,8 @@ Core and Builtins
 - Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by
   Chi Hsuan Yen.
 
+- bpo-22257: Clean up interpreter startup (see PEP 432).
+
 - Issue #26919: On Android, operating system data is now always encoded/decoded
   to/from UTF-8, instead of the locale encoding to avoid inconsistencies with
   os.fsencode() and os.fsdecode() which are already using UTF-8.