Merge remote-tracking branch 'upstream/master' into super_call

Author: vladima

Doc/data/stable_abi.dat

@@ -776,7 +776,6 @@ Py_SetPath
 Py_SetProgramName
 Py_SetPythonHome
 Py_SetRecursionLimit
-Py_SymtableString
 Py_UTF8Mode
 Py_VaBuildValue
 Py_XNewRef

Doc/using/unix.rst

@@ -134,3 +134,53 @@ some Unices may not have the :program:`env` command, so you may need to hardcode
 ``/usr/bin/python3`` as the interpreter path.
 
 To use shell commands in your Python scripts, look at the :mod:`subprocess` module.
+
+
+Custom OpenSSL
+==============
+
+1. To use your vendor's OpenSSL configuration and system trust store, locate
+   the directory with ``openssl.cnf`` file or symlink in ``/etc``. On most
+   distribution the file is either in ``/etc/ssl`` or ``/etc/pki/tls``. The
+   directory should also contain a ``cert.pem`` file and/or a ``certs``
+   directory.
+
+   .. code-block:: shell-session
+
+      $ find /etc/ -name openssl.cnf -printf "%h\n"
+      /etc/ssl
+
+2. Download, build, and install OpenSSL. Make sure you use ``install_sw`` and
+   not ``install``. The ``install_sw`` target does not override
+   ``openssl.cnf``.
+
+   .. code-block:: shell-session
+
+      $ curl -O https://www.openssl.org/source/openssl-VERSION.tar.gz
+         $ tar xzf openssl-VERSION
+         $ pushd openssl-VERSION
+         $ ./config \
+              --prefix=/usr/local/custom-openssl \
+              --openssldir=/etc/ssl
+         $ make -j1 depend
+         $ make -j8
+         $ make install_sw
+         $ popd
+
+3. Build Python with custom OpenSSL
+
+   .. code-block:: shell-session
+
+      $ pushd python-3.x.x
+      $ ./configure -C \
+          --with-openssl=/usr/local/custom-openssl \
+          --with-openssl-rpath=auto \
+          --prefix=/usr/local/python-3.x.x
+      $ make -j8
+      $ make altinstall
+
+.. note::
+
+   Patch releases of OpenSSL have a backwards compatible ABI. You don't need
+   to recompile Python to update OpenSSL. It's sufficient to replace the
+   custom OpenSSL installation with a newer version.

Doc/whatsnew/3.10.rst

@@ -175,6 +175,8 @@ Tracing events, with the correct line number, are generated for all lines of cod
 
 The ``f_lineo`` attribute of frame objects will always contain the expected line number.
 
+The ``co_lnotab`` attribute of code objects is deprecated and will be removed in 3.12.
+Code that needs to convert from offset to line number should use the new ``co_lines()`` method instead.
 
 PEP 634: Structural Pattern Matching
 ------------------------------------
@@ -1181,6 +1183,12 @@ Build Changes
   and ``--with-tcltk-libs`` configuration options.
   (Contributed by Manolis Stamatogiannakis in :issue:`42603`.)
 
+* Add ``--with-openssl-rpath`` option to ``configure`` script. The option
+  simplifies building Python with a custom OpenSSL installation, e.g.
+  ``./configure --with-openssl=/path/to/openssl --with-openssl-rpath=auto``.
+  (Contributed by Christian Heimes in :issue:`43466`.)
+
+
 
 C API Changes
 =============
@@ -1358,3 +1366,19 @@ Removed
   AST object (``mod_ty`` type) with the public C API. The function was already
   excluded from the limited C API (:pep:`384`).
   (Contributed by Victor Stinner in :issue:`43244`.)
+
+* Remove the ``symtable.h`` header file and the undocumented functions:
+
+  * ``PyST_GetScope()``
+  * ``PySymtable_Build()``
+  * ``PySymtable_BuildObject()``
+  * ``PySymtable_Free()``
+  * ``Py_SymtableString()``
+  * ``Py_SymtableStringObject()``
+
+  The ``Py_SymtableString()`` function was part the stable ABI by mistake but
+  it could not be used, because the ``symtable.h`` header file was excluded
+  from the limited C API.
+
+  The Python :mod:`symtable` module remains available and is unchanged.
+  (Contributed by Victor Stinner in :issue:`43244`.)

Include/cpython/pythonrun.h

@@ -77,17 +77,6 @@ PyAPI_FUNC(const char *) _Py_SourceAsString(
     PyCompilerFlags *cf,
     PyObject **cmd_copy);
 
-PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
-    const char *str,
-    PyObject *filename,
-    int start);
-
-PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags(
-    const char *str,
-    PyObject *filename,
-    int start,
-    PyCompilerFlags *flags);
-
 
 /* A function flavor is also exported by libpython. It is required when
     libpython is accessed directly rather than using header files which defines

Include/symtable.h renamed to Include/internal/pycore_symtable.h

@@ -1,15 +1,14 @@
-#ifndef Py_LIMITED_API
-#ifndef Py_SYMTABLE_H
-#define Py_SYMTABLE_H
+#ifndef Py_INTERNAL_SYMTABLE_H
+#define Py_INTERNAL_SYMTABLE_H
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "Python-ast.h"   /* mod_ty */
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
 
-/* XXX(ncoghlan): This is a weird mix of public names and interpreter internal
- *                names.
- */
+#include "Python-ast.h"   /* mod_ty */
 
 typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
     _Py_block_ty;
@@ -68,23 +67,19 @@ typedef struct _symtable_entry {
     struct symtable *ste_table;
 } PySTEntryObject;
 
-PyAPI_DATA(PyTypeObject) PySTEntry_Type;
+extern PyTypeObject PySTEntry_Type;
 
 #define PySTEntry_Check(op) Py_IS_TYPE(op, &PySTEntry_Type)
 
-PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
+extern int _PyST_GetScope(PySTEntryObject *, PyObject *);
 
-PyAPI_FUNC(struct symtable *) PySymtable_Build(
-    mod_ty mod,
-    const char *filename,       /* decoded from the filesystem encoding */
-    PyFutureFeatures *future);
-PyAPI_FUNC(struct symtable *) PySymtable_BuildObject(
+extern struct symtable* _PySymtable_Build(
     mod_ty mod,
     PyObject *filename,
     PyFutureFeatures *future);
 PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
 
-PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
+extern void _PySymtable_Free(struct symtable *);
 
 /* Flags for def-use information */
 
@@ -117,8 +112,14 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define GENERATOR 1
 #define GENERATOR_EXPRESSION 2
 
+// Used by symtablemodule.c
+extern struct symtable* _Py_SymtableStringObjectFlags(
+    const char *str,
+    PyObject *filename,
+    int start,
+    PyCompilerFlags *flags);
+
 #ifdef __cplusplus
 }
 #endif
-#endif /* !Py_SYMTABLE_H */
-#endif /* !Py_LIMITED_API */
+#endif /* !Py_INTERNAL_SYMTABLE_H */

Include/pythonrun.h

@@ -9,11 +9,6 @@ extern "C" {
 
 PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
 
-PyAPI_FUNC(struct symtable *) Py_SymtableString(
-    const char *str,
-    const char *filename,       /* decoded from the filesystem encoding */
-    int start);
-
 PyAPI_FUNC(void) PyErr_Print(void);
 PyAPI_FUNC(void) PyErr_PrintEx(int);
 PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *);

Lib/test/test_dis.py

@@ -809,6 +809,31 @@ def f(self): return super().x
    0: __class__"""
         self.assertEqual(dedent(dis.code_info(g["C"].f)), expected_dis_info)
 
+    def test_super_attr_load_self_cell(self):
+        src = """
+        class C:
+            def f(self):
+                lambda: self
+                return super().x
+        """
+        expected = """\
+  4           0 LOAD_CLOSURE             0 (self)
+              2 BUILD_TUPLE              1
+              4 LOAD_CONST               1 (<code object <lambda> at 0x..., file "<string>", line 4>)
+              6 LOAD_CONST               2 ('C.f.<locals>.<lambda>')
+              8 MAKE_FUNCTION            8 (closure)
+             10 POP_TOP
+
+  5          12 LOAD_GLOBAL              0 (super)
+             14 LOAD_DEREF               1 (__class__)
+             16 LOAD_DEREF               0 (self)
+             18 LOAD_ATTR_SUPER          3 ((1, True))
+             20 RETURN_VALUE
+"""
+        g = {}
+        exec(dedent(src), g)
+        self.do_disassembly_test(g["C"].f, expected)
+
     def test_super_attr_store(self):
         src = """
         class C:

Lib/test/test_gdb.py

@@ -928,12 +928,12 @@ def test_wrapper_call(self):
         cmd = textwrap.dedent('''
             class MyList(list):
                 def __init__(self):
-                    super().__init__()   # wrapper_call()
+                    super().__init__()   # wrapperdescr_call()
 
             id("first break point")
             l = MyList()
         ''')
-        cmds_after_breakpoint = ['break wrapper_call', 'continue']
+        cmds_after_breakpoint = ['break wrapperdescr_call', 'continue']
         if CET_PROTECTION:
             # bpo-32962: same case as in get_stack_trace():
             # we need an additional 'next' command in order to read
@@ -945,7 +945,7 @@ def __init__(self):
         gdb_output = self.get_stack_trace(cmd,
                                           cmds_after_breakpoint=cmds_after_breakpoint)
         self.assertRegex(gdb_output,
-                         r"<method-wrapper u?'__init__' of MyList object at ")
+                         r"methoddescr-wrapper '__init__'>, args=\(<MyList at ")
 
 
 class PyPrintTests(DebuggerTests):

Makefile.pre.in

@@ -202,6 +202,7 @@ ENSUREPIP=      @ENSUREPIP@
 OPENSSL_INCLUDES=@OPENSSL_INCLUDES@
 OPENSSL_LIBS=@OPENSSL_LIBS@
 OPENSSL_LDFLAGS=@OPENSSL_LDFLAGS@
+OPENSSL_RPATH=@OPENSSL_RPATH@
 
 # Default zoneinfo.TZPATH. Added here to expose it in sysconfig.get_config_var
 TZPATH=@TZPATH@
@@ -1086,7 +1087,6 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/sliceobject.h \
 		$(srcdir)/Include/structmember.h \
 		$(srcdir)/Include/structseq.h \
-		$(srcdir)/Include/symtable.h \
 		$(srcdir)/Include/sysmodule.h \
 		$(srcdir)/Include/token.h \
 		$(srcdir)/Include/traceback.h \
@@ -1166,6 +1166,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_pymem.h \
 		$(srcdir)/Include/internal/pycore_pystate.h \
 		$(srcdir)/Include/internal/pycore_runtime.h \
+		$(srcdir)/Include/internal/pycore_symtable.h \
 		$(srcdir)/Include/internal/pycore_sysmodule.h \
 		$(srcdir)/Include/internal/pycore_traceback.h \
 		$(srcdir)/Include/internal/pycore_tuple.h \

Misc/NEWS.d/next/Build/2021-03-11-00-14-47.bpo-43466.N861Z5.rst

@@ -0,0 +1 @@
+The ``configure`` script now supports ``--with-openssl-rpath`` option.

Misc/NEWS.d/next/C API/2021-03-17-23-53-14.bpo-43244.kfPqA_.rst

@@ -0,0 +1,16 @@
+Remove the ``symtable.h`` header file and the undocumented functions:
+
+* ``PyST_GetScope()``
+* ``PySymtable_Build()``
+* ``PySymtable_BuildObject()``
+* ``PySymtable_Free()``
+* ``Py_SymtableString()``
+* ``Py_SymtableStringObject()``
+
+The ``Py_SymtableString()`` function was part the stable ABI by mistake but it
+could not be used, because the ``symtable.h`` header file was excluded from the
+limited C API.
+
+The Python :mod:`symtable` module remains available and is unchanged.
+
+Patch by Victor Stinner.

Modules/symtablemodule.c

@@ -1,8 +1,7 @@
 #include "Python.h"
 
-#include "code.h"
 #include "Python-ast.h"
-#include "symtable.h"
+#include "pycore_symtable.h"      // struct symtable
 
 #include "clinic/symtablemodule.c.h"
 /*[clinic input]
@@ -62,7 +61,7 @@ _symtable_symtable_impl(PyObject *module, PyObject *source,
     t = (PyObject *)st->st_top;
     Py_INCREF(t);
     PyMem_Free((void *)st->st_future);
-    PySymtable_Free(st);
+    _PySymtable_Free(st);
     return t;
 }
 

PC/python3dll.c

@@ -80,7 +80,6 @@ EXPORT_FUNC(Py_SetPath)
 EXPORT_FUNC(Py_SetProgramName)
 EXPORT_FUNC(Py_SetPythonHome)
 EXPORT_FUNC(Py_SetRecursionLimit)
-EXPORT_FUNC(Py_SymtableString)
 EXPORT_FUNC(Py_VaBuildValue)
 EXPORT_FUNC(Py_XNewRef)
 EXPORT_FUNC(PyArg_Parse)

PCbuild/pythoncore.vcxproj

@@ -208,6 +208,7 @@
     <ClInclude Include="..\Include\internal\pycore_pystate.h" />
     <ClInclude Include="..\Include\internal\pycore_runtime.h" />
     <ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
+    <ClInclude Include="..\Include\internal\pycore_symtable.h" />
     <ClInclude Include="..\Include\internal\pycore_traceback.h" />
     <ClInclude Include="..\Include\internal\pycore_tuple.h" />
     <ClInclude Include="..\Include\internal\pycore_ucnhash.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -585,6 +585,9 @@
     <ClInclude Include="..\Include\internal\pycore_sysmodule.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_symtable.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_traceback.h">
       <Filter>Include\internal</Filter>
     </ClInclude>