Manage memory lifetime for all type-related objects.

Remove internal usages of PyMethodDef and PyGetSetDef.

For PyMethodDef, change PyCFunctionObject to replace the PyMethodDef* member
with a PyCFunctionBase member. The PyCFunctionBase is a new struct to hold the
managed values of a PyMethodDef. This type is shared between PyCFunction and the
various callable descriptor objects. A PyCFunctionBase is like a PyMethodDef but
replaces the char* members with PyObject* members.

For PyGetSetDef, inline the members on the resulting PyGetSetDescrObject,
replacing all char* members with PyObject* members. The memory for the closure
is *not* managed, adding support for that would likely require an API change and
can be done in a future change.

For the tp_name field, instead of setting it directly to the value of
PyType_Spec.name, set it to the result of PyUnicode_AsUTF8(ht_name), where
ht_name is the PyUnicode object created from the original spec name. This is the
same trick used to properly manage this pointer for heap types when the __name__
is reassigned.

Author: Joe Jevnik

Doc/c-api/type.rst

@@ -128,6 +128,9 @@ The following functions and structs are used to create
 
    This function calls :c:func:`PyType_Ready` on the new type.
 
+   Until Python 3.9, the memory backing the ``Py_tp_methods``, ``Py_tp_getset``,
+   and :c:member:`PyType_Spec.name` must outlive the returned type.
+
    .. versionadded:: 3.3
 
 .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)

Include/bltinmodule.h

@@ -8,6 +8,10 @@ PyAPI_DATA(PyTypeObject) PyFilter_Type;
 PyAPI_DATA(PyTypeObject) PyMap_Type;
 PyAPI_DATA(PyTypeObject) PyZip_Type;
 
+#ifdef Py_BUILD_CORE
+PyObject* _PyBuiltin_Print(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Include/cpython/object.h

@@ -282,12 +282,20 @@ typedef struct _heaptypeobject {
     PyBufferProcs as_buffer;
     PyObject *ht_name, *ht_slots, *ht_qualname;
     struct _dictkeysobject *ht_cached_keys;
-    /* here are optional user slots, followed by the members. */
+    /* here are optional user slots, followed by the offsets of the object members. */
 } PyHeapTypeObject;
 
-/* access macro to the members which are floating "behind" the object */
-#define PyHeapType_GET_MEMBERS(etype) \
-    ((PyMemberDef *)(((char *)etype) + Py_TYPE(etype)->tp_basicsize))
+#ifdef Py_BUILD_CORE
+typedef struct {
+    Py_ssize_t offset;
+    int flags;
+} _PyObject_MemberSlot;
+
+/* access macro to the offsets of the object type members which are floating
+   "behind" the object */
+#define _PyHeapType_GET_OBJECT_MEMBER_OFFSETS(etype)                     \
+    ((_PyObject_MemberSlot *)(((char *)etype) + Py_TYPE(etype)->tp_basicsize))
+#endif
 
 PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *);
 PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *);

Include/descrobject.h

@@ -52,18 +52,24 @@ typedef struct {
 
 typedef struct {
     PyDescr_COMMON;
-    PyMethodDef *d_method;
+    PyCFunctionBase d_base;
     vectorcallfunc vectorcall;
 } PyMethodDescrObject;
 
 typedef struct {
     PyDescr_COMMON;
-    struct PyMemberDef *d_member;
+    int d_member_type;
+    Py_ssize_t d_offset;
+    int d_flags;
+    PyObject *d_doc;
 } PyMemberDescrObject;
 
 typedef struct {
     PyDescr_COMMON;
-    PyGetSetDef *d_getset;
+    getter d_get;
+    setter d_set;
+    PyObject *d_doc;
+    void *d_closure;
 } PyGetSetDescrObject;
 
 typedef struct {

Include/methodobject.h

@@ -32,12 +32,12 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *);
    done, so use with care. */
 #ifndef Py_LIMITED_API
 #define PyCFunction_GET_FUNCTION(func) \
-        (((PyCFunctionObject *)func) -> m_ml -> ml_meth)
+        (((PyCFunctionObject *)func) -> m_base.meth)
 #define PyCFunction_GET_SELF(func) \
-        (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \
+        (((PyCFunctionObject *)func) -> m_base.flags & METH_STATIC ? \
          NULL : ((PyCFunctionObject *)func) -> m_self)
 #define PyCFunction_GET_FLAGS(func) \
-        (((PyCFunctionObject *)func) -> m_ml -> ml_flags)
+        (((PyCFunctionObject *)func) -> m_base.flags)
 #endif
 PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *);
 
@@ -99,24 +99,36 @@ PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *,
 #endif
 
 #ifndef Py_LIMITED_API
+typedef struct {
+    PyObject   *name;
+    PyCFunction meth;
+    PyObject    *doc;
+    PyObject    *signature;
+    int         flags;
+} PyCFunctionBase;
+
 typedef struct {
     PyObject_HEAD
-    PyMethodDef *m_ml; /* Description of the C function to call */
-    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
-    PyObject    *m_module; /* The __module__ attribute, can be anything */
-    PyObject    *m_weakreflist; /* List of weak references */
+    PyCFunctionBase m_base;
+    PyObject       *m_self; /* Passed as 'self' arg to the C func, can be NULL */
+    PyObject       *m_module; /* The __module__ attribute, can be anything */
+    PyObject       *m_weakreflist; /* List of weak references */
     vectorcallfunc vectorcall;
 } PyCFunctionObject;
 
-PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallDict(
-    PyMethodDef *method,
+PyAPI_FUNC(int) _PyCFunctionBase_FromMethodDef(PyCFunctionBase *, PyMethodDef *);
+PyAPI_FUNC(void) _PyCFunctionBase_Clear(PyCFunctionBase *);
+
+PyAPI_FUNC(PyObject *) _PyCFunction_NewFromBase(PyCFunctionBase *, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) _PyCFunctionBase_RawFastCallDict(
+    PyCFunctionBase *func,
     PyObject *self,
     PyObject *const *args,
     Py_ssize_t nargs,
     PyObject *kwargs);
 
-PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallKeywords(
-    PyMethodDef *method,
+PyAPI_FUNC(PyObject *) _PyCFunctionBase_RawFastCallKeywords(
+    PyCFunctionBase *func,
     PyObject *self,
     PyObject *const *args,
     Py_ssize_t nargs,

Include/structmember.h

@@ -67,6 +67,11 @@ typedef struct PyMemberDef {
 PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, struct PyMemberDef *);
 PyAPI_FUNC(int) PyMember_SetOne(char *, struct PyMemberDef *, PyObject *);
 
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(PyObject *) _PyMemberDescr_GetOne(const char *, PyMemberDescrObject *);
+PyAPI_FUNC(int) _PyMemberDescr_SetOne(char *, PyMemberDescrObject *, PyObject *);
+#endif
+
 
 #ifdef __cplusplus
 }

Lib/test/test_cprofile.py

@@ -86,39 +86,39 @@ def main():
         8    0.312    0.039    0.400    0.050 profilee.py:88(helper2)
         8    0.064    0.008    0.080    0.010 profilee.py:98(subhelper)"""
 _ProfileOutput['print_callers'] = """\
-profilee.py:110(__getattr__)                      <-      16    0.016    0.016  profilee.py:98(subhelper)
-profilee.py:25(testfunc)                          <-       1    0.270    1.000  <string>:1(<module>)
-profilee.py:35(factorial)                         <-       1    0.014    0.130  profilee.py:25(testfunc)
-                                                        20/3    0.130    0.147  profilee.py:35(factorial)
-                                                           2    0.006    0.040  profilee.py:84(helper2_indirect)
-profilee.py:48(mul)                               <-      20    0.020    0.020  profilee.py:35(factorial)
-profilee.py:55(helper)                            <-       2    0.040    0.600  profilee.py:25(testfunc)
-profilee.py:73(helper1)                           <-       4    0.116    0.120  profilee.py:55(helper)
-profilee.py:84(helper2_indirect)                  <-       2    0.000    0.140  profilee.py:55(helper)
-profilee.py:88(helper2)                           <-       6    0.234    0.300  profilee.py:55(helper)
-                                                           2    0.078    0.100  profilee.py:84(helper2_indirect)
-profilee.py:98(subhelper)                         <-       8    0.064    0.080  profilee.py:88(helper2)
-{built-in method builtins.hasattr}                <-       4    0.000    0.004  profilee.py:73(helper1)
-                                                           8    0.000    0.008  profilee.py:88(helper2)
-{built-in method sys.exc_info}                    <-       4    0.000    0.000  profilee.py:73(helper1)
-{method 'append' of 'list' objects}               <-       4    0.000    0.000  profilee.py:73(helper1)"""
+profilee.py:110(__getattr__)         <-      16    0.016    0.016  profilee.py:98(subhelper)
+profilee.py:25(testfunc)             <-       1    0.270    1.000  <string>:1(<module>)
+profilee.py:35(factorial)            <-       1    0.014    0.130  profilee.py:25(testfunc)
+                                           20/3    0.130    0.147  profilee.py:35(factorial)
+                                              2    0.006    0.040  profilee.py:84(helper2_indirect)
+profilee.py:48(mul)                  <-      20    0.020    0.020  profilee.py:35(factorial)
+profilee.py:55(helper)               <-       2    0.040    0.600  profilee.py:25(testfunc)
+profilee.py:73(helper1)              <-       4    0.116    0.120  profilee.py:55(helper)
+profilee.py:84(helper2_indirect)     <-       2    0.000    0.140  profilee.py:55(helper)
+profilee.py:88(helper2)              <-       6    0.234    0.300  profilee.py:55(helper)
+                                              2    0.078    0.100  profilee.py:84(helper2_indirect)
+profilee.py:98(subhelper)            <-       8    0.064    0.080  profilee.py:88(helper2)
+{built-in method builtins.hasattr}   <-       4    0.000    0.004  profilee.py:73(helper1)
+                                              8    0.000    0.008  profilee.py:88(helper2)
+{built-in method sys.exc_info}       <-       4    0.000    0.000  profilee.py:73(helper1)
+{method 'append' of 'list' objects}  <-       4    0.000    0.000  profilee.py:73(helper1)"""
 _ProfileOutput['print_callees'] = """\
-<string>:1(<module>)                              ->       1    0.270    1.000  profilee.py:25(testfunc)
-profilee.py:110(__getattr__)                      ->
-profilee.py:25(testfunc)                          ->       1    0.014    0.130  profilee.py:35(factorial)
-                                                           2    0.040    0.600  profilee.py:55(helper)
-profilee.py:35(factorial)                         ->    20/3    0.130    0.147  profilee.py:35(factorial)
-                                                          20    0.020    0.020  profilee.py:48(mul)
-profilee.py:48(mul)                               ->
-profilee.py:55(helper)                            ->       4    0.116    0.120  profilee.py:73(helper1)
-                                                           2    0.000    0.140  profilee.py:84(helper2_indirect)
-                                                           6    0.234    0.300  profilee.py:88(helper2)
-profilee.py:73(helper1)                           ->       4    0.000    0.004  {built-in method builtins.hasattr}
-profilee.py:84(helper2_indirect)                  ->       2    0.006    0.040  profilee.py:35(factorial)
-                                                           2    0.078    0.100  profilee.py:88(helper2)
-profilee.py:88(helper2)                           ->       8    0.064    0.080  profilee.py:98(subhelper)
-profilee.py:98(subhelper)                         ->      16    0.016    0.016  profilee.py:110(__getattr__)
-{built-in method builtins.hasattr}                ->      12    0.012    0.012  profilee.py:110(__getattr__)"""
+<string>:1(<module>)                 ->       1    0.270    1.000  profilee.py:25(testfunc)
+profilee.py:110(__getattr__)         ->
+profilee.py:25(testfunc)             ->       1    0.014    0.130  profilee.py:35(factorial)
+                                              2    0.040    0.600  profilee.py:55(helper)
+profilee.py:35(factorial)            ->    20/3    0.130    0.147  profilee.py:35(factorial)
+                                             20    0.020    0.020  profilee.py:48(mul)
+profilee.py:48(mul)                  ->
+profilee.py:55(helper)               ->       4    0.116    0.120  profilee.py:73(helper1)
+                                              2    0.000    0.140  profilee.py:84(helper2_indirect)
+                                              6    0.234    0.300  profilee.py:88(helper2)
+profilee.py:73(helper1)              ->       4    0.000    0.004  {built-in method builtins.hasattr}
+profilee.py:84(helper2_indirect)     ->       2    0.006    0.040  profilee.py:35(factorial)
+                                              2    0.078    0.100  profilee.py:88(helper2)
+profilee.py:88(helper2)              ->       8    0.064    0.080  profilee.py:98(subhelper)
+profilee.py:98(subhelper)            ->      16    0.016    0.016  profilee.py:110(__getattr__)
+{built-in method builtins.hasattr}   ->      12    0.012    0.012  profilee.py:110(__getattr__)"""
 
 if __name__ == "__main__":
     main()

Lib/test/test_sys.py

@@ -1080,7 +1080,7 @@ def test_objecttypes(self):
         # buffer
         # XXX
         # builtin_function_or_method
-        check(len, size('5P'))
+        check(len, size('8Pi'))
         # bytearray
         samples = [b'', b'u'*100000]
         for sample in samples:
@@ -1111,15 +1111,15 @@ def inner():
         # complex
         check(complex(0,1), size('2d'))
         # method_descriptor (descriptor object)
-        check(str.lower, size('3PPP'))
+        check(str.lower, size('3P4PiP'))
         # classmethod_descriptor (descriptor object)
         # XXX
         # member_descriptor (descriptor object)
         import datetime
-        check(datetime.timedelta.days, size('3PP'))
+        check(datetime.timedelta.days, size('3PiniP'))
         # getset_descriptor (descriptor object)
         import collections
-        check(collections.defaultdict.default_factory, size('3PP'))
+        check(collections.defaultdict.default_factory, size('3P4P'))
         # wrapper_descriptor (descriptor object)
         check(int.__add__, size('3P2P'))
         # method-wrapper (descriptor object)

Misc/NEWS.d/next/C API/2019-06-18-00-55-12.bpo-37270.2z98X7.rst

@@ -0,0 +1,6 @@
+Ensure that all of the memory needed by a type created with
+:c:func:`PyType_FromSpec` or :c:func:`PyType_FromSpecWithBases` is managed by
+Python. Previously, the memory referred to by ``Py_tp_methods``,
+``Py_tp_getset``, ``Py_tp_members`` and the various ``char*`` fields they
+contained needed to outlive the type. Now, these arrays and strings are copied
+and managed by the type itself.

Modules/_lsprof.c

@@ -132,42 +132,40 @@ normalizeUserObj(PyObject *obj)
         if (modname != NULL) {
             if (!_PyUnicode_EqualToASCIIString(modname, "builtins")) {
                 PyObject *result;
-                result = PyUnicode_FromFormat("<%U.%s>", modname,
-                                              fn->m_ml->ml_name);
+                result = PyUnicode_FromFormat("<%U.%S>", modname,
+                                              fn->m_base.name);
                 Py_DECREF(modname);
                 return result;
             }
             Py_DECREF(modname);
         }
-        return PyUnicode_FromFormat("<%s>", fn->m_ml->ml_name);
+        return PyUnicode_FromFormat("<%S>", fn->m_base.name);
     }
     else {
         /* built-in method: try to return
             repr(getattr(type(__self__), __name__))
         */
         PyObject *self = fn->m_self;
-        PyObject *name = PyUnicode_FromString(fn->m_ml->ml_name);
+        PyObject *name = fn->m_base.name;
         PyObject *modname = fn->m_module;
 
-        if (name != NULL) {
-            PyObject *mo = _PyType_Lookup(Py_TYPE(self), name);
-            Py_XINCREF(mo);
-            Py_DECREF(name);
-            if (mo != NULL) {
-                PyObject *res = PyObject_Repr(mo);
-                Py_DECREF(mo);
-                if (res != NULL)
-                    return res;
-            }
+        PyObject *mo = _PyType_Lookup(Py_TYPE(self), name);
+        Py_XINCREF(mo);
+        if (mo != NULL) {
+            PyObject *res = PyObject_Repr(mo);
+            Py_DECREF(mo);
+            if (res != NULL)
+                return res;
         }
+
         /* Otherwise, use __module__ */
         PyErr_Clear();
         if (modname != NULL && PyUnicode_Check(modname))
-            return PyUnicode_FromFormat("<built-in method %S.%s>",
-                                        modname,  fn->m_ml->ml_name);
+            return PyUnicode_FromFormat("<built-in method %S.%S>",
+                                        modname,  fn->m_base.name);
         else
-            return PyUnicode_FromFormat("<built-in method %s>",
-                                        fn->m_ml->ml_name);
+            return PyUnicode_FromFormat("<built-in method %S>",
+                                        fn->m_base.name);
     }
 }
 
@@ -409,7 +407,7 @@ profiler_callback(PyObject *self, PyFrameObject *frame, int what,
         if ((((ProfilerObject *)self)->flags & POF_BUILTINS)
             && PyCFunction_Check(arg)) {
             ptrace_enter_call(self,
-                              ((PyCFunctionObject *)arg)->m_ml,
+                              &((PyCFunctionObject *)arg)->m_base,
                               arg);
         }
         break;
@@ -421,7 +419,7 @@ profiler_callback(PyObject *self, PyFrameObject *frame, int what,
         if ((((ProfilerObject *)self)->flags & POF_BUILTINS)
             && PyCFunction_Check(arg)) {
             ptrace_leave_call(self,
-                              ((PyCFunctionObject *)arg)->m_ml);
+                              &((PyCFunctionObject *)arg)->m_base);
         }
         break;
 

Objects/abstract.c

@@ -5,6 +5,7 @@
 #include <ctype.h>
 #include "structmember.h" /* we need the offsetof() macro from there */
 #include "longintrepr.h"
+#include "bltinmodule.h"
 
 
 
@@ -840,7 +841,7 @@ binary_op(PyObject *v, PyObject *w, const int op_slot, const char *op_name)
 
         if (op_slot == NB_SLOT(nb_rshift) &&
             PyCFunction_Check(v) &&
-            strcmp(((PyCFunctionObject *)v)->m_ml->ml_name, "print") == 0)
+            PyCFunction_GET_FUNCTION(v) == (PyCFunction)_PyBuiltin_Print)
         {
             PyErr_Format(PyExc_TypeError,
                 "unsupported operand type(s) for %.100s: "