Merge branch 'main' into superopt_spec

* main:
  gh-103801: Tools/wasm linting and formatting (#103796)
  gh-103673: Add missing ForkingUnixStreamServer and ForkingUnixDatagramServer socketservers (#103674)
  gh-95795: Move types.next_version_tag to PyInterpreterState (gh-102343)

Author: carljm

Doc/library/socketserver.rst

@@ -140,9 +140,16 @@ server is the address family.
            ForkingUDPServer
            ThreadingTCPServer
            ThreadingUDPServer
+           ForkingUnixStreamServer
+           ForkingUnixDatagramServer
+           ThreadingUnixStreamServer
+           ThreadingUnixDatagramServer
 
    These classes are pre-defined using the mix-in classes.
 
+.. versionadded:: 3.12
+   The ``ForkingUnixStreamServer`` and ``ForkingUnixDatagramServer`` classes
+   were added.
 
 To implement a service, you must derive a class from :class:`BaseRequestHandler`
 and redefine its :meth:`~BaseRequestHandler.handle` method.

Include/internal/pycore_runtime.h

@@ -25,6 +25,7 @@ extern "C" {
 #include "pycore_signal.h"          // struct _signals_runtime_state
 #include "pycore_time.h"            // struct _time_runtime_state
 #include "pycore_tracemalloc.h"     // struct _tracemalloc_runtime_state
+#include "pycore_typeobject.h"      // struct types_runtime_state
 #include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_ids
 
 struct _getargs_runtime_state {
@@ -150,13 +151,7 @@ typedef struct pyruntimestate {
     struct _py_object_runtime_state object_state;
     struct _Py_float_runtime_state float_state;
     struct _Py_unicode_runtime_state unicode_state;
-
-    struct {
-        /* Used to set PyTypeObject.tp_version_tag */
-        // bpo-42745: next_version_tag remains shared by all interpreters
-        // because of static types.
-        unsigned int next_version_tag;
-    } types;
+    struct _types_runtime_state types;
 
     /* All the objects that are shared by the runtime's interpreters. */
     struct _Py_static_objects static_objects;

Include/internal/pycore_runtime_init.h

@@ -112,6 +112,9 @@ extern PyTypeObject _PyExc_MemoryError;
         .func_state = { \
             .next_version = 1, \
         }, \
+        .types = { \
+            .next_version_tag = _Py_TYPE_BASE_VERSION_TAG, \
+        }, \
         .static_objects = { \
             .singletons = { \
                 ._not_used = 1, \

Include/internal/pycore_typeobject.h

@@ -11,22 +11,17 @@ extern "C" {
 #endif
 
 
-/* runtime lifecycle */
-
-extern PyStatus _PyTypes_InitTypes(PyInterpreterState *);
-extern void _PyTypes_FiniTypes(PyInterpreterState *);
-extern void _PyTypes_Fini(PyInterpreterState *);
-
+/* state */
 
-/* other API */
-
-/* Length of array of slotdef pointers used to store slots with the
-   same __name__.  There should be at most MAX_EQUIV-1 slotdef entries with
-   the same __name__, for any __name__. Since that's a static property, it is
-   appropriate to declare fixed-size arrays for this. */
-#define MAX_EQUIV 10
+#define _Py_TYPE_BASE_VERSION_TAG (2<<16)
+#define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1)
 
-typedef struct wrapperbase pytype_slotdef;
+struct _types_runtime_state {
+    /* Used to set PyTypeObject.tp_version_tag for core static types. */
+    // bpo-42745: next_version_tag remains shared by all interpreters
+    // because of static types.
+    unsigned int next_version_tag;
+};
 
 
 // Type attribute lookup cache: speed up attribute and method lookups,
@@ -57,6 +52,36 @@ typedef struct {
     PyObject *tp_weaklist;
 } static_builtin_state;
 
+struct types_state {
+    /* Used to set PyTypeObject.tp_version_tag.
+       It starts at _Py_MAX_GLOBAL_TYPE_VERSION_TAG + 1,
+       where all those lower numbers are used for core static types. */
+    unsigned int next_version_tag;
+
+    struct type_cache type_cache;
+    size_t num_builtins_initialized;
+    static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES];
+};
+
+
+/* runtime lifecycle */
+
+extern PyStatus _PyTypes_InitTypes(PyInterpreterState *);
+extern void _PyTypes_FiniTypes(PyInterpreterState *);
+extern void _PyTypes_Fini(PyInterpreterState *);
+
+
+/* other API */
+
+/* Length of array of slotdef pointers used to store slots with the
+   same __name__.  There should be at most MAX_EQUIV-1 slotdef entries with
+   the same __name__, for any __name__. Since that's a static property, it is
+   appropriate to declare fixed-size arrays for this. */
+#define MAX_EQUIV 10
+
+typedef struct wrapperbase pytype_slotdef;
+
+
 static inline PyObject **
 _PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state)
 {
@@ -78,12 +103,6 @@ _PyType_GetModuleState(PyTypeObject *type)
     return mod->md_state;
 }
 
-struct types_state {
-    struct type_cache type_cache;
-    size_t num_builtins_initialized;
-    static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES];
-};
-
 
 extern int _PyStaticType_InitBuiltin(PyTypeObject *type);
 extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *);

Lib/socketserver.py

@@ -141,6 +141,8 @@ class will essentially render the service "deaf" while one request is
     __all__.extend(["UnixStreamServer","UnixDatagramServer",
                     "ThreadingUnixStreamServer",
                     "ThreadingUnixDatagramServer"])
+    if hasattr(os, "fork"):
+        __all__.extend(["ForkingUnixStreamServer", "ForkingUnixDatagramServer"])
 
 # poll/select have the advantage of not requiring any extra file descriptor,
 # contrarily to epoll/kqueue (also, they require a single syscall).
@@ -727,6 +729,11 @@ class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): pass
 
     class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): pass
 
+    if hasattr(os, "fork"):
+        class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer): pass
+
+        class ForkingUnixDatagramServer(ForkingMixIn, UnixDatagramServer): pass
+
 class BaseRequestHandler:
 
     """Base class for request handler classes.

Lib/test/test_socketserver.py

@@ -47,14 +47,6 @@ def receive(sock, n, timeout=test.support.SHORT_TIMEOUT):
     else:
         raise RuntimeError("timed out on %r" % (sock,))
 
-if HAVE_UNIX_SOCKETS and HAVE_FORKING:
-    class ForkingUnixStreamServer(socketserver.ForkingMixIn,
-                                  socketserver.UnixStreamServer):
-        pass
-
-    class ForkingUnixDatagramServer(socketserver.ForkingMixIn,
-                                    socketserver.UnixDatagramServer):
-        pass
 
 @test.support.requires_fork()
 @contextlib.contextmanager
@@ -211,7 +203,7 @@ def test_ThreadingUnixStreamServer(self):
     @requires_forking
     def test_ForkingUnixStreamServer(self):
         with simple_subprocess(self):
-            self.run_server(ForkingUnixStreamServer,
+            self.run_server(socketserver.ForkingUnixStreamServer,
                             socketserver.StreamRequestHandler,
                             self.stream_examine)
 
@@ -247,7 +239,7 @@ def test_ThreadingUnixDatagramServer(self):
     @requires_unix_sockets
     @requires_forking
     def test_ForkingUnixDatagramServer(self):
-        self.run_server(ForkingUnixDatagramServer,
+        self.run_server(socketserver.ForkingUnixDatagramServer,
                         socketserver.DatagramRequestHandler,
                         self.dgram_examine)
 

Misc/ACKS

@@ -160,6 +160,7 @@ Brice Berna
 Olivier Bernard
 Vivien Bernet-Rollande
 Maxwell Bernstein
+Jay Berry
 Eric Beser
 Steven Bethard
 Stephen Bevan

Misc/NEWS.d/next/Core and Builtins/2023-04-24-21-47-38.gh-issue-103801.WaBanq.rst

@@ -0,0 +1 @@
+Adds three minor linting fixes to the wasm module caught that were caught by ruff.

Misc/NEWS.d/next/Library/2023-04-22-02-41-06.gh-issue-103673.oE7S_k.rst

@@ -0,0 +1,2 @@
+:mod:`socketserver` gains ``ForkingUnixStreamServer`` and
+``ForkingUnixDatagramServer`` classes. Patch by Jay Berry.

Objects/typeobject.c

@@ -45,7 +45,9 @@ class object "PyObject *" "&PyBaseObject_Type"
         PyUnicode_IS_READY(name) &&                             \
         (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE)
 
-#define next_version_tag (_PyRuntime.types.next_version_tag)
+#define NEXT_GLOBAL_VERSION_TAG _PyRuntime.types.next_version_tag
+#define NEXT_VERSION_TAG(interp) \
+    (interp)->types.next_version_tag
 
 typedef struct PySlot_Offset {
     short subslot_offset;
@@ -332,7 +334,7 @@ _PyType_ClearCache(PyInterpreterState *interp)
     // use Py_SETREF() rather than using slower Py_XSETREF().
     type_cache_clear(cache, Py_None);
 
-    return next_version_tag - 1;
+    return NEXT_VERSION_TAG(interp) - 1;
 }
 
 
@@ -401,7 +403,7 @@ PyType_ClearWatcher(int watcher_id)
     return 0;
 }
 
-static int assign_version_tag(PyTypeObject *type);
+static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type);
 
 int
 PyType_Watch(int watcher_id, PyObject* obj)
@@ -416,7 +418,7 @@ PyType_Watch(int watcher_id, PyObject* obj)
         return -1;
     }
     // ensure we will get a callback on the next modification
-    assign_version_tag(type);
+    assign_version_tag(interp, type);
     type->tp_watched |= (1 << watcher_id);
     return 0;
 }
@@ -549,7 +551,9 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) {
         }
     }
     return;
+
  clear:
+    assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN));
     type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
     type->tp_version_tag = 0; /* 0 is not a valid version tag */
     if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
@@ -560,7 +564,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) {
 }
 
 static int
-assign_version_tag(PyTypeObject *type)
+assign_version_tag(PyInterpreterState *interp, PyTypeObject *type)
 {
     /* Ensure that the tp_version_tag is valid and set
        Py_TPFLAGS_VALID_VERSION_TAG.  To respect the invariant, this
@@ -574,18 +578,30 @@ assign_version_tag(PyTypeObject *type)
         return 0;
     }
 
-    if (next_version_tag == 0) {
-        /* We have run out of version numbers */
-        return 0;
+    if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
+        /* static types */
+        if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) {
+            /* We have run out of version numbers */
+            return 0;
+        }
+        type->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++;
+        assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG);
+    }
+    else {
+        /* heap types */
+        if (NEXT_VERSION_TAG(interp) == 0) {
+            /* We have run out of version numbers */
+            return 0;
+        }
+        type->tp_version_tag = NEXT_VERSION_TAG(interp)++;
+        assert (type->tp_version_tag != 0);
     }
-    type->tp_version_tag = next_version_tag++;
-    assert (type->tp_version_tag != 0);
 
     PyObject *bases = type->tp_bases;
     Py_ssize_t n = PyTuple_GET_SIZE(bases);
     for (Py_ssize_t i = 0; i < n; i++) {
         PyObject *b = PyTuple_GET_ITEM(bases, i);
-        if (!assign_version_tag(_PyType_CAST(b)))
+        if (!assign_version_tag(interp, _PyType_CAST(b)))
             return 0;
     }
     type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
@@ -594,7 +610,8 @@ assign_version_tag(PyTypeObject *type)
 
 int PyUnstable_Type_AssignVersionTag(PyTypeObject *type)
 {
-    return assign_version_tag(type);
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    return assign_version_tag(interp, type);
 }
 
 
@@ -2346,7 +2363,15 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro)
        from the custom MRO */
     type_mro_modified(type, type->tp_bases);
 
-    PyType_Modified(type);
+    // XXX Expand this to Py_TPFLAGS_IMMUTABLETYPE?
+    if (!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)) {
+        PyType_Modified(type);
+    }
+    else {
+        /* For static builtin types, this is only called during init
+           before the method cache has been populated. */
+        assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG));
+    }
 
     if (p_old_mro != NULL)
         *p_old_mro = old_mro;  /* transfer the ownership */
@@ -4181,6 +4206,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name)
 {
     PyObject *res;
     int error;
+    PyInterpreterState *interp = _PyInterpreterState_GET();
 
     unsigned int h = MCACHE_HASH_METHOD(type, name);
     struct type_cache *cache = get_type_cache();
@@ -4215,7 +4241,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name)
         return NULL;
     }
 
-    if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(type)) {
+    if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(interp, type)) {
         h = MCACHE_HASH_METHOD(type, name);
         struct type_cache_entry *entry = &cache->hashtable[h];
         entry->version = type->tp_version_tag;
@@ -6676,8 +6702,11 @@ type_ready_mro(PyTypeObject *type)
     assert(type->tp_mro != NULL);
     assert(PyTuple_Check(type->tp_mro));
 
-    /* All bases of statically allocated type should be statically allocated */
+    /* All bases of statically allocated type should be statically allocated,
+       and static builtin types must have static builtin bases. */
     if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+        assert(type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE);
+        int isbuiltin = type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN;
         PyObject *mro = type->tp_mro;
         Py_ssize_t n = PyTuple_GET_SIZE(mro);
         for (Py_ssize_t i = 0; i < n; i++) {
@@ -6689,6 +6718,7 @@ type_ready_mro(PyTypeObject *type)
                              type->tp_name, base->tp_name);
                 return -1;
             }
+            assert(!isbuiltin || (base->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN));
         }
     }
     return 0;
@@ -7000,7 +7030,11 @@ PyType_Ready(PyTypeObject *type)
 int
 _PyStaticType_InitBuiltin(PyTypeObject *self)
 {
-    self->tp_flags = self->tp_flags | _Py_TPFLAGS_STATIC_BUILTIN;
+    self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN;
+
+    assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG);
+    self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++;
+    self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
 
     static_builtin_state_init(self);
 

Tools/wasm/wasm_assets.py

@@ -6,7 +6,8 @@
 
 - a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
 - os.py as marker module {PREFIX}/lib/python3.11/os.py
-- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty
+- empty lib-dynload directory, to make sure it is copied into the bundle:
+    {PREFIX}/lib/python3.11/lib-dynload/.empty
 """
 
 import argparse

Tools/wasm/wasm_build.py

@@ -73,7 +73,7 @@
 run "make clean -C '{SRCDIR}'".
 """
 
-INSTALL_NATIVE = f"""
+INSTALL_NATIVE = """
 Builds require a C compiler (gcc, clang), make, pkg-config, and development
 headers for dependencies like zlib.
 
@@ -598,7 +598,7 @@ def run_browser(self, bind="127.0.0.1", port=8000):
         end = time.monotonic() + 3.0
         while time.monotonic() < end and srv.returncode is None:
             try:
-                with socket.create_connection((bind, port), timeout=0.1) as s:
+                with socket.create_connection((bind, port), timeout=0.1) as _:
                     pass
             except OSError:
                 time.sleep(0.01)