Merge branch 'main' into 3.14-empty-force-color-no-color

Author: hugovk

Doc/conf.py

@@ -28,6 +28,7 @@
     'changes',
     'glossary_search',
     'lexers',
+    'pydoc_topics',
     'pyspecific',
     'sphinx.ext.coverage',
     'sphinx.ext.doctest',

Doc/deprecations/c-api-pending-removal-in-3.18.rst

@@ -0,0 +1,16 @@
+Pending removal in Python 3.18
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* Deprecated private functions (:gh:`128863`):
+
+  * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`.
+  * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`.
+  * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`.
+  * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`.
+  * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`.
+  * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`.
+  * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`.
+
+  The `pythoncapi-compat project
+  <https://github.com/python/pythoncapi-compat/>`__ can be used to get these
+  new public functions on Python 3.13 and older.

Doc/library/asyncio-graph.rst

@@ -0,0 +1,145 @@
+.. currentmodule:: asyncio
+
+
+.. _asyncio-graph:
+
+========================
+Call Graph Introspection
+========================
+
+**Source code:** :source:`Lib/asyncio/graph.py`
+
+-------------------------------------
+
+asyncio has powerful runtime call graph introspection utilities
+to trace the entire call graph of a running *coroutine* or *task*, or
+a suspended *future*.  These utilities and the underlying machinery
+can be used from within a Python program or by external profilers
+and debuggers.
+
+.. versionadded:: next
+
+
+.. function:: print_call_graph(future=None, /, *, file=None, depth=1, limit=None)
+
+   Print the async call graph for the current task or the provided
+   :class:`Task` or :class:`Future`.
+
+   This function prints entries starting from the top frame and going
+   down towards the invocation point.
+
+   The function receives an optional *future* argument.
+   If not passed, the current running task will be used.
+
+   If the function is called on *the current task*, the optional
+   keyword-only *depth* argument can be used to skip the specified
+   number of frames from top of the stack.
+
+   If the optional keyword-only *limit* argument is provided, each call stack
+   in the resulting graph is truncated to include at most ``abs(limit)``
+   entries. If *limit* is positive, the entries left are the closest to
+   the invocation point. If *limit* is negative, the topmost entries are
+   left. If *limit* is omitted or ``None``, all entries are present.
+   If *limit* is ``0``, the call stack is not printed at all, only
+   "awaited by" information is printed.
+
+   If *file* is omitted or ``None``, the function will print
+   to :data:`sys.stdout`.
+
+   **Example:**
+
+   The following Python code:
+
+   .. code-block:: python
+
+      import asyncio
+
+      async def test():
+          asyncio.print_call_graph()
+
+      async def main():
+          async with asyncio.TaskGroup() as g:
+              g.create_task(test())
+
+      asyncio.run(main())
+
+   will print::
+
+      * Task(name='Task-2', id=0x1039f0fe0)
+      + Call stack:
+      |   File 't2.py', line 4, in async test()
+      + Awaited by:
+         * Task(name='Task-1', id=0x103a5e060)
+            + Call stack:
+            |   File 'taskgroups.py', line 107, in async TaskGroup.__aexit__()
+            |   File 't2.py', line 7, in async main()
+
+.. function:: format_call_graph(future=None, /, *, depth=1, limit=None)
+
+   Like :func:`print_call_graph`, but returns a string.
+   If *future* is ``None`` and there's no current task,
+   the function returns an empty string.
+
+
+.. function:: capture_call_graph(future=None, /, *, depth=1, limit=None)
+
+   Capture the async call graph for the current task or the provided
+   :class:`Task` or :class:`Future`.
+
+   The function receives an optional *future* argument.
+   If not passed, the current running task will be used. If there's no
+   current task, the function returns ``None``.
+
+   If the function is called on *the current task*, the optional
+   keyword-only *depth* argument can be used to skip the specified
+   number of frames from top of the stack.
+
+   Returns a ``FutureCallGraph`` data class object:
+
+   * ``FutureCallGraph(future, call_stack, awaited_by)``
+
+      Where *future* is a reference to a :class:`Future` or
+      a :class:`Task` (or their subclasses.)
+
+      ``call_stack`` is a tuple of ``FrameCallGraphEntry`` objects.
+
+      ``awaited_by`` is a tuple of ``FutureCallGraph`` objects.
+
+   * ``FrameCallGraphEntry(frame)``
+
+      Where *frame* is a frame object of a regular Python function
+      in the call stack.
+
+
+Low level utility functions
+===========================
+
+To introspect an async call graph asyncio requires cooperation from
+control flow structures, such as :func:`shield` or :class:`TaskGroup`.
+Any time an intermediate :class:`Future` object with low-level APIs like
+:meth:`Future.add_done_callback() <asyncio.Future.add_done_callback>` is
+involved, the following two functions should be used to inform asyncio
+about how exactly such intermediate future objects are connected with
+the tasks they wrap or control.
+
+
+.. function:: future_add_to_awaited_by(future, waiter, /)
+
+   Record that *future* is awaited on by *waiter*.
+
+   Both *future* and *waiter* must be instances of
+   :class:`Future` or :class:`Task` or their subclasses,
+   otherwise the call would have no effect.
+
+   A call to ``future_add_to_awaited_by()`` must be followed by an
+   eventual call to the :func:`future_discard_from_awaited_by` function
+   with the same arguments.
+
+
+.. function:: future_discard_from_awaited_by(future, waiter, /)
+
+   Record that *future* is no longer awaited on by *waiter*.
+
+   Both *future* and *waiter* must be instances of
+   :class:`Future` or :class:`Task` or their subclasses, otherwise
+   the call would have no effect.

Doc/library/asyncio.rst

@@ -99,6 +99,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`:
    asyncio-subprocess.rst
    asyncio-queue.rst
    asyncio-exceptions.rst
+   asyncio-graph.rst
 
 .. toctree::
    :caption: Low-level APIs

Doc/library/inspect.rst

@@ -150,6 +150,12 @@ attributes (see :ref:`import-mod-attrs` for module attributes):
 |                 | f_locals          | local namespace seen by   |
 |                 |                   | this frame                |
 +-----------------+-------------------+---------------------------+
+|                 | f_generator       | returns the generator or  |
+|                 |                   | coroutine object that     |
+|                 |                   | owns this frame, or       |
+|                 |                   | ``None`` if the frame is  |
+|                 |                   | of a regular function     |
++-----------------+-------------------+---------------------------+
 |                 | f_trace           | tracing function for this |
 |                 |                   | frame, or ``None``        |
 +-----------------+-------------------+---------------------------+
@@ -310,6 +316,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes):
 
    Add ``__builtins__`` attribute to functions.
 
+.. versionchanged:: next
+
+   Add ``f_generator`` attribute to frames.
+
 .. function:: getmembers(object[, predicate])
 
    Return all the members of an object in a list of ``(name, value)``

Doc/tools/extensions/pydoc_topics.py

@@ -0,0 +1,187 @@
+"""Support for building "topic help" for pydoc."""
+
+from __future__ import annotations
+
+from time import asctime
+from typing import TYPE_CHECKING
+
+from sphinx.builders.text import TextBuilder
+from sphinx.util import logging
+from sphinx.util.display import status_iterator
+from sphinx.util.docutils import new_document
+from sphinx.writers.text import TextTranslator
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence, Set
+
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+logger = logging.getLogger(__name__)
+
+_PYDOC_TOPIC_LABELS: Sequence[str] = sorted({
+    "assert",
+    "assignment",
+    "assignment-expressions",
+    "async",
+    "atom-identifiers",
+    "atom-literals",
+    "attribute-access",
+    "attribute-references",
+    "augassign",
+    "await",
+    "binary",
+    "bitwise",
+    "bltin-code-objects",
+    "bltin-ellipsis-object",
+    "bltin-null-object",
+    "bltin-type-objects",
+    "booleans",
+    "break",
+    "callable-types",
+    "calls",
+    "class",
+    "comparisons",
+    "compound",
+    "context-managers",
+    "continue",
+    "conversions",
+    "customization",
+    "debugger",
+    "del",
+    "dict",
+    "dynamic-features",
+    "else",
+    "exceptions",
+    "execmodel",
+    "exprlists",
+    "floating",
+    "for",
+    "formatstrings",
+    "function",
+    "global",
+    "id-classes",
+    "identifiers",
+    "if",
+    "imaginary",
+    "import",
+    "in",
+    "integers",
+    "lambda",
+    "lists",
+    "naming",
+    "nonlocal",
+    "numbers",
+    "numeric-types",
+    "objects",
+    "operator-summary",
+    "pass",
+    "power",
+    "raise",
+    "return",
+    "sequence-types",
+    "shifting",
+    "slicings",
+    "specialattrs",
+    "specialnames",
+    "string-methods",
+    "strings",
+    "subscriptions",
+    "truth",
+    "try",
+    "types",
+    "typesfunctions",
+    "typesmapping",
+    "typesmethods",
+    "typesmodules",
+    "typesseq",
+    "typesseq-mutable",
+    "unary",
+    "while",
+    "with",
+    "yield",
+})
+
+
+class PydocTopicsBuilder(TextBuilder):
+    name = "pydoc-topics"
+
+    def init(self) -> None:
+        super().init()
+        self.topics: dict[str, str] = {}
+
+    def get_outdated_docs(self) -> str:
+        # Return a string describing what an update build will build.
+        return "all pydoc topics"
+
+    def write_documents(self, _docnames: Set[str]) -> None:
+        env = self.env
+
+        labels: dict[str, tuple[str, str, str]]
+        labels = env.domains.standard_domain.labels
+
+        # docname -> list of (topic_label, label_id) pairs
+        doc_labels: dict[str, list[tuple[str, str]]] = {}
+        for topic_label in _PYDOC_TOPIC_LABELS:
+            try:
+                docname, label_id, _section_name = labels[topic_label]
+            except KeyError:
+                logger.warning("label %r not in documentation", topic_label)
+                continue
+            doc_labels.setdefault(docname, []).append((topic_label, label_id))
+
+        for docname, label_ids in status_iterator(
+            doc_labels.items(),
+            "building topics... ",
+            length=len(doc_labels),
+            stringify_func=_display_labels,
+        ):
+            doctree = env.get_and_resolve_doctree(docname, builder=self)
+            doc_ids = doctree.ids
+            for topic_label, label_id in label_ids:
+                document = new_document("<section node>")
+                document.append(doc_ids[label_id])
+                visitor = TextTranslator(document, builder=self)
+                document.walkabout(visitor)
+                self.topics[topic_label] = visitor.body
+
+    def finish(self) -> None:
+        topics_repr = "\n".join(
+            f"    '{topic}': {_repr(self.topics[topic])},"
+            for topic in sorted(self.topics)
+        )
+        topics = f"""\
+# Autogenerated by Sphinx on {asctime()}
+# as part of the release process.
+
+topics = {{
+{topics_repr}
+}}
+"""
+        self.outdir.joinpath("topics.py").write_text(topics, encoding="utf-8")
+
+
+def _display_labels(item: tuple[str, Sequence[tuple[str, str]]]) -> str:
+    _docname, label_ids = item
+    labels = [name for name, _id in label_ids]
+    if len(labels) > 4:
+        return f"{labels[0]}, {labels[1]}, ..., {labels[-2]}, {labels[-1]}"
+    return ", ".join(labels)
+
+
+def _repr(text: str, /) -> str:
+    """Return a triple-single-quoted representation of text."""
+    if "'''" not in text:
+        return f"r'''{text}'''"
+    text = text.replace("\\", "\\\\").replace("'''", r"\'\'\'")
+    return f"'''{text}'''"
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    app.add_builder(PydocTopicsBuilder)
+
+    return {
+        "version": "1.0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }