[8.0.x] Improve GitHub release workflow

Author: nicoddemus

.github/workflows/deploy.yml

@@ -82,9 +82,14 @@ jobs:
         python -m pip install --upgrade pip
         pip install --upgrade tox
 
-    - name: Publish GitHub release notes
-      env:
-        GH_RELEASE_NOTES_TOKEN: ${{ github.token }}
+    - name: Generate release notes
       run: |
         sudo apt-get install pandoc
-        tox -e publish-gh-release-notes
+        tox -e generate-gh-release-notes -- ${{ github.event.inputs.version }} scripts/latest-release-notes.md
+
+    - name: Publish GitHub Release
+      uses: softprops/action-gh-release@v1
+      with:
+        body_path: scripts/latest-release-notes.md
+        files: dist/*
+        tag_name: ${{ github.event.inputs.version }}

.pre-commit-config.yaml

@@ -59,14 +59,15 @@ repos:
     rev: v1.8.0
     hooks:
     -   id: mypy
-        files: ^(src/|testing/)
+        files: ^(src/|testing/|scripts/)
         args: []
         additional_dependencies:
           - iniconfig>=1.1.0
           - attrs>=19.2.0
           - packaging
           - tomli
           - types-pkg_resources
+          - types-tabulate
             # for mypy running on python>=3.11 since exceptiongroup is only a dependency
             # on <3.11
           - exceptiongroup>=1.0.0rc8

scripts/.gitignore

@@ -0,0 +1 @@
+latest-release-notes.md

scripts/publish-gh-release-notes.py renamed to scripts/generate-gh-release-notes.py

@@ -1,3 +1,4 @@
+# mypy: disallow-untyped-defs
 """
 Script used to publish GitHub release notes extracted from CHANGELOG.rst.
 
@@ -19,27 +20,19 @@
 
 Requires Python3.6+.
 """
-import os
 import re
 import sys
 from pathlib import Path
+from typing import Sequence
 
-import github3
 import pypandoc
 
 
-def publish_github_release(slug, token, tag_name, body):
-    github = github3.login(token=token)
-    owner, repo = slug.split("/")
-    repo = github.repository(owner, repo)
-    return repo.create_release(tag_name=tag_name, body=body)
-
-
-def parse_changelog(tag_name):
+def parse_changelog(tag_name: str) -> str:
     p = Path(__file__).parent.parent / "doc/en/changelog.rst"
     changelog_lines = p.read_text(encoding="UTF-8").splitlines()
 
-    title_regex = re.compile(r"pytest (\d\.\d+\.\d+) \(\d{4}-\d{2}-\d{2}\)")
+    title_regex = re.compile(r"pytest (\d\.\d+\.\d+\w*) \(\d{4}-\d{2}-\d{2}\)")
     consuming_version = False
     version_lines = []
     for line in changelog_lines:
@@ -57,43 +50,26 @@ def parse_changelog(tag_name):
     return "\n".join(version_lines)
 
 
-def convert_rst_to_md(text):
-    return pypandoc.convert_text(
+def convert_rst_to_md(text: str) -> str:
+    result = pypandoc.convert_text(
         text, "md", format="rst", extra_args=["--wrap=preserve"]
     )
+    assert isinstance(result, str), repr(result)
+    return result
 
 
-def main(argv):
-    if len(argv) > 1:
-        tag_name = argv[1]
-    else:
-        tag_name = os.environ.get("GITHUB_REF")
-        if not tag_name:
-            print("tag_name not given and $GITHUB_REF not set", file=sys.stderr)
-            return 1
-        if tag_name.startswith("refs/tags/"):
-            tag_name = tag_name[len("refs/tags/") :]
-
-    token = os.environ.get("GH_RELEASE_NOTES_TOKEN")
-    if not token:
-        print("GH_RELEASE_NOTES_TOKEN not set", file=sys.stderr)
-        return 1
-
-    slug = os.environ.get("GITHUB_REPOSITORY")
-    if not slug:
-        print("GITHUB_REPOSITORY not set", file=sys.stderr)
-        return 1
-
-    rst_body = parse_changelog(tag_name)
-    md_body = convert_rst_to_md(rst_body)
-    if not publish_github_release(slug, token, tag_name, md_body):
-        print("Could not publish release notes:", file=sys.stderr)
-        print(md_body, file=sys.stderr)
-        return 5
+def main(argv: Sequence[str]) -> int:
+    if len(argv) != 3:
+        print("Usage: generate-gh-release-notes VERSION FILE")
+        return 2
 
+    version, filename = argv[1:3]
+    print(f"Generating GitHub release notes for version {version}")
+    rst_body = parse_changelog(version)
+    md_body = convert_rst_to_md(rst_body)
+    Path(filename).write_text(md_body, encoding="UTF-8")
     print()
-    print(f"Release notes for {tag_name} published successfully:")
-    print(f"https://github.com/{slug}/releases/tag/{tag_name}")
+    print(f"Done: {filename}")
     print()
     return 0
 

scripts/prepare-release-pr.py

@@ -1,3 +1,4 @@
+# mypy: disallow-untyped-defs
 """
 This script is part of the pytest release process which is triggered manually in the Actions
 tab of the repository.

scripts/release.py

@@ -1,3 +1,4 @@
+# mypy: disallow-untyped-defs
 """Invoke development tasks."""
 import argparse
 import os
@@ -10,15 +11,15 @@
 from colorama import init
 
 
-def announce(version, template_name, doc_version):
+def announce(version: str, template_name: str, doc_version: str) -> None:
     """Generates a new release announcement entry in the docs."""
     # Get our list of authors
-    stdout = check_output(["git", "describe", "--abbrev=0", "--tags"])
-    stdout = stdout.decode("utf-8")
+    stdout = check_output(["git", "describe", "--abbrev=0", "--tags"], encoding="UTF-8")
     last_version = stdout.strip()
 
-    stdout = check_output(["git", "log", f"{last_version}..HEAD", "--format=%aN"])
-    stdout = stdout.decode("utf-8")
+    stdout = check_output(
+        ["git", "log", f"{last_version}..HEAD", "--format=%aN"], encoding="UTF-8"
+    )
 
     contributors = {
         name
@@ -61,7 +62,7 @@ def announce(version, template_name, doc_version):
     check_call(["git", "add", str(target)])
 
 
-def regen(version):
+def regen(version: str) -> None:
     """Call regendoc tool to update examples and pytest output in the docs."""
     print(f"{Fore.CYAN}[generate.regen] {Fore.RESET}Updating docs")
     check_call(
@@ -70,21 +71,23 @@ def regen(version):
     )
 
 
-def fix_formatting():
+def fix_formatting() -> None:
     """Runs pre-commit in all files to ensure they are formatted correctly"""
     print(
         f"{Fore.CYAN}[generate.fix linting] {Fore.RESET}Fixing formatting using pre-commit"
     )
     call(["pre-commit", "run", "--all-files"])
 
 
-def check_links():
+def check_links() -> None:
     """Runs sphinx-build to check links"""
     print(f"{Fore.CYAN}[generate.check_links] {Fore.RESET}Checking links")
     check_call(["tox", "-e", "docs-checklinks"])
 
 
-def pre_release(version, template_name, doc_version, *, skip_check_links):
+def pre_release(
+    version: str, template_name: str, doc_version: str, *, skip_check_links: bool
+) -> None:
     """Generates new docs, release announcements and creates a local tag."""
     announce(version, template_name, doc_version)
     regen(version)
@@ -102,12 +105,12 @@ def pre_release(version, template_name, doc_version, *, skip_check_links):
     print("Please push your branch and open a PR.")
 
 
-def changelog(version, write_out=False):
+def changelog(version: str, write_out: bool = False) -> None:
     addopts = [] if write_out else ["--draft"]
     check_call(["towncrier", "--yes", "--version", version] + addopts)
 
 
-def main():
+def main() -> None:
     init(autoreset=True)
     parser = argparse.ArgumentParser()
     parser.add_argument("version", help="Release version")

scripts/towncrier-draft-to-file.py

@@ -1,11 +1,12 @@
+# mypy: disallow-untyped-defs
 import sys
 from subprocess import call
 
 
-def main():
+def main() -> int:
     """
-    Platform agnostic wrapper script for towncrier.
-    Fixes the issue (#7251) where windows users are unable to natively run tox -e docs to build pytest docs.
+    Platform-agnostic wrapper script for towncrier.
+    Fixes the issue (#7251) where Windows users are unable to natively run tox -e docs to build pytest docs.
     """
     with open(
         "doc/en/_changelog_towncrier_draft.rst", "w", encoding="utf-8"

scripts/update-plugin-list.py

@@ -1,8 +1,13 @@
+# mypy: disallow-untyped-defs
 import datetime
 import pathlib
 import re
 from textwrap import dedent
 from textwrap import indent
+from typing import Any
+from typing import Iterable
+from typing import Iterator
+from typing import TypedDict
 
 import packaging.version
 import platformdirs
@@ -109,7 +114,17 @@ def pytest_plugin_projects_from_pypi(session: CachedSession) -> dict[str, int]:
     }
 
 
-def iter_plugins():
+class PluginInfo(TypedDict):
+    """Relevant information about a plugin to generate the summary."""
+
+    name: str
+    summary: str
+    last_release: str
+    status: str
+    requires: str
+
+
+def iter_plugins() -> Iterator[PluginInfo]:
     session = get_session()
     name_2_serial = pytest_plugin_projects_from_pypi(session)
 
@@ -136,7 +151,7 @@ def iter_plugins():
                     requires = requirement
                     break
 
-        def version_sort_key(version_string):
+        def version_sort_key(version_string: str) -> Any:
             """
             Return the sort key for the given version string
             returned by the API.
@@ -162,20 +177,20 @@ def version_sort_key(version_string):
         yield {
             "name": name,
             "summary": summary.strip(),
-            "last release": last_release,
+            "last_release": last_release,
             "status": status,
             "requires": requires,
         }
 
 
-def plugin_definitions(plugins):
+def plugin_definitions(plugins: Iterable[PluginInfo]) -> Iterator[str]:
     """Return RST for the plugin list that fits better on a vertical page."""
 
     for plugin in plugins:
         yield dedent(
             f"""
             {plugin['name']}
-               *last release*: {plugin["last release"]},
+               *last release*: {plugin["last_release"]},
                *status*: {plugin["status"]},
                *requires*: {plugin["requires"]}
 
@@ -184,7 +199,7 @@ def plugin_definitions(plugins):
         )
 
 
-def main():
+def main() -> None:
     plugins = [*iter_plugins()]
 
     reference_dir = pathlib.Path("doc", "en", "reference")

tox.ini

@@ -177,18 +177,13 @@ passenv = {[testenv:release]passenv}
 deps = {[testenv:release]deps}
 commands = python scripts/prepare-release-pr.py {posargs}
 
-[testenv:publish-gh-release-notes]
-description = create GitHub release after deployment
+[testenv:generate-gh-release-notes]
+description = generate release notes that can be published as GitHub Release
 basepython = python3
 usedevelop = True
-passenv =
-    GH_RELEASE_NOTES_TOKEN
-    GITHUB_REF
-    GITHUB_REPOSITORY
 deps =
-    github3.py
     pypandoc
-commands = python scripts/publish-gh-release-notes.py {posargs}
+commands = python scripts/generate-gh-release-notes.py {posargs}
 
 [flake8]
 max-line-length = 120