tests: Test relevant prereleases and allow to ignore releases (#4073)

If a package has a prerelease of a higher version than the highest
released stable version, make sure to test it, too. We consider alpha,
beta, and RC releases.

Also add an option to ignore specific releases (this is related to the
above since the script now pulls in two irrelevant alpha releases of
starlite).

Closes #4030

---------

Co-authored-by: Daniel Szoke <[email protected]>

Author: sentrivana

scripts/populate_tox/README.md

@@ -45,9 +45,15 @@ integration_name: {
          rule2: [package3, package4, ...],
      },
      "python": python_version_specifier,
+     "include": package_version_specifier,
 }
 ```
 
+When talking about version specifiers, we mean
+[version specifiers as defined](https://packaging.python.org/en/latest/specifications/version-specifiers/#id5)
+by the Python Packaging Authority. See also the actual implementation
+in [packaging.specifiers](https://packaging.pypa.io/en/stable/specifiers.html).
+
 ### `package`
 
 The name of the third party package as it's listed on PyPI. The script will
@@ -118,6 +124,35 @@ metadata or the SDK is explicitly not supporting some packages on specific
 Python versions (because of, for example, broken context vars), the `python`
 key can be used.
 
+### `include`
+
+Sometimes we only want to consider testing some specific versions of packages.
+For example, the Starlite package has two alpha prereleases of version 2.0.0, but
+we do not want to test these, since Starlite 2.0 was renamed to Litestar.
+
+The value of the `include` key expects a version specifier defining which
+versions should be considered for testing. For example, since we only want to test
+versions below 2.x in Starlite, we can use
+
+```python
+"starlite": {
+    "include": "<2",
+    ...
+}
+```
+
+The `include` key can also be used to exclude a set of specific versions by using
+`!=` version specifiers. For example, the Starlite restriction above could equivalently
+be expressed like so:
+
+
+```python
+"starlite": {
+    "include": "!=2.0.0a1,!=2.0.0a2",
+    ...
+}
+```
+
 
 ## How-Tos
 

scripts/populate_tox/config.py

@@ -129,6 +129,7 @@
             ],
         },
         "python": "<=3.11",
+        "include": "!=2.0.0a1,!=2.0.0a2",  # these are not relevant as there will never be a stable 2.0 release (starlite continues as litestar)
     },
     "statsig": {
         "package": "statsig",

scripts/populate_tox/populate_tox.py

@@ -111,7 +111,7 @@ def fetch_release(package: str, version: Version) -> dict:
 
 def _prefilter_releases(
     integration: str, releases: dict[str, dict], older_than: Optional[datetime] = None
-) -> list[Version]:
+) -> tuple[list[Version], Optional[Version]]:
     """
     Filter `releases`, removing releases that are for sure unsupported.
 
@@ -120,6 +120,10 @@ def _prefilter_releases(
     they require additional API calls to be made. The purpose of this function is
     to slim down the list so that we don't have to make more API calls than
     necessary for releases that are for sure not supported.
+
+    The function returns a tuple with:
+    - the list of prefiltered releases
+    - an optional prerelease if there is one that should be tested
     """
     min_supported = _MIN_VERSIONS.get(integration)
     if min_supported is not None:
@@ -129,7 +133,14 @@ def _prefilter_releases(
             f"  {integration} doesn't have a minimum version defined in sentry_sdk/integrations/__init__.py. Consider defining one"
         )
 
+    include_versions = None
+    if TEST_SUITE_CONFIG[integration].get("include") is not None:
+        include_versions = SpecifierSet(
+            TEST_SUITE_CONFIG[integration]["include"], prereleases=True
+        )
+
     filtered_releases = []
+    last_prerelease = None
 
     for release, data in releases.items():
         if not data:
@@ -149,9 +160,15 @@ def _prefilter_releases(
         if min_supported and version < min_supported:
             continue
 
-        if version.is_prerelease or version.is_postrelease:
-            # TODO: consider the newest prerelease unless obsolete
-            # https://github.com/getsentry/sentry-python/issues/4030
+        if version.is_postrelease or version.is_devrelease:
+            continue
+
+        if include_versions is not None and version not in include_versions:
+            continue
+
+        if version.is_prerelease:
+            if last_prerelease is None or version > last_prerelease:
+                last_prerelease = version
             continue
 
         for i, saved_version in enumerate(filtered_releases):
@@ -166,18 +183,30 @@ def _prefilter_releases(
         else:
             filtered_releases.append(version)
 
-    return sorted(filtered_releases)
+    filtered_releases.sort()
+
+    # Check if the latest prerelease is relevant (i.e., it's for a version higher
+    # than the last released version); if not, don't consider it
+    if last_prerelease is not None:
+        if not filtered_releases or last_prerelease > filtered_releases[-1]:
+            return filtered_releases, last_prerelease
+
+    return filtered_releases, None
 
 
 def get_supported_releases(
     integration: str, pypi_data: dict, older_than: Optional[datetime] = None
-) -> list[Version]:
+) -> tuple[list[Version], Optional[Version]]:
     """
     Get a list of releases that are currently supported by the SDK.
 
     This takes into account a handful of parameters (Python support, the lowest
     version we've defined for the framework, the date of the release).
 
+    We return the list of supported releases and optionally also the newest
+    prerelease, if it should be tested (meaning it's for a version higher than
+    the current stable version).
+
     If an `older_than` timestamp is provided, no release newer than that will be
     considered.
     """
@@ -186,7 +215,9 @@ def get_supported_releases(
     # Get a consolidated list without taking into account Python support yet
     # (because that might require an additional API call for some
     # of the releases)
-    releases = _prefilter_releases(integration, pypi_data["releases"], older_than)
+    releases, latest_prerelease = _prefilter_releases(
+        integration, pypi_data["releases"], older_than
+    )
 
     # Determine Python support
     expected_python_versions = TEST_SUITE_CONFIG[integration].get("python")
@@ -210,14 +241,18 @@ def _supports_lowest(release: Version) -> bool:
             # version(s) that we do, cut off the rest
             releases = releases[i:]
 
-    return releases
+    return releases, latest_prerelease
 
 
-def pick_releases_to_test(releases: list[Version]) -> list[Version]:
+def pick_releases_to_test(
+    releases: list[Version], last_prerelease: Optional[Version]
+) -> list[Version]:
     """Pick a handful of releases to test from a sorted list of supported releases."""
     # If the package has majors (or major-like releases, even if they don't do
     # semver), we want to make sure we're testing them all. If not, we just pick
     # the oldest, the newest, and a couple in between.
+    #
+    # If there is a relevant prerelease, also test that in addition to the above.
     has_majors = len(set([v.major for v in releases])) > 1
     filtered_releases = set()
 
@@ -252,7 +287,11 @@ def pick_releases_to_test(releases: list[Version]) -> list[Version]:
             releases[-1],  # latest
         }
 
-    return sorted(filtered_releases)
+    filtered_releases = sorted(filtered_releases)
+    if last_prerelease is not None:
+        filtered_releases.append(last_prerelease)
+
+    return filtered_releases
 
 
 def supported_python_versions(
@@ -553,19 +592,24 @@ def main(fail_on_changes: bool = False) -> None:
             pypi_data = fetch_package(package)
 
             # Get the list of all supported releases
-            # If in check mode, ignore releases newer than `last_updated`
+
+            # If in fail-on-changes mode, ignore releases newer than `last_updated`
             older_than = last_updated if fail_on_changes else None
-            releases = get_supported_releases(integration, pypi_data, older_than)
+
+            releases, latest_prerelease = get_supported_releases(
+                integration, pypi_data, older_than
+            )
+
             if not releases:
                 print("  Found no supported releases.")
                 continue
 
             _compare_min_version_with_defined(integration, releases)
 
             # Pick a handful of the supported releases to actually test against
-            # and fetch the PYPI data for each to determine which Python versions
+            # and fetch the PyPI data for each to determine which Python versions
             # to test it on
-            test_releases = pick_releases_to_test(releases)
+            test_releases = pick_releases_to_test(releases, latest_prerelease)
 
             for release in test_releases:
                 _add_python_versions_to_release(integration, package, release)

tox.ini

@@ -10,7 +10,7 @@
 # The file (and all resulting CI YAMLs) then need to be regenerated via
 # "scripts/generate-test-files.sh".
 #
-# Last generated: 2025-02-18T12:57:32.874168+00:00
+# Last generated: 2025-02-19T12:41:15.689786+00:00
 
 [tox]
 requires =
@@ -211,10 +211,11 @@ envlist =
     {py3.8,py3.10,py3.11}-ariadne-v0.20.1
     {py3.8,py3.11,py3.12}-ariadne-v0.22
     {py3.8,py3.11,py3.12}-ariadne-v0.24.0
-    {py3.8,py3.11,py3.12}-ariadne-v0.25.2
+    {py3.9,py3.12,py3.13}-ariadne-v0.26.0
 
     {py3.6,py3.9,py3.10}-gql-v3.4.1
     {py3.7,py3.11,py3.12}-gql-v3.5.0
+    {py3.9,py3.12,py3.13}-gql-v3.6.0b4
 
     {py3.6,py3.9,py3.10}-graphene-v3.3
     {py3.8,py3.12,py3.13}-graphene-v3.4.3
@@ -236,6 +237,7 @@ envlist =
     {py3.6,py3.7,py3.8}-celery-v4.4.7
     {py3.6,py3.7,py3.8}-celery-v5.0.5
     {py3.8,py3.11,py3.12}-celery-v5.4.0
+    {py3.8,py3.12,py3.13}-celery-v5.5.0rc4
 
     {py3.6,py3.7}-dramatiq-v1.9.0
     {py3.6,py3.8,py3.9}-dramatiq-v1.12.3
@@ -592,13 +594,14 @@ deps =
     ariadne-v0.20.1: ariadne==0.20.1
     ariadne-v0.22: ariadne==0.22
     ariadne-v0.24.0: ariadne==0.24.0
-    ariadne-v0.25.2: ariadne==0.25.2
+    ariadne-v0.26.0: ariadne==0.26.0
     ariadne: fastapi
     ariadne: flask
     ariadne: httpx
 
     gql-v3.4.1: gql[all]==3.4.1
     gql-v3.5.0: gql[all]==3.5.0
+    gql-v3.6.0b4: gql[all]==3.6.0b4
 
     graphene-v3.3: graphene==3.3
     graphene-v3.4.3: graphene==3.4.3
@@ -630,6 +633,7 @@ deps =
     celery-v4.4.7: celery==4.4.7
     celery-v5.0.5: celery==5.0.5
     celery-v5.4.0: celery==5.4.0
+    celery-v5.5.0rc4: celery==5.5.0rc4
     celery: newrelic
     celery: redis
     py3.7-celery: importlib-metadata<5.0