Support recursive extras defined in pyproject.toml (#2905)

* test_package_pyproject: recursive extras

Add regression test for issue #2904

* test_package_pyproject: when project deps has a self-referential extra

the project depends on an extra defined within itself

* Support recursive extras defined in pyproject.toml

Expand extras that reference an extra of the same package name to respect local
changes to package metadata.

Fix #2904

Author: masenf

docs/changelog/2904.bugfix.rst

@@ -0,0 +1,3 @@
+Tox will now expand self-referential extras discovered in package deps to respect local modifications to package
+metadata. This allows a package extra to explicitly depend on another package extra, which previously only worked with
+non-static metadata - by :user:`masenf`.

src/tox/tox_env/python/virtual_env/package/pyproject.py

@@ -33,7 +33,7 @@
 from tox.util.file_view import create_session_view
 
 from ..api import VirtualEnv
-from .util import dependencies_with_extras
+from .util import dependencies_with_extras, dependencies_with_extras_from_markers
 
 if sys.version_info >= (3, 8):  # pragma: no cover (py38+)
     from importlib.metadata import Distribution, PathDistribution
@@ -253,11 +253,17 @@ def _load_deps_from_static(self, for_env: EnvConfigSet) -> list[Requirement] | N
             if dynamic == "dependencies" or (extras and dynamic == "optional-dependencies"):
                 return None  # if any dependencies are dynamic we can just calculate all dynamically
 
-        deps: list[Requirement] = [Requirement(i) for i in project.get("dependencies", [])]
+        deps_with_markers: list[tuple[Requirement, set[str | None]]] = [
+            (Requirement(i), {None}) for i in project.get("dependencies", [])
+        ]
         optional_deps = project.get("optional-dependencies", {})
-        for extra in extras:
-            deps.extend(Requirement(i) for i in optional_deps.get(extra, []))
-        return deps
+        for extra, reqs in optional_deps.items():
+            deps_with_markers.extend((Requirement(req), {extra}) for req in (reqs or []))
+        return dependencies_with_extras_from_markers(
+            deps_with_markers=deps_with_markers,
+            extras=extras,
+            package_name=project.get("name", "."),
+        )
 
     def _load_deps_from_built_metadata(self, for_env: EnvConfigSet) -> list[Requirement]:
         # dependencies might depend on the python environment we're running in => if we build a wheel use that env

src/tox/tox_env/python/virtual_env/package/util.py

@@ -8,7 +8,14 @@
 
 
 def dependencies_with_extras(deps: list[Requirement], extras: set[str], package_name: str) -> list[Requirement]:
-    deps_with_markers = extract_extra_markers(deps)
+    return dependencies_with_extras_from_markers(extract_extra_markers(deps), extras, package_name)
+
+
+def dependencies_with_extras_from_markers(
+    deps_with_markers: list[tuple[Requirement, set[str | None]]],
+    extras: set[str],
+    package_name: str,
+) -> list[Requirement]:
     result: list[Requirement] = []
     found: set[str] = set()
     todo: set[str | None] = extras | {None}

tests/tox_env/python/virtual_env/package/test_package_pyproject.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+from textwrap import dedent
 
 import pytest
 
@@ -92,6 +93,57 @@ def test_package_root_via_testenv(tox_project: ToxProjectCreator, demo_pkg_inlin
             ["A"],
             id="deps_with_dynamic_optional_no_extra",
         ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                dependencies=['foo[alpha]']
+                optional-dependencies.alpha=['A']""",
+            ),
+            "",
+            ["A"],
+            id="deps_reference_extra",
+        ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                dependencies=['A']
+                optional-dependencies.alpha=['B']
+                optional-dependencies.beta=['foo[alpha]']""",
+            ),
+            "beta",
+            ["A", "B"],
+            id="deps_with_recursive_extra",
+        ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                dependencies=['A']
+                optional-dependencies.alpha=['B']
+                optional-dependencies.beta=['foo[alpha]']
+                optional-dependencies.delta=['foo[beta]', 'D']""",
+            ),
+            "delta",
+            ["A", "B", "D"],
+            id="deps_with_two_recursive_extra",
+        ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                optional-dependencies.alpha=['foo[beta]', 'A']
+                optional-dependencies.beta=['foo[alpha]', 'B']""",
+            ),
+            "alpha",
+            ["A", "B"],
+            id="deps_with_circular_recursive_extra",
+        ),
     ],
 )
 def test_pyproject_deps_from_static(