CI, ENH: Check each minimum dependency is enforced in *.yaml and environment.yml

environment.yml

@@ -38,7 +38,7 @@ dependencies:
   - lxml
   - matplotlib>=3.6.1
   - numba>=0.53.1
-  - numexpr>=2.8.0  # pin for "Run checks on imported code" job
+  - numexpr>=2.8.0 # pin for "Run checks on imported code" job
   - openpyxl
   - odfpy
   - py
@@ -85,12 +85,12 @@ dependencies:
   - ruff=0.0.215
 
   # documentation
-  - gitpython  # obtain contributors from git for whatsnew
+  - gitpython # obtain contributors from git for whatsnew
   - gitdb
-  - natsort  # DataFrame.sort_values doctest
+  - natsort # DataFrame.sort_values doctest
   - numpydoc
   - pydata-sphinx-theme<0.11
-  - pytest-cython  # doctest
+  - pytest-cython # doctest
   - sphinx
   - sphinx-panels
   - sphinx-copybutton
@@ -109,7 +109,7 @@ dependencies:
   - ipykernel
 
   # web
-  - jinja2  # in optional dependencies, but documented here as needed
+  - jinja2 # in optional dependencies, but documented here as needed
   - markdown
   - feedparser
   - pyyaml

scripts/validate_min_versions_in_sync.py

@@ -14,9 +14,12 @@
 """
 from __future__ import annotations
 
+import os
 import pathlib
 import sys
 
+import yaml
+
 if sys.version_info >= (3, 11):
     import tomllib
 else:
@@ -28,6 +31,8 @@
 )
 CODE_PATH = pathlib.Path("pandas/compat/_optional.py").resolve()
 SETUP_PATH = pathlib.Path("pyproject.toml").resolve()
+YAML_PATH = pathlib.Path("ci/deps")
+ENV_PATH = pathlib.Path("environment.yml")
 EXCLUDE_DEPS = {"tzdata", "blosc"}
 # pandas package is not available
 # in pre-commit environment
@@ -41,6 +46,84 @@
 import _optional
 
 
+def pin_min_versions_to_ci_deps():
+    exclusion_list = {
+        "python=3.8[build=*_pypy]": None,
+    }
+    toml_dependencies = get_versions_from_toml()
+    all_yaml_files = list(YAML_PATH.iterdir())
+    all_yaml_files.append(ENV_PATH)
+    for curr_file in all_yaml_files:
+        with open(curr_file) as yaml_f:
+            data = yaml_f.read()
+            yaml_file = yaml.safe_load(data)
+            yaml_deps = yaml_file["dependencies"]
+            res = []
+            [res.append(x) for x in yaml_deps if x not in res]
+            yaml_deps = res
+            for dep_line in yaml_deps:
+                replace_text = operator = yaml_left = ""
+                search_text = str(dep_line)
+                if str(dep_line) in exclusion_list:
+                    continue
+                if ">=" in dep_line:
+                    operator = ">="
+                elif "==" in dep_line:
+                    operator = "=="
+                elif "=" in dep_line:
+                    operator = "="
+                elif "<" in dep_line:
+                    operator = "<"
+                elif ">" in dep_line:
+                    operator = ">"
+                else:
+                    operator = ""
+                if operator == "":
+                    yaml_package, yaml_version = str(dep_line).strip(), None
+                    yaml_left = yaml_package
+                else:
+                    yaml_package, yaml_version = str(dep_line).strip().split(operator)
+                    if operator == "<" or operator == ">":
+                        if yaml_package in toml_dependencies:
+                            if version.parse(yaml_version) <= version.parse(
+                                toml_dependencies[yaml_package]
+                            ):
+                                yaml_left = yaml_package
+                            else:
+                                yaml_left = str(dep_line) + ", "
+                    else:
+                        yaml_left = yaml_package + operator
+                if yaml_package in toml_dependencies:
+                    if ">" in yaml_left or "<" in yaml_left:
+                        if "," in yaml_left:
+                            # ex: "numpy<1.24.0," + ">=" + "1.2"
+                            replace_text = (
+                                yaml_left + ">=" + toml_dependencies[yaml_package]
+                            )
+                        else:
+                            replace_text = yaml_left + toml_dependencies[yaml_package]
+                    # update yaml package version to TOML min version
+                    elif yaml_version is not None:
+                        if version.parse(
+                            toml_dependencies[yaml_package]
+                        ) > version.parse(yaml_version):
+                            # ex: "hypothesis>=" + "6.34.2"
+                            replace_text = yaml_left + toml_dependencies[yaml_package]
+                        elif version.parse(
+                            toml_dependencies[yaml_package]
+                        ) == version.parse(yaml_version):
+                            replace_text = dep_line
+                    else:
+                        # ex: "hypothesis + ">=" + 6.34.2"
+                        replace_text = (
+                            yaml_package + ">=" + toml_dependencies[yaml_package]
+                        )
+                    data = data.replace(search_text, replace_text)
+            os.remove(curr_file)
+            with open(curr_file, "w") as f:
+                f.write(data)
+
+
 def get_versions_from_code() -> dict[str, str]:
     """Min versions for checking within pandas code."""
     install_map = _optional.INSTALL_MAPPING
@@ -108,11 +191,11 @@ def get_versions_from_toml() -> dict[str, str]:
 
     for item in EXCLUDE_DEPS:
         optional_dependencies.pop(item, None)
-
     return optional_dependencies
 
 
 def main():
+    pin_min_versions_to_ci_deps()
     with open(CI_PATH, encoding="utf-8") as f:
         _, ci_optional = get_versions_from_ci(f.readlines())
     code_optional = get_versions_from_code()