Merge remote-tracking branch 'origin/main' into env-order

Author: joerick

.github/workflows/test.yml

@@ -62,6 +62,13 @@ jobs:
         sudo apt-get update
         sudo apt-get -y install podman
 
+    # free some space to prevent reaching GHA disk space limits
+    - name: Clean docker images
+      if: runner.os == 'Linux'
+      run: |
+        docker system prune -a -f
+        df -h
+
     - name: Install dependencies
       run: |
         python -m pip install ".[test]"
@@ -103,7 +110,7 @@ jobs:
 
     - name: Set up QEMU
       id: qemu
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
       with:
         platforms: all
 

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
   - id: black
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.288
+  rev: v0.0.290
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]

README.md

@@ -97,7 +97,7 @@ jobs:
       - uses: actions/setup-python@v3
 
       - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.15.0
+        run: python -m pip install cibuildwheel==2.16.0
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse
@@ -213,6 +213,18 @@ Changelog
 
 <!-- this section was generated by bin/update_readme_changelog.py -- do not edit manually -->
 
+### v2.16.0
+
+_18 September 2023_
+
+- ✨ Add the ability to pass additional flags to a build frontend through the [CIBW_BUILD_FRONTEND](https://cibuildwheel.readthedocs.io/en/stable/options/#build-frontend) option (#1588).
+- ✨ The environment variable SOURCE_DATE_EPOCH is now automatically passed through to container Linux builds (useful for [reproducible builds](https://reproducible-builds.org/docs/source-date-epoch/)!) (#1589)
+- 🛠 Updates the prerelease CPython 3.12 version to 3.12.0rc2 (#1604)
+- 🐛 Fix `requires_python` auto-detection from setup.py when the call to `setup()` is within an `if __name__ == "__main__" block (#1613)
+- 🐛 Fix a bug that prevented building Linux wheels in Docker on a Windows host (#1573)
+- 🐛 `--only` can now select prerelease-pythons (#1564)
+- 📚 Docs & examples updates (#1582, #1593, #1598, #1615)
+
 ### v2.15.0
 
 _8 August 2023_
@@ -243,16 +255,6 @@ _10 June 2023_
 - 🛠 Updates the prerelease CPython 3.12 version to 3.12.0b2. (#1516)
 - 🛠 Adds a moving `v<major>.<minor>` tag for use in GitHub Actions workflow files. If you use this, you'll get the latest patch release within a minor version. Additionally, Dependabot won't send you PRs for patch releases. (#1517)
 
-### v2.13.0
-
-_28 May 2023_
-
-- ✨ Adds CPython 3.12 support, under the prerelease flag [CIBW_PRERELEASE_PYTHONS](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons). This version of cibuildwheel uses 3.12.0b1.
-
-    While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.12 will be available in cibuildwheel without the flag. (#1507)
-
-- ✨ Adds the ability to pass arguments to the container engine when the container is created, using the [CIBW_CONTAINER_ENGINE](https://cibuildwheel.readthedocs.io/en/stable/options/#container-engine) option. (#1499)
-
 <!-- END bin/update_readme_changelog.py -->
 
 ---

cibuildwheel/__init__.py

@@ -1,3 +1,3 @@
 from __future__ import annotations
 
-__version__ = "2.15.0"
+__version__ = "2.16.0"

cibuildwheel/linux.py

@@ -16,9 +16,9 @@
 from .typing import PathOrStr
 from .util import (
     AlreadyBuiltWheelError,
+    BuildFrontendConfig,
     BuildSelector,
     NonPlatformWheelError,
-    build_frontend_or_default,
     find_compatible_wheel,
     get_build_verbosity_extra_flags,
     prepare_command,
@@ -177,7 +177,7 @@ def build_in_container(
     for config in platform_configs:
         log.build_start(config.identifier)
         build_options = options.build_options(config.identifier)
-        build_frontend = build_frontend_or_default(build_options.build_frontend)
+        build_frontend = build_options.build_frontend or BuildFrontendConfig("pip")
 
         dependency_constraint_flags: list[PathOrStr] = []
 
@@ -243,9 +243,10 @@ def build_in_container(
             container.call(["rm", "-rf", built_wheel_dir])
             container.call(["mkdir", "-p", built_wheel_dir])
 
-            extra_flags = split_config_settings(build_options.config_settings, build_frontend)
+            extra_flags = split_config_settings(build_options.config_settings, build_frontend.name)
+            extra_flags += build_frontend.args
 
-            if build_frontend == "pip":
+            if build_frontend.name == "pip":
                 extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity)
                 container.call(
                     [
@@ -260,7 +261,7 @@ def build_in_container(
                     ],
                     env=env,
                 )
-            elif build_frontend == "build":
+            elif build_frontend.name == "build":
                 if not 0 <= build_options.build_verbosity < 2:
                     msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring."
                     log.warning(msg)
@@ -422,7 +423,7 @@ def build(options: Options, tmp_path: Path) -> None:  # noqa: ARG001
 
             with OCIContainer(
                 image=build_step.container_image,
-                simulate_32_bit=build_step.platform_tag.endswith("i686"),
+                enforce_32_bit=build_step.platform_tag.endswith("i686"),
                 cwd=container_project_path,
                 engine=options.globals.container_engine,
             ) as container:

cibuildwheel/macos.py

@@ -25,10 +25,10 @@
 from .util import (
     CIBW_CACHE_PATH,
     AlreadyBuiltWheelError,
-    BuildFrontend,
+    BuildFrontendConfig,
+    BuildFrontendName,
     BuildSelector,
     NonPlatformWheelError,
-    build_frontend_or_default,
     call,
     detect_ci_provider,
     download,
@@ -165,7 +165,7 @@ def setup_python(
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
-    build_frontend: BuildFrontend,
+    build_frontend: BuildFrontendName,
 ) -> dict[str, str]:
     tmp.mkdir()
     implementation_id = python_configuration.identifier.split("-")[0]
@@ -334,7 +334,7 @@ def build(options: Options, tmp_path: Path) -> None:
 
         for config in python_configurations:
             build_options = options.build_options(config.identifier)
-            build_frontend = build_frontend_or_default(build_options.build_frontend)
+            build_frontend = build_options.build_frontend or BuildFrontendConfig("pip")
             log.build_start(config.identifier)
 
             identifier_tmp_dir = tmp_path / config.identifier
@@ -357,7 +357,7 @@ def build(options: Options, tmp_path: Path) -> None:
                 config,
                 dependency_constraint_flags,
                 build_options.environment,
-                build_frontend,
+                build_frontend.name,
             )
 
             compatible_wheel = find_compatible_wheel(built_wheels, config.identifier)
@@ -378,9 +378,12 @@ def build(options: Options, tmp_path: Path) -> None:
                 log.step("Building wheel...")
                 built_wheel_dir.mkdir()
 
-                extra_flags = split_config_settings(build_options.config_settings, build_frontend)
+                extra_flags = split_config_settings(
+                    build_options.config_settings, build_frontend.name
+                )
+                extra_flags += build_frontend.args
 
-                if build_frontend == "pip":
+                if build_frontend.name == "pip":
                     extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity)
                     # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
                     # see https://github.com/pypa/cibuildwheel/pull/369
@@ -395,7 +398,7 @@ def build(options: Options, tmp_path: Path) -> None:
                         *extra_flags,
                         env=env,
                     )
-                elif build_frontend == "build":
+                elif build_frontend.name == "build":
                     if not 0 <= build_options.build_verbosity < 2:
                         msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring."
                         log.warning(msg)

cibuildwheel/oci_container.py

@@ -17,7 +17,7 @@
 from typing import IO, Dict, Literal
 
 from .typing import PathOrStr, PopenBytes
-from .util import CIProvider, detect_ci_provider, parse_key_value_string
+from .util import CIProvider, call, detect_ci_provider, parse_key_value_string
 
 ContainerEngineName = Literal["docker", "podman"]
 
@@ -29,7 +29,9 @@ class OCIContainerEngineConfig:
 
     @staticmethod
     def from_config_string(config_string: str) -> OCIContainerEngineConfig:
-        config_dict = parse_key_value_string(config_string, ["name"])
+        config_dict = parse_key_value_string(
+            config_string, ["name"], ["create_args", "create-args"]
+        )
         name = " ".join(config_dict["name"])
         if name not in {"docker", "podman"}:
             msg = f"unknown container engine {name}"
@@ -83,7 +85,7 @@ def __init__(
         self,
         *,
         image: str,
-        simulate_32_bit: bool = False,
+        enforce_32_bit: bool = False,
         cwd: PathOrStr | None = None,
         engine: OCIContainerEngineConfig = DEFAULT_ENGINE,
     ):
@@ -92,7 +94,7 @@ def __init__(
             raise ValueError(msg)
 
         self.image = image
-        self.simulate_32_bit = simulate_32_bit
+        self.enforce_32_bit = enforce_32_bit
         self.cwd = cwd
         self.name: str | None = None
         self.engine = engine
@@ -108,13 +110,24 @@ def __enter__(self) -> OCIContainer:
         if detect_ci_provider() == CIProvider.travis_ci and platform.machine() == "ppc64le":
             network_args = ["--network=host"]
 
-        shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"]
+        simulate_32_bit = False
+        if self.enforce_32_bit:
+            # If the architecture running the image is already the right one
+            # or the image entrypoint takes care of enforcing this, then we don't need to
+            # simulate this
+            container_machine = call(
+                self.engine.name, "run", "--rm", self.image, "uname", "-m", capture_stdout=True
+            ).strip()
+            simulate_32_bit = container_machine != "i686"
+
+        shell_args = ["linux32", "/bin/bash"] if simulate_32_bit else ["/bin/bash"]
 
         subprocess.run(
             [
                 self.engine.name,
                 "create",
                 "--env=CIBUILDWHEEL",
+                "--env=SOURCE_DATE_EPOCH",
                 f"--name={self.name}",
                 "--interactive",
                 "--volume=/:/host",  # ignored on CircleCI

cibuildwheel/options.py

@@ -27,7 +27,7 @@
 from .util import (
     MANYLINUX_ARCHS,
     MUSLLINUX_ARCHS,
-    BuildFrontend,
+    BuildFrontendConfig,
     BuildSelector,
     DependencyConstraints,
     TestSelector,
@@ -92,7 +92,7 @@ class BuildOptions:
     test_requires: list[str]
     test_extras: str
     build_verbosity: int
-    build_frontend: BuildFrontend | Literal["default"]
+    build_frontend: BuildFrontendConfig | None
     config_settings: str
 
     @property
@@ -488,7 +488,6 @@ def build_options(self, identifier: str | None) -> BuildOptions:
         with self.reader.identifier(identifier):
             before_all = self.reader.get("before-all", sep=" && ")
 
-            build_frontend_str = self.reader.get("build-frontend", env_plat=False)
             environment_config = self.reader.get(
                 "environment", table={"item": '{k}="{v}"', "sep": " "}
             )
@@ -506,17 +505,20 @@ def build_options(self, identifier: str | None) -> BuildOptions:
             test_extras = self.reader.get("test-extras", sep=",")
             build_verbosity_str = self.reader.get("build-verbosity")
 
-            build_frontend: BuildFrontend | Literal["default"]
-            if build_frontend_str == "build":
-                build_frontend = "build"
-            elif build_frontend_str == "pip":
-                build_frontend = "pip"
-            elif build_frontend_str == "default":
-                build_frontend = "default"
+            build_frontend_str = self.reader.get(
+                "build-frontend",
+                env_plat=False,
+                table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote},
+            )
+            build_frontend: BuildFrontendConfig | None
+            if not build_frontend_str or build_frontend_str == "default":
+                build_frontend = None
             else:
-                msg = f"cibuildwheel: Unrecognised build frontend {build_frontend_str!r}, only 'pip' and 'build' are supported"
-                print(msg, file=sys.stderr)
-                sys.exit(2)
+                try:
+                    build_frontend = BuildFrontendConfig.from_config_string(build_frontend_str)
+                except ValueError as e:
+                    print(f"cibuildwheel: {e}", file=sys.stderr)
+                    sys.exit(2)
 
             try:
                 environment = parse_environment(environment_config)

cibuildwheel/projectfiles.py

@@ -8,6 +8,43 @@
 from ._compat import tomllib
 
 
+def get_parent(node: ast.AST | None, depth: int = 1) -> ast.AST | None:
+    for _ in range(depth):
+        node = getattr(node, "parent", None)
+    return node
+
+
+def is_main(parent: ast.AST | None) -> bool:
+    if parent is None:
+        return False
+
+    # This would be much nicer with 3.10's pattern matching!
+    if not isinstance(parent, ast.If):
+        return False
+    if not isinstance(parent.test, ast.Compare):
+        return False
+
+    try:
+        (op,) = parent.test.ops
+        (comp,) = parent.test.comparators
+    except ValueError:
+        return False
+
+    if not isinstance(op, ast.Eq):
+        return False
+
+    values = {comp, parent.test.left}
+
+    mains = {x for x in values if isinstance(x, ast.Constant) and x.value == "__main__"}
+    if len(mains) != 1:
+        return False
+    consts = {x for x in values if isinstance(x, ast.Name) and x.id == "__name__"}
+    if len(consts) != 1:
+        return False
+
+    return True
+
+
 class Analyzer(ast.NodeVisitor):
     def __init__(self) -> None:
         self.requires_python: str | None = None
@@ -19,13 +56,22 @@ def visit(self, node: ast.AST) -> None:
         super().visit(node)
 
     def visit_keyword(self, node: ast.keyword) -> None:
+        # Must not be nested except for if __name__ == "__main__"
+
         self.generic_visit(node)
-        # Must not be nested in an if or other structure
         # This will be Module -> Expr -> Call -> keyword
+        parent = get_parent(node, 4)
+        unnested = parent is None
+
+        # This will be Module -> If -> Expr -> Call -> keyword
+        name_main_unnested = (
+            parent is not None and get_parent(parent) is None and is_main(get_parent(node, 3))
+        )
+
         if (
             node.arg == "python_requires"
-            and not hasattr(node.parent.parent.parent, "parent")  # type: ignore[attr-defined]
             and isinstance(node.value, ast.Constant)
+            and (unnested or name_main_unnested)
         ):
             self.requires_python = node.value.value