[Bazel] Add support for Bzlmod

The workspace build is now gated behind `--config=deprecated_workspace`.

See `utils/bazel/examples` for the new setup instructions.

While not necessarily the most intuitive implementation, this seems to
be the only way retain the flexibility of all potential usecases as of
Bazel 8. The module build supports:

- The in-tree build from within the `utils/bazel` directory.
- The llvm-project cloned into an arbitrary directory (e.g. for local
  development, submodule setups and and Nix users).
- The llvm-project fetched from github via an archive.

Since module overrides are invisible to third-party dependents, every
project that has the llvm project "somewhere" in its dependency tree
needs to create an explicit override in their root module. It's possible
to avoid this inconvenience by creating custom bazel registries, but
only makes sense to do for the next stable tag that contains this commit.

Notes for zlib users
--------------------

The configuration options for zlib-ng and zstd have been moved to
accomodate both the workspace and the module build:

```
// Old
--@llvm-project-overlay//:llvm_enable_zstd
--@llvm-project-overlay//:llvm_enable_zlib

// New
--@llvm-project//llvm:enable_zstd
--@llvm-project//llvm:enable_zlib
```

This also fixes a subtle issue where the way that we used `zlib` and
`zstd` was not idiomatic. Our existing implementation relied on custom
defines exposed by build files. This bled llvm-specific configuration
into what were supposed to be independent third-party dependencies.

If you need to link against nonhermetic system variants of these
libraries use custom overrides, for instance like so:

```python
'''MODULE.bazel'''
new_local_repository = use_repo_rule(
    "@bazel_tools//tools/build_defs/repo:local.bzl",
    "new_local_repository",
)

new_local_repository(
    name = "zlib-ng",
    path = "xxxx",
    build_file_content = "xxxx",
)

override_repo(llvm_project_overlay, "zlib-ng")
```

Author: aaronmondal

utils/bazel/.bazelrc

@@ -9,7 +9,9 @@
 
 # Flip off to disable MODULE.bazel until we're ready.
 # https://github.com/llvm/llvm-project/issues/55924
-common --enable_bzlmod=false --enable_workspace
+# 
+# WARNING: This option will be removed soon. Please migrate to bzlmod.
+common:deprecated_workspace --enable_bzlmod=false --enable_workspace
 
 # TODO: Remove lit test reliance on this
 common --legacy_external_runfiles
@@ -58,14 +60,6 @@ build --experimental_cc_shared_library
 # https://github.com/bazelbuild/bazel/commit/03246077f948f2790a83520e7dccc2625650e6df
 build --build_runfile_links=false
 
-###############################################################################
-# Options to select different strategies for linking potential dependent
-# libraries. The default leaves it disabled.
-###############################################################################
-
-build:zlib_external --repo_env=BAZEL_LLVM_ZLIB_STRATEGY=external
-build:zlib_system --repo_env=BAZEL_LLVM_ZLIB_STRATEGY=system
-
 ###############################################################################
 # Options for "generic_clang" builds: these options should generally apply to
 # builds using a Clang-based compiler, and default to the `clang` executable on

utils/bazel/.bazelversion

@@ -1 +1 @@
-8.0.0
+8.0.1

utils/bazel/MODULE.bazel

@@ -0,0 +1,43 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+module(
+    name = "llvm-project-overlay",
+    version = "main",
+    compatibility_level = 0,
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
+bazel_dep(name = "platforms", version = "0.0.11")
+bazel_dep(name = "rules_cc", version = "0.0.17")
+bazel_dep(name = "rules_python", version = "1.1.0")
+bazel_dep(name = "apple_support", version = "1.17.1")
+bazel_dep(name = "rules_foreign_cc", version = "0.13.0")
+bazel_dep(name = "robin-map", version = "1.3.0")
+bazel_dep(name = "zlib-ng", version = "2.0.7")
+bazel_dep(name = "zstd", version = "1.5.6")
+
+llvm_project_overlay = use_extension(
+    "//:extensions.bzl",
+    "llvm_project_overlay",
+)
+
+# Don't add targets here. An empty list resolves to "all targets" and allows
+# downstream repos to configure subsets.
+llvm_project_overlay.configure()
+
+use_repo(
+    llvm_project_overlay,
+    "llvm-project",
+    "llvm-raw",
+
+    # Vendored.
+    "vulkan_headers",
+    "vulkan_sdk",
+    "gmp",
+    "mpfr",
+    "pfm",
+    "pybind11",
+    "nanobind",
+)

utils/bazel/MODULE.bazel.lock

@@ -43,7 +43,7 @@ maybe(
 
 maybe(
     http_archive,
-    name = "llvm_zlib",
+    name = "zlib-ng",
     build_file = "@llvm-raw//utils/bazel/third_party_build:zlib-ng.BUILD",
     sha256 = "e36bb346c00472a1f9ff2a0a4643e590a254be6379da7cddd9daeb9a7f296731",
     strip_prefix = "zlib-ng-2.0.7",
@@ -71,13 +71,13 @@ maybe(
 )
 
 http_archive(
-    name = "build_bazel_apple_support",
+    name = "apple_support",
     sha256 = "c4bb2b7367c484382300aee75be598b92f847896fb31bbd22f3a2346adf66a80",
     url = "https://github.com/bazelbuild/apple_support/releases/download/1.15.1/apple_support.1.15.1.tar.gz",
 )
 
 load(
-    "@build_bazel_apple_support//lib:repositories.bzl",
+    "@apple_support//lib:repositories.bzl",
     "apple_support_dependencies",
 )
 
@@ -137,7 +137,7 @@ maybe(
 
 maybe(
     http_archive,
-    name = "llvm_zstd",
+    name = "zstd",
     build_file = "@llvm-raw//utils/bazel/third_party_build:zstd.BUILD",
     sha256 = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
     strip_prefix = "zstd-1.5.2",
@@ -157,7 +157,7 @@ maybe(
 
 maybe(
     http_archive,
-    name = "robin_map",
+    name = "robin-map",
     build_file = "@llvm-raw//utils/bazel/third_party_build:robin_map.BUILD",
     sha256 = "a8424ad3b0affd4c57ed26f0f3d8a29604f0e1f2ef2089f497f614b1c94c7236",
     strip_prefix = "robin-map-1.3.0",

utils/bazel/WORKSPACE

@@ -31,10 +31,10 @@ DEFAULT_TARGETS = [
 ]
 
 def _overlay_directories(repository_ctx):
-    src_path = repository_ctx.path(Label("@llvm-raw//:WORKSPACE")).dirname
-    bazel_path = src_path.get_child("utils").get_child("bazel")
-    overlay_path = bazel_path.get_child("llvm-project-overlay")
-    script_path = bazel_path.get_child("overlay_directories.py")
+    workspace_path = repository_ctx.path(Label("@llvm-raw//utils/bazel:WORKSPACE")).dirname
+    src_path = workspace_path.get_child("../..")
+    overlay_path = workspace_path.get_child("llvm-project-overlay")
+    script_path = workspace_path.get_child("overlay_directories.py")
 
     python_bin = repository_ctx.which("python3")
     if not python_bin:
@@ -81,7 +81,7 @@ def _extract_cmake_settings(repository_ctx, llvm_cmake):
 
     # It would be easier to use external commands like sed(1) and python.
     # For portability, the parser should run on Starlark.
-    llvm_cmake_path = repository_ctx.path(Label("//:" + llvm_cmake))
+    llvm_cmake_path = repository_ctx.path(Label("@llvm-raw//:" + llvm_cmake))
     for line in repository_ctx.read(llvm_cmake_path).splitlines():
         # Extract "set ( FOO bar ... "
         setfoo = line.partition("(")

utils/bazel/configure.bzl

@@ -0,0 +1,58 @@
+module(
+    name = "bzlmod-example",
+    version = "0.0.0",
+    compatibility_level = 0,
+)
+ 
+bazel_dep(name = "llvm-raw", version = "main")
+
+LLVM_COMMIT="0000000000000000000000000000000000000000"
+
+# Add the dependencies from "utils/bazel/MODULE.bazel" but name the module
+# "llvm-raw". This way you don't need to fetch the llvm sources twice and retain
+# compatibility for third-party dependts as module overrides are ignored in
+# non-root modules.
+archive_override(
+    module_name = "llvm-raw",
+    strip_prefix = "llvm-project-{}".format(LLVM_COMMIT),
+    patch_cmds = [
+        '''printf 'module(name=\"llvm-raw\")
+bazel_dep(name = "apple_support", version = "1.17.1")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
+bazel_dep(name = "platforms", version = "0.0.11")
+bazel_dep(name = "robin-map", version = "1.3.0")
+bazel_dep(name = "rules_cc", version = "0.0.17")
+bazel_dep(name = "rules_foreign_cc", version = "0.13.0")
+bazel_dep(name = "rules_python", version = "1.1.0")
+bazel_dep(name = "zlib-ng", version = "2.0.7")
+bazel_dep(name = "zstd", version = "1.5.6")' > MODULE.bazel'''
+    ],
+    build_file_content = "# Empty.",
+    integrity = "sha256-0000000000000000000000000000000000000000000=",
+    urls = [
+        "https://github.com/llvm/llvm-project/archive/{}.zip".format(
+            LLVM_COMMIT,
+        ),
+    ],
+)
+
+llvm_project_overlay = use_extension(
+    "@llvm-raw//utils/bazel:extensions.bzl",
+    "llvm_project_overlay",
+)
+
+llvm_project_overlay.configure()
+
+use_repo(
+    llvm_project_overlay,
+    "llvm-project",
+
+    # Vendored.
+    "vulkan_headers",
+    "vulkan_sdk",
+    "gmp",
+    "mpfr",
+    "pfm",
+    "pybind11",
+    "nanobind",
+)

utils/bazel/examples/bzlmod-archive/MODULE.bazel

@@ -0,0 +1,74 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Since the llvm project doesn't have `MODULE.bazel` and `BUILD.bazel` files in
+# its root directory you need to create an override for the `llvm-raw` sources
+# and then inject it into the `llvm-project-overlay` module extension.
+
+module(
+    name = "bzlmod-example",
+    version = "0.0.0",
+    compatibility_level = 0,
+)
+
+new_local_repository = use_repo_rule(
+    "@bazel_tools//tools/build_defs/repo:local.bzl",
+    "new_local_repository",
+)
+
+# WARNING: This is not a module override. If a third party depends on your
+#          repository the repository will get lifted to the root module and
+#          treated as if it was in the third party module.
+#
+#          This means that if your repo is the top level module this behaves as
+#          you'd expect, but third parties need to create another
+#          repository in conjunction with a `repo_override`.
+#
+#          See: https://bazel.build/rules/lib/repo/local#new_local_repository
+#               https://bazel.build/rules/lib/repo/http#http_archive
+#               https://bazel.build/rules/lib/globals/module#override_repo
+new_local_repository(
+    name = "llvm-raw",
+    path = "third_party/llvm-project",
+    build_file_content = "# Empty.",
+)
+
+bazel_dep(name = "llvm-project-overlay", version = "main")
+
+# WARNING: Sice this is a module override it'll be ignored if a third party
+#          depends on your project.
+#
+#          This means that if your repo is the top level module this behaves as
+#          you'd expect, but third parties need to create a new module override
+#          as described in https://bazel.build/rules/lib/globals/module.
+local_path_override(
+    module_name = "llvm-project-overlay",
+    path = "third_party/llvm-project/utils/bazel",
+)
+
+llvm_project_overlay = use_extension(
+    "@llvm-project-overlay//:extensions.bzl",
+    "llvm_project_overlay",
+)
+
+inject_repo(
+    llvm_project_overlay,
+    "llvm-raw",
+)
+
+llvm_project_overlay.configure()
+
+use_repo(
+    llvm_project_overlay,
+    "llvm-project",
+
+    # Vendored.
+    "vulkan_headers",
+    "vulkan_sdk",
+    "gmp",
+    "mpfr",
+    "pfm",
+    "pybind11",
+    "nanobind",
+)

utils/bazel/examples/bzlmod-submodule-and-directories/MODULE.bazel

@@ -0,0 +1,125 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository")
+load(":vulkan_sdk.bzl", "vulkan_sdk_setup")
+load(":configure.bzl", "llvm_configure", "DEFAULT_TARGETS")
+
+def _llvm_configure_extension_impl(ctx):
+    targets = []
+
+    # Aggregate targets across imports.
+    targets.extend([
+        target
+        for module in ctx.modules 
+        for config in module.tags.configure 
+        for target in config.targets 
+        if target not in targets
+    ])
+
+    # Fall back to the default targets if all configurations of this extension
+    # omit the `target` attribute.
+    if targets == []:
+        targets = DEFAULT_TARGETS
+
+    llvm_configure(name = "llvm-project", targets = targets)
+
+    # Deliberately omit the "llvm-raw" directory if we're not in the utils/bazel
+    # directory.
+    #
+    # In downstream repos this intentionally causes the extension to immediately
+    # error out if "llvm-raw" wasn't injected explicitly.
+    #
+    # We can't add this repo to the utils/bazel/MODULE.bazel as it would cause
+    # submodule imports to resolve the new_local_repository at wrong paths.
+    [
+        new_local_repository(
+            name = "llvm-raw",
+            path = "../..",
+            build_file_content = "# Empty."
+        )
+        for module in ctx.modules
+        if module.name == "llvm-project-overlay" and module.is_root
+    ]
+
+    print(ctx.path(Label("@llvm-raw//:BUILD.bazel")))
+
+    http_archive(
+        name = "vulkan_headers",
+        build_file = "@llvm-raw//utils/bazel/third_party_build:vulkan_headers.BUILD",
+        sha256 = "19f491784ef0bc73caff877d11c96a48b946b5a1c805079d9006e3fbaa5c1895",
+        strip_prefix = "Vulkan-Headers-9bd3f561bcee3f01d22912de10bb07ce4e23d378",
+        urls = [
+            "https://github.com/KhronosGroup/Vulkan-Headers/archive/9bd3f561bcee3f01d22912de10bb07ce4e23d378.tar.gz",
+        ],
+    )
+
+    vulkan_sdk_setup(name = "vulkan_sdk")
+
+    http_archive(
+        name = "gmp",
+        build_file = "@llvm-raw//utils/bazel/third_party_build:gmp.BUILD",
+        sha256 = "fd4829912cddd12f84181c3451cc752be224643e87fac497b69edddadc49b4f2",
+        strip_prefix = "gmp-6.2.1",
+        urls = [
+            "https://gmplib.org/download/gmp/gmp-6.2.1.tar.xz",
+            "https://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz",
+        ],
+    )
+
+    # https://www.mpfr.org/mpfr-current/
+    #
+    # When updating to a newer version, don't use URLs with "mpfr-current" in them.
+    # Instead, find a stable URL like the one used currently.
+    http_archive(
+        name = "mpfr",
+        build_file = "@llvm-raw//utils/bazel/third_party_build:mpfr.BUILD",
+        sha256 = "9cbed5d0af0d9ed5e9f8dd013e17838eb15e1db9a6ae0d371d55d35f93a782a7",
+        strip_prefix = "mpfr-4.1.1",
+        urls = ["https://www.mpfr.org/mpfr-4.1.1/mpfr-4.1.1.tar.gz"],
+    )
+
+    http_archive(
+        name = "pfm",
+        build_file = "@llvm-raw//utils/bazel/third_party_build:pfm.BUILD",
+        sha256 = "d18b97764c755528c1051d376e33545d0eb60c6ebf85680436813fa5b04cc3d1",
+        strip_prefix = "libpfm-4.13.0",
+        urls = ["https://versaweb.dl.sourceforge.net/project/perfmon2/libpfm4/libpfm-4.13.0.tar.gz"],
+    )
+
+    http_archive(
+        name = "pybind11",
+        build_file = "@llvm-raw//utils/bazel/third_party_build:pybind.BUILD",
+        sha256 = "201966a61dc826f1b1879a24a3317a1ec9214a918c8eb035be2f30c3e9cfbdcb",
+        strip_prefix = "pybind11-2.10.3",
+        url = "https://github.com/pybind/pybind11/archive/v2.10.3.zip",
+    )
+
+    # TODO: Make `NB_BUILD=1` and `NB_SHARED=1` configurable in the BCR variant
+    #       and import this via `MODULE.bazel`.
+    http_archive(
+        name = "nanobind",
+        build_file = "@llvm-raw//utils/bazel/third_party_build:nanobind.BUILD",
+        sha256 = "bb35deaed7efac5029ed1e33880a415638352f757d49207a8e6013fefb6c49a7",
+        strip_prefix = "nanobind-2.4.0",
+        url = "https://github.com/wjakob/nanobind/archive/refs/tags/v2.4.0.tar.gz",
+    )
+
+
+llvm_project_overlay = module_extension(
+    doc = """Configure the llvm-project.
+
+    Tags:
+        targets: List of targets which Clang should support.
+    """,
+    implementation = _llvm_configure_extension_impl,
+    tag_classes = {
+        "configure": tag_class(
+            attrs = {
+                "targets": attr.string_list(),
+            },
+        ),
+    },
+)