feat: add bzlmod support for gazelle plugin (#1077)

* feat: add optional pkg alias generation to pip_repository

* feat: support using aliases in gazelle

* doc: migrate gazelle example to use user friendly aliases

* feat: gazelle supports bzlmod

* chore: update gazelle plugin deps

* chore: bazel run //:gazelle

* fix: remove deps that are imported as bazel deps

See: bazel-contrib/bazel-gazelle#1403

* ci: add build_file_generation to show that we support bzlmod

Author: aignas

.bazelci/presubmit.yml

@@ -73,6 +73,14 @@ tasks:
     build_targets: ["//..."]
     test_targets: ["//..."]
     working_directory: gazelle
+  gazelle_extension_bzlmod:
+    <<: *common_bzlmod_flags
+    name: Test the Gazelle extension under bzlmod
+    platform: ubuntu2004
+    build_targets: ["//..."]
+    test_targets: ["//..."]
+    working_directory: gazelle
+
   ubuntu_min:
     <<: *minimum_supported_version
     <<: *reusable_config
@@ -138,6 +146,32 @@ tasks:
     working_directory: examples/build_file_generation
     platform: windows
 
+  integration_test_build_file_generation_bzlmod_ubuntu:
+    <<: *minimum_supported_bzlmod_version
+    <<: *common_bzlmod_flags
+    <<: *reusable_build_test_all
+    name: build_file_generation_bzlmod integration tests on Ubuntu
+    working_directory: examples/build_file_generation
+    platform: ubuntu2004
+  integration_test_build_file_generation_bzlmod_debian:
+    <<: *common_bzlmod_flags
+    <<: *reusable_build_test_all
+    name: build_file_generation_bzlmod integration tests on Debian
+    working_directory: examples/build_file_generation
+    platform: debian11
+  integration_test_build_file_generation_bzlmod_macos:
+    <<: *common_bzlmod_flags
+    <<: *reusable_build_test_all
+    name: build_file_generation_bzlmod integration tests on macOS
+    working_directory: examples/build_file_generation
+    platform: macos
+  integration_test_build_file_generation_bzlmod_windows:
+    <<: *common_bzlmod_flags
+    <<: *reusable_build_test_all
+    name: build_file_generation_bzlmod integration tests on Windows
+    working_directory: examples/build_file_generation
+    platform: windows
+
   integration_test_bzlmod_ubuntu_min:
     <<: *minimum_supported_bzlmod_version
     <<: *reusable_build_test_all

docs/pip_repository.md

@@ -43,6 +43,9 @@ gazelle_python_manifest(
     modules_mapping = ":modules_map",
     pip_repository_name = "pip",
     requirements = "//:requirements_lock.txt",
+    # NOTE: we can use this flag in order to make our setup compatible with
+    # bzlmod.
+    use_pip_repository_aliases = True,
 )
 
 # Our gazelle target points to the python gazelle binary.
@@ -65,7 +68,7 @@ py_library(
     visibility = ["//:__subpackages__"],
     deps = [
         "//random_number_generator",
-        "@pip_flask//:pkg",
+        "@pip//flask",
     ],
 )
 

examples/build_file_generation/BUILD.bazel

@@ -0,0 +1,43 @@
+module(
+    name = "example_bzlmod",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "rules_python", version = "0.19.0")
+bazel_dep(name = "rules_python_gazelle_plugin", version = "0.19.0")
+bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle")
+
+# local overrides for the packages for CI purposes.
+# for usual setups you should remove this block.
+local_path_override(
+    module_name = "rules_python",
+    path = "../..",
+)
+
+local_path_override(
+    module_name = "rules_python_gazelle_plugin",
+    path = "../../gazelle",
+)
+
+# Register python toolchain
+python = use_extension("@rules_python//python:extensions.bzl", "python")
+python.toolchain(
+    name = "python3_9",
+    python_version = "3.9",
+)
+use_repo(python, "python3_9_toolchains")
+
+register_toolchains(
+    "@python3_9_toolchains//:all",
+)
+
+pip = use_extension("@rules_python//python:extensions.bzl", "pip")
+pip.parse(
+    name = "pip",
+    # Generate user friendly alias labels for each dependency that we have.
+    incompatible_generate_aliases = True,
+    requirements_lock = "//:requirements_lock.txt",
+    requirements_windows = "//:requirements_windows.txt",
+)
+use_repo(pip, "pip")

examples/build_file_generation/MODULE.bazel

@@ -90,6 +90,8 @@ load("@rules_python//python:pip.bzl", "pip_parse")
 # You can instead check this `requirements.bzl` file into your repo.
 pip_parse(
     name = "pip",
+    # Generate user friendly alias labels for each dependency that we have.
+    incompatible_generate_aliases = True,
     # (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that
     # acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.:
     # 1. Python interpreter that you compile in the build file.

examples/build_file_generation/WORKSPACE

@@ -114,4 +114,5 @@ manifest:
     zipp.py310compat: zipp
   pip_repository:
     name: pip
-integrity: 2c84a3cabeaff134a1d045e5a173a3178086f236ab20f895ffbd7f3b7a6e5bb0
+    use_pip_repository_aliases: true
+integrity: 85f073e37e31339508aaaf5e0d5472adae5148fd5f054e9cc586343c026660e1

examples/build_file_generation/gazelle_python.yaml

@@ -0,0 +1,20 @@
+module(
+    name = "rules_python_gazelle_plugin",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "rules_python", version = "0.18.0")
+bazel_dep(name = "rules_go", version = "0.38.1", repo_name = "io_bazel_rules_go")
+bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle")
+
+go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps")
+go_deps.from_file(go_mod = "//:go.mod")
+use_repo(
+    go_deps,
+    "com_github_bazelbuild_buildtools",
+    "com_github_bmatcuk_doublestar",
+    "com_github_emirpasic_gods",
+    "com_github_ghodss_yaml",
+    "in_gopkg_yaml_v2",
+)

gazelle/MODULE.bazel

@@ -28,37 +28,22 @@ def gazelle_deps():
         sum = "h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=",
         version = "v0.0.0-20190523083050-ea95bdfd59fc",
     )
-    go_repository(
-        name = "com_github_bazelbuild_bazel_gazelle",
-        importpath = "github.com/bazelbuild/bazel-gazelle",
-        sum = "h1:+/ZhUxlDy4XnyMIGeKkbRZoIGssy1eO51GijwIvvuwE=",
-        version = "v0.27.0",
-    )
+
     go_repository(
         name = "com_github_bazelbuild_buildtools",
         build_naming_convention = "go_default_library",
         importpath = "github.com/bazelbuild/buildtools",
         sum = "h1:jhiMzJ+8unnLRtV8rpbWBFE9pFNzIqgUTyZU5aA++w8=",
         version = "v0.0.0-20221004120235-7186f635531b",
     )
-    go_repository(
-        name = "com_github_bazelbuild_rules_go",
-        importpath = "github.com/bazelbuild/rules_go",
-        sum = "h1:ViPR65vOrg74JKntAUFY6qZkheBKGB6to7wFd8gCRU4=",
-        version = "v0.35.0",
-    )
+
     go_repository(
         name = "com_github_bmatcuk_doublestar",
         importpath = "github.com/bmatcuk/doublestar",
         sum = "h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=",
         version = "v1.3.4",
     )
-    go_repository(
-        name = "com_github_bmatcuk_doublestar_v4",
-        importpath = "github.com/bmatcuk/doublestar/v4",
-        sum = "h1:Qu+u9wR3Vd89LnlLMHvnZ5coJMWKQamqdz9/p5GNthA=",
-        version = "v4.2.0",
-    )
+
     go_repository(
         name = "com_github_burntsushi_toml",
         importpath = "github.com/BurntSushi/toml",
@@ -113,12 +98,7 @@ def gazelle_deps():
         sum = "h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=",
         version = "v0.1.0",
     )
-    go_repository(
-        name = "com_github_fsnotify_fsnotify",
-        importpath = "github.com/fsnotify/fsnotify",
-        sum = "h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=",
-        version = "v1.5.4",
-    )
+
     go_repository(
         name = "com_github_ghodss_yaml",
         importpath = "github.com/ghodss/yaml",
@@ -134,33 +114,22 @@ def gazelle_deps():
     go_repository(
         name = "com_github_golang_mock",
         importpath = "github.com/golang/mock",
-        sum = "h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=",
-        version = "v1.6.0",
+        sum = "h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=",
+        version = "v1.1.1",
     )
     go_repository(
         name = "com_github_golang_protobuf",
         importpath = "github.com/golang/protobuf",
-        sum = "h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=",
-        version = "v1.5.2",
+        sum = "h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=",
+        version = "v1.4.3",
     )
     go_repository(
         name = "com_github_google_go_cmp",
         importpath = "github.com/google/go-cmp",
         sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=",
         version = "v0.5.9",
     )
-    go_repository(
-        name = "com_github_pelletier_go_toml",
-        importpath = "github.com/pelletier/go-toml",
-        sum = "h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=",
-        version = "v1.9.5",
-    )
-    go_repository(
-        name = "com_github_pmezard_go_difflib",
-        importpath = "github.com/pmezard/go-difflib",
-        sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=",
-        version = "v1.0.0",
-    )
+
     go_repository(
         name = "com_github_prometheus_client_model",
         importpath = "github.com/prometheus/client_model",
@@ -218,8 +187,8 @@ def gazelle_deps():
     go_repository(
         name = "org_golang_google_protobuf",
         importpath = "google.golang.org/protobuf",
-        sum = "h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=",
-        version = "v1.28.0",
+        sum = "h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=",
+        version = "v1.25.0",
     )
     go_repository(
         name = "org_golang_x_crypto",
@@ -260,8 +229,8 @@ def gazelle_deps():
     go_repository(
         name = "org_golang_x_sync",
         importpath = "golang.org/x/sync",
-        sum = "h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A=",
-        version = "v0.0.0-20220907140024-f12130a52804",
+        sum = "h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=",
+        version = "v0.0.0-20220722155255-886fb9371eb4",
     )
     go_repository(
         name = "org_golang_x_sys",

gazelle/deps.bzl

@@ -3,9 +3,7 @@ module github.com/bazelbuild/rules_python/gazelle
 go 1.19
 
 require (
-	github.com/bazelbuild/bazel-gazelle v0.27.0
 	github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b
-	github.com/bazelbuild/rules_go v0.35.0
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/emirpasic/gods v1.18.1
 	github.com/ghodss/yaml v1.0.0

gazelle/go.mod

@@ -24,13 +24,16 @@ def gazelle_python_manifest(
         modules_mapping,
         pip_repository_name = "",
         pip_deps_repository_name = "",
-        manifest = ":gazelle_python.yaml"):
+        manifest = ":gazelle_python.yaml",
+        use_pip_repository_aliases = False):
     """A macro for defining the updating and testing targets for the Gazelle manifest file.
 
     Args:
         name: the name used as a base for the targets.
         requirements: the target for the requirements.txt file.
         pip_repository_name: the name of the pip_install or pip_repository target.
+        use_pip_repository_aliases: boolean flag to enable using user-friendly
+            python package aliases.
         pip_deps_repository_name: deprecated - the old pip_install target name.
         modules_mapping: the target for the generated modules_mapping.json file.
         manifest: the target for the Gazelle manifest file.
@@ -67,6 +70,12 @@ def gazelle_python_manifest(
         update_target_label,
     ]
 
+    if use_pip_repository_aliases:
+        update_args += [
+            "--use-pip-repository-aliases",
+            "true",
+        ]
+
     go_binary(
         name = update_target,
         embed = [Label("//manifest/generate:generate_lib")],

gazelle/manifest/defs.bzl

@@ -38,12 +38,15 @@ func init() {
 }
 
 func main() {
-	var manifestGeneratorHashPath string
-	var requirementsPath string
-	var pipRepositoryName string
-	var modulesMappingPath string
-	var outputPath string
-	var updateTarget string
+	var (
+		manifestGeneratorHashPath string
+		requirementsPath          string
+		pipRepositoryName         string
+		usePipRepositoryAliases   bool
+		modulesMappingPath        string
+		outputPath                string
+		updateTarget              string
+	)
 	flag.StringVar(
 		&manifestGeneratorHashPath,
 		"manifest-generator-hash",
@@ -60,6 +63,11 @@ func main() {
 		"pip-repository-name",
 		"",
 		"The name of the pip_install or pip_repository target.")
+	flag.BoolVar(
+		&usePipRepositoryAliases,
+		"use-pip-repository-aliases",
+		false,
+		"Whether to use the pip-repository aliases, which are generated when passing 'incompatible_generate_aliases = True'.")
 	flag.StringVar(
 		&modulesMappingPath,
 		"modules-mapping",
@@ -103,7 +111,8 @@ func main() {
 	manifestFile := manifest.NewFile(&manifest.Manifest{
 		ModulesMapping: modulesMapping,
 		PipRepository: &manifest.PipRepository{
-			Name:        pipRepositoryName,
+			Name:                    pipRepositoryName,
+			UsePipRepositoryAliases: usePipRepositoryAliases,
 		},
 	})
 	if err := writeOutput(

gazelle/manifest/generate/generate.go

@@ -144,4 +144,7 @@ type Manifest struct {
 type PipRepository struct {
 	// The name of the pip_install or pip_repository target.
 	Name string
+	// UsePipRepositoryAliases allows to use aliases generated pip_repository
+	// when passing incompatible_generate_aliases = True.
+	UsePipRepositoryAliases bool `yaml:"use_pip_repository_aliases,omitempty"`
 }