Bazel integration for appliance tests

Author: Craig Furman

dev/tool_deps.bzl

@@ -4,6 +4,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file"
 
 DOCSITE_VERSION = "1.9.4"
 SRC_CLI_VERSION = "5.3.0"
+KUBEBUILDER_ASSETS_VERSION = "1.28.0"
 CTAGS_VERSION = "6.0.0.2783f009"
 PACKER_VERSION = "1.8.3"
 P4_FUSION_VERSION = "v1.13.2-sg.04a293a"
@@ -26,6 +27,14 @@ filegroup(
 )
 """
 
+KUBEBUILDER_ASSETS_BUILDFILE = """
+filegroup(
+    name = "kubebuilder-assets",
+    srcs = glob(["*"]),
+    visibility = ["//visibility:public"],
+)
+"""
+
 GCLOUD_VERSION = "456.0.0"
 GCLOUD_BUILDFILE = """
 package(default_visibility = ["//visibility:public"])
@@ -102,6 +111,31 @@ def tool_deps():
         url = "https://github.com/sourcegraph/src-cli/releases/download/{0}/src-cli_{0}_darwin_arm64.tar.gz".format(SRC_CLI_VERSION),
     )
 
+    # Needed for internal/appliance tests
+    http_archive(
+        name = "kubebuilder-assets-darwin-arm64",
+        build_file_content = KUBEBUILDER_ASSETS_BUILDFILE,
+        sha256 = "c87c6b3c0aec4233e68a12dc9690bcbe2f8d6cd72c23e670602b17b2d7118325",
+        urls = ["https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-{}-darwin-arm64.tar.gz".format(KUBEBUILDER_ASSETS_VERSION)],
+        strip_prefix = "kubebuilder/bin",
+    )
+
+    http_archive(
+        name = "kubebuilder-assets-darwin-amd64",
+        build_file_content = KUBEBUILDER_ASSETS_BUILDFILE,
+        sha256 = "a02e33a3981712c8d2702520f95357bd6c7d03d24b83a4f8ac1c89a9ba4d78c1",
+        urls = ["https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-{}-darwin-amd64.tar.gz".format(KUBEBUILDER_ASSETS_VERSION)],
+        strip_prefix = "kubebuilder/bin",
+    )
+
+    http_archive(
+        name = "kubebuilder-assets-linux-amd64",
+        build_file_content = KUBEBUILDER_ASSETS_BUILDFILE,
+        sha256 = "8c816871604cbe119ca9dd8072b576552ae369b96eebc3cdaaf50edd7e3c0c7b",
+        urls = ["https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-{}-linux-amd64.tar.gz".format(KUBEBUILDER_ASSETS_VERSION)],
+        strip_prefix = "kubebuilder/bin",
+    )
+
     # universal-ctags
     #
     # Two step process to update these. First land a commit in main updating

dev/tools/BUILD.bazel

@@ -107,3 +107,13 @@ sh_binary(
     }),
     visibility = ["//visibility:public"],
 )
+
+alias(
+    name = "kubebuilder-assets",
+    actual = select({
+        "@bazel_tools//src/conditions:darwin_x86_64": "@kubebuilder-assets-darwin-amd64//:kubebuilder-assets",
+        "@bazel_tools//src/conditions:darwin_arm64": "@kubebuilder-assets-darwin-arm64//:kubebuilder-assets",
+        "@bazel_tools//src/conditions:linux_x86_64": "@kubebuilder-assets-linux-amd64//:kubebuilder-assets",
+    }),
+    visibility = ["//visibility:public"],
+)

go.mod

@@ -252,6 +252,7 @@ require (
 	github.com/aws/constructs-go/constructs/v10 v10.2.69
 	github.com/aws/jsii-runtime-go v1.84.0
 	github.com/bazelbuild/bazel-gazelle v0.35.0
+	github.com/bazelbuild/rules_go v0.47.0
 	github.com/derision-test/go-mockgen/v2 v2.0.1
 	github.com/dghubble/gologin/v2 v2.4.0
 	github.com/edsrzf/mmap-go v1.1.0

go.sum

@@ -298,8 +298,8 @@ github.com/bazelbuild/bazel-gazelle v0.35.0 h1:Bvg+zEHWYwWrhJT4WxyvcU3y1DEJpT/Xl
 github.com/bazelbuild/bazel-gazelle v0.35.0/go.mod h1:o2+s90f3w3U6jjw0gcdok0EJOfNK0AK/9RyVP7QkRDk=
 github.com/bazelbuild/buildtools v0.0.0-20231115204819-d4c9dccdfbb1 h1:2Gc2Q6hVR1SJ8bBI9Ybzoggp8u/ED2WkM4MfvEIn9+c=
 github.com/bazelbuild/buildtools v0.0.0-20231115204819-d4c9dccdfbb1/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
-github.com/bazelbuild/rules_go v0.44.0 h1:uJStI9o5obVWSwquy9WxKNWfZxf2sKA2rpEsX6x5RVM=
-github.com/bazelbuild/rules_go v0.44.0/go.mod h1:z7Y8GZ90V4mwgYizbNbEQKmOmx93Q3Btcel4vX7pXoc=
+github.com/bazelbuild/rules_go v0.47.0 h1:TTl2buKt0lqJe5s6up5FtaB1F95Wg997Lv15MWetU88=
+github.com/bazelbuild/rules_go v0.47.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs=
 github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA=
 github.com/becheran/wildmatch-go v1.0.0/go.mod h1:gbMvj0NtVdJ15Mg/mH9uxk2R1QCistMyU7d9KFzroX4=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
@@ -847,8 +847,8 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
-github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
+github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

internal/appliance/BUILD.bazel

@@ -37,20 +37,31 @@ go_library(
     ],
 )
 
+filegroup(
+    name = "testdata",
+    srcs = glob(["testdata/**"]),
+)
+
 go_test(
     name = "appliance_test",
     srcs = [
         "blobstore_test.go",
         "golden_test.go",
-        "kubernetes_test.go",
-        "reconcile_test.go",
+        "helpers_test.go",
+    ],
+    data = [
+        ":testdata",
+        "//dev/tools:kubebuilder-assets",
     ],
-    data = glob(["testdata/**"]),
     embed = [":appliance"],
+    env = {
+        "KUBEBUILDER_ASSET_PATHS": "$(rlocationpaths //dev/tools:kubebuilder-assets)",
+    },
     deps = [
         "@com_github_go_logr_stdr//:stdr",
         "@com_github_stretchr_testify//require",
         "@com_github_stretchr_testify//suite",
+        "@io_bazel_rules_go//go/runfiles:go_default_library",
         "@io_k8s_api//core/v1:core",
         "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
         "@io_k8s_apimachinery//pkg/runtime/schema",

internal/appliance/development.md

@@ -0,0 +1,19 @@
+# Appliance development
+
+## Running tests
+
+To (re)generate golden fixtures, pass the following argument to `go test`:
+
+```
+go test -args appliance-update-golden-files
+```
+
+In order to run `go test` (with or without arguments), you must have
+`setup-envtest` available:
+
+```
+go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+```
+
+This is not a requirement for the Bazel environment (`bazel test
+:appliance_test` in this module).

internal/appliance/kubernetes_test.go renamed to internal/appliance/helpers_test.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/bazelbuild/rules_go/go/runfiles"
 	"github.com/go-logr/stdr"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -58,19 +59,8 @@ func (suite *ApplianceTestSuite) setupEnvtest() {
 	t := suite.T()
 	logger := stdr.New(stdlog.New(os.Stderr, "", stdlog.LstdFlags))
 
-	// TODO figure out a way to make this work with bazel. Either download the
-	// etcd/k8s executables that setup-envtest downloads and pass that directory
-	// into BinaryAssetsDirectory below, or download setup-envtest without
-	// assuming it's already on the PATH.
-	// Ideally these assets could be cached in buildkite too.
-	setupEnvTestCmd := exec.Command("setup-envtest", "use", "1.28.0", "--bin-dir", "/tmp/envtest", "-p", "path")
-	var envtestOut bytes.Buffer
-	setupEnvTestCmd.Stdout = &envtestOut
-	err := setupEnvTestCmd.Run()
-	require.NoError(t, err)
-
 	suite.testEnv = &envtest.Environment{
-		BinaryAssetsDirectory: strings.TrimSpace(envtestOut.String()),
+		BinaryAssetsDirectory: suite.kubebuilderAssetPath(),
 	}
 	cfg, err := suite.testEnv.Start()
 	require.NoError(t, err)
@@ -107,6 +97,31 @@ func (suite *ApplianceTestSuite) setupEnvtest() {
 	}()
 }
 
+func (suite *ApplianceTestSuite) kubebuilderAssetPath() string {
+	if os.Getenv("BAZEL_TEST") == "" {
+		return suite.kubebuilderAssetPathLocalDev()
+	}
+
+	assetPaths := strings.Split(os.Getenv("KUBEBUILDER_ASSET_PATHS"), " ")
+	suite.Require().Greater(len(assetPaths), 0)
+	arbAssetPath, err := runfiles.Rlocation(assetPaths[0])
+	suite.Require().NoError(err)
+	return filepath.Dir(arbAssetPath)
+}
+
+// In the hermetic bazel environment, we skip setup-envtest, handling the assets
+// directly in a hermetic and cachable way.
+// If we're using `go test`, which can be convenient for local dev, we fall back
+// on expecting setup-envtest to be present on the developer machine.
+func (suite *ApplianceTestSuite) kubebuilderAssetPathLocalDev() string {
+	setupEnvTestCmd := exec.Command("setup-envtest", "use", "1.28.0", "--bin-dir", "/tmp/envtest", "-p", "path")
+	var envtestOut bytes.Buffer
+	setupEnvTestCmd.Stdout = &envtestOut
+	err := setupEnvTestCmd.Run()
+	suite.Require().NoError(err, "Did you remember to `go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest`?")
+	return strings.TrimSpace(envtestOut.String())
+}
+
 func (suite *ApplianceTestSuite) createConfigMap(fixtureFileName string) string {
 	// Create a random namespace for each test
 	namespace := "test-appliance-" + suite.randomSlug()