Skip to content
This repository was archived by the owner on Sep 30, 2024. It is now read-only.

Commit ed557bd

Browse files
author
Craig Furman
committed
appliance: golden/snapshot testing
Use envtest (https://book.kubebuilder.io/reference/envtest.html) to start a real Kubernetes API server, backed by a real etcd instance, in tests. The only controller running is our appliance controller, so these tests assert on the changes our controller actually makes to Kubernetes API objects in response to some input, but don't depend on complex infrastruture like container runtimes or persistent volume controllers (since no such controllers are actually running, unlike in a useful Kubernetes cluster). This commit adds a test harness to the appliance package: * It starts the k8s API server only once, not once per test, as this is a relatively slow operation. * Helper functions to run each test in a unique namespace to avoid inter-test dependency. This opens the door to parallelism, but note that testify/suite doesn't currently support parallel tests. * Helper functions to load input ConfigMaps from fixture files. * Passing `-args appliance-update-golden-files` to `go test` will cause checked-in golden files to be updated.
1 parent 1f09814 commit ed557bd

File tree

13 files changed

+788
-16
lines changed

13 files changed

+788
-16
lines changed

internal/appliance/blobstore.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,19 @@ package appliance
33
import (
44
"context"
55

6+
appsv1 "k8s.io/api/apps/v1"
7+
corev1 "k8s.io/api/core/v1"
8+
"k8s.io/apimachinery/pkg/api/resource"
9+
"k8s.io/apimachinery/pkg/util/intstr"
10+
"sigs.k8s.io/controller-runtime/pkg/client"
11+
612
"github.com/sourcegraph/sourcegraph/internal/k8s/resource/container"
713
"github.com/sourcegraph/sourcegraph/internal/k8s/resource/deployment"
814
"github.com/sourcegraph/sourcegraph/internal/k8s/resource/pod"
915
"github.com/sourcegraph/sourcegraph/internal/k8s/resource/pvc"
1016
"github.com/sourcegraph/sourcegraph/internal/k8s/resource/service"
1117
"github.com/sourcegraph/sourcegraph/lib/errors"
1218
"github.com/sourcegraph/sourcegraph/lib/pointers"
13-
appsv1 "k8s.io/api/apps/v1"
14-
corev1 "k8s.io/api/core/v1"
15-
"k8s.io/apimachinery/pkg/api/resource"
16-
"k8s.io/apimachinery/pkg/util/intstr"
17-
"sigs.k8s.io/controller-runtime/pkg/client"
1819
)
1920

2021
func (r *Reconciler) reconcileBlobstore(ctx context.Context, sg *Sourcegraph, owner client.Object) error {
@@ -108,7 +109,7 @@ func (r *Reconciler) reconcileBlobstoreServices(ctx context.Context, sg *Sourceg
108109
func buildBlobstoreDeployment(sg *Sourcegraph) (appsv1.Deployment, error) {
109110
name := "blobstore"
110111

111-
containerImage := ""
112+
containerImage := "TODO"
112113

113114
containerPorts := corev1.ContainerPort{
114115
Name: name,
@@ -186,7 +187,7 @@ func buildBlobstoreDeployment(sg *Sourcegraph) (appsv1.Deployment, error) {
186187
},
187188
}
188189

189-
podTemplate, err := pod.NewPodTemplate(name, sg.Namespace,
190+
podTemplate, err := pod.NewPodTemplate(name,
190191
pod.WithContainers(defaultContainer),
191192
pod.WithVolumes(podVolumes),
192193
)

internal/appliance/blobstore_test.go

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,46 @@
11
package appliance
2+
3+
import (
4+
"time"
5+
)
6+
7+
// Simple test cases in which we want to assert that a given configmap causes a
8+
// certain set of resources to be deployed can go here. sg and golden fixtures
9+
// are in testdata/ and named after the test case name.
10+
func (suite *ApplianceTestSuite) TestDeployBlobstore() {
11+
for _, tc := range []struct {
12+
name string
13+
}{
14+
{
15+
name: "blobstore-default",
16+
},
17+
} {
18+
suite.Run(tc.name, func() {
19+
namespace := suite.createConfigMap(tc.name)
20+
21+
// Wait for reconciliation to be finished.
22+
suite.Require().Eventually(func() bool {
23+
return suite.getConfigMapReconcileEventCount(namespace) > 0
24+
}, time.Second*10, time.Millisecond*200)
25+
26+
suite.makeGoldenAssertions(namespace, tc.name)
27+
})
28+
}
29+
}
30+
31+
// More complex test cases involving updates to the configmap can have their own
32+
// test blocks
33+
func (suite *ApplianceTestSuite) TestBlobstoreResourcesDeletedWhenDisabled() {
34+
namespace := suite.createConfigMap("blobstore-default")
35+
suite.Require().Eventually(func() bool {
36+
return suite.getConfigMapReconcileEventCount(namespace) > 0
37+
}, time.Second*10, time.Millisecond*200)
38+
39+
eventsSeenSoFar := suite.getConfigMapReconcileEventCount(namespace)
40+
suite.updateConfigMap(namespace, "everything-disabled")
41+
suite.Require().Eventually(func() bool {
42+
return suite.getConfigMapReconcileEventCount(namespace) > eventsSeenSoFar
43+
}, time.Second*10, time.Millisecond*200)
44+
45+
suite.makeGoldenAssertions(namespace, "blobstore-subsequent-disable")
46+
}

internal/appliance/golden_test.go

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package appliance
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"time"
7+
8+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"k8s.io/apimachinery/pkg/runtime/schema"
10+
"sigs.k8s.io/controller-runtime/pkg/client"
11+
"sigs.k8s.io/yaml"
12+
)
13+
14+
// Test helpers
15+
16+
// creationTimestamp and uid need to be normalized
17+
var magicTime = metav1.NewTime(time.Date(2024, time.April, 19, 0, 0, 0, 0, time.UTC))
18+
19+
type goldenFile struct {
20+
Resources []client.Object `json:"resources"`
21+
}
22+
23+
func (suite *ApplianceTestSuite) makeGoldenAssertions(namespace, goldenFileName string) {
24+
require := suite.Require()
25+
26+
goldenFilePath := filepath.Join("testdata", "golden-fixtures", goldenFileName+".yaml")
27+
obtainedResources := goldenFile{Resources: suite.gatherResources(namespace)}
28+
obtainedBytes, err := yaml.Marshal(obtainedResources)
29+
require.NoError(err)
30+
if len(os.Args) > 0 && os.Args[len(os.Args)-1] == "appliance-update-golden-files" {
31+
err := os.WriteFile(goldenFilePath, obtainedBytes, 0600)
32+
require.NoError(err)
33+
}
34+
35+
goldenBytes, err := os.ReadFile(goldenFilePath)
36+
require.NoError(err)
37+
38+
// testify prints a readable yaml diff
39+
require.Equal(string(goldenBytes), string(obtainedBytes))
40+
}
41+
42+
// When new owned types are declared in SetupWithManager() in reconcile.go, we
43+
// must gather them here for golden testing to be reliable.
44+
func (suite *ApplianceTestSuite) gatherResources(namespace string) []client.Object {
45+
var objs []client.Object
46+
47+
// We set the GVK ourselves, as this is missing from the List response:
48+
// https://github.com/kubernetes/client-go/issues/861
49+
// This makes eyeballing golden file diffs a little easier, as we can see
50+
// which object is being changed.
51+
//
52+
// Certain common fields must be normalized in order to make golden testing
53+
// work, such as the creationTimestamp and UID, which would differ every
54+
// test run. Some resource-specific normalizations are also performed.
55+
deps, err := suite.k8sClient.AppsV1().Deployments(namespace).List(suite.ctx, metav1.ListOptions{})
56+
suite.Require().NoError(err)
57+
for _, obj := range deps.Items {
58+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"})
59+
normalizeObj(&obj)
60+
objs = append(objs, &obj)
61+
}
62+
ssets, err := suite.k8sClient.AppsV1().StatefulSets(namespace).List(suite.ctx, metav1.ListOptions{})
63+
suite.Require().NoError(err)
64+
for _, obj := range ssets.Items {
65+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"})
66+
normalizeObj(&obj)
67+
objs = append(objs, &obj)
68+
}
69+
cmaps, err := suite.k8sClient.CoreV1().ConfigMaps(namespace).List(suite.ctx, metav1.ListOptions{})
70+
suite.Require().NoError(err)
71+
for _, obj := range cmaps.Items {
72+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"})
73+
normalizeObj(&obj)
74+
objs = append(objs, &obj)
75+
}
76+
pvcs, err := suite.k8sClient.CoreV1().PersistentVolumeClaims(namespace).List(suite.ctx, metav1.ListOptions{})
77+
suite.Require().NoError(err)
78+
for _, obj := range pvcs.Items {
79+
if obj.DeletionTimestamp != nil {
80+
obj.DeletionTimestamp = &magicTime
81+
}
82+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PersistentVolumeClaim"})
83+
normalizeObj(&obj)
84+
objs = append(objs, &obj)
85+
}
86+
pods, err := suite.k8sClient.CoreV1().Pods(namespace).List(suite.ctx, metav1.ListOptions{})
87+
suite.Require().NoError(err)
88+
for _, obj := range pods.Items {
89+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"})
90+
normalizeObj(&obj)
91+
objs = append(objs, &obj)
92+
}
93+
94+
// These are just test secrets, nothing truly sensitive should end up in the
95+
// golden files.
96+
secrets, err := suite.k8sClient.CoreV1().Secrets(namespace).List(suite.ctx, metav1.ListOptions{})
97+
suite.Require().NoError(err)
98+
for _, obj := range secrets.Items {
99+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"})
100+
normalizeObj(&obj)
101+
objs = append(objs, &obj)
102+
}
103+
104+
sas, err := suite.k8sClient.CoreV1().ServiceAccounts(namespace).List(suite.ctx, metav1.ListOptions{})
105+
suite.Require().NoError(err)
106+
for _, obj := range sas.Items {
107+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ServiceAccount"})
108+
normalizeObj(&obj)
109+
objs = append(objs, &obj)
110+
}
111+
svcs, err := suite.k8sClient.CoreV1().Services(namespace).List(suite.ctx, metav1.ListOptions{})
112+
suite.Require().NoError(err)
113+
for _, obj := range svcs.Items {
114+
obj.Spec.ClusterIP = "NORMALIZED_FOR_TESTING"
115+
obj.Spec.ClusterIPs = []string{"NORMALIZED_FOR_TESTING"}
116+
obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"})
117+
normalizeObj(&obj)
118+
objs = append(objs, &obj)
119+
}
120+
121+
return objs
122+
}
123+
124+
func normalizeObj(obj client.Object) {
125+
obj.SetUID("NORMALIZED_FOR_TESTING")
126+
obj.SetCreationTimestamp(magicTime)
127+
obj.SetManagedFields(nil)
128+
obj.SetNamespace("NORMALIZED_FOR_TESTING")
129+
obj.SetResourceVersion("NORMALIZED_FOR_TESTING")
130+
}

internal/appliance/kubernetes.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333
// limitations of Go generics.
3434
func createOrUpdateObject[R client.Object](
3535
ctx context.Context, r *Reconciler, updateIfChanged any,
36-
owner client.Object, obj client.Object, objKind R,
36+
owner client.Object, obj, objKind R,
3737
) error {
3838
logger := log.FromContext(ctx).WithValues("kind", obj.GetObjectKind().GroupVersionKind(), "namespace", obj.GetNamespace(), "name", obj.GetName())
3939
namespacedName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}

0 commit comments

Comments
 (0)