Clean-up installed CRDs on Stop()

But only if USE_EXISTING_CLUSTER is set to true as there's no point in doing it
if the control-plane is going away.

Signed-off-by: Pires <[email protected]>

Author: pires

pkg/envtest/crd.go

@@ -27,6 +27,7 @@ import (
 
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -180,6 +181,32 @@ func (p *poller) poll() (done bool, err error) {
 	return allFound, nil
 }
 
+// UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory
+func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
+
+	// Read the CRD yamls into options.CRDs
+	if err := readCRDFiles(&options); err != nil {
+		return err
+	}
+
+	// Delete the CRDs from the apiserver
+	cs, err := clientset.NewForConfig(config)
+	if err != nil {
+		return err
+	}
+
+	// Create each CRD
+	for _, crd := range options.CRDs {
+		log.V(1).Info("uninstalling CRD", "crd", crd.Name)
+		if err := cs.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(crd.Name, &metav1.DeleteOptions{}); err != nil {
+			return err
+		}
+	}
+	return nil
+
+	return nil
+}
+
 // CreateCRDs creates the CRDs
 func CreateCRDs(config *rest.Config, crds []*apiextensionsv1beta1.CustomResourceDefinition) error {
 	cs, err := clientset.NewForConfig(config)

pkg/envtest/envtest_test.go

@@ -25,6 +25,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -52,6 +53,18 @@ var _ = Describe("Test", func() {
 	// Cleanup CRDs
 	AfterEach(func(done Done) {
 		for _, crd := range crds {
+			// Delete only if CRD exists.
+			crdObjectKey := client.ObjectKey{
+				Namespace: crd.Namespace,
+				Name:      crd.Name,
+			}
+			var placeholder v1beta1.CustomResourceDefinition
+			err := c.Get(context.TODO(), crdObjectKey, &placeholder)
+			if err != nil && apierrors.IsNotFound(err) {
+				// CRD doesn't need to be deleted.
+				continue
+			}
+			Expect(err).NotTo(HaveOccurred())
 			Expect(c.Delete(context.TODO(), crd)).To(Succeed())
 		}
 		close(done)
@@ -216,4 +229,105 @@ var _ = Describe("Test", func() {
 			close(done)
 		}, 5)
 	})
+
+	Describe("UninstallCRDs", func() {
+		It("should uninstall the CRDs from the cluster", func(done Done) {
+
+			crds, err = InstallCRDs(env.Config, CRDInstallOptions{
+				Paths: []string{filepath.Join(".", "testdata")},
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			// Expect to find the CRDs
+
+			crd := &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Foo"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Baz"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Captain"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("FirstMate"))
+
+			crd = &v1beta1.CustomResourceDefinition{}
+			err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crd.Spec.Names.Kind).To(Equal("Driver"))
+
+			err = WaitForCRDs(env.Config, []*v1beta1.CustomResourceDefinition{
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "qux.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "bazs",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "bar.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "foos",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "crew.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "captains",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group:   "crew.example.com",
+						Version: "v1beta1",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "firstmates",
+						}},
+				},
+				{
+					Spec: v1beta1.CustomResourceDefinitionSpec{
+						Group: "crew.example.com",
+						Names: v1beta1.CustomResourceDefinitionNames{
+							Plural: "drivers",
+						},
+						Versions: []v1beta1.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Storage: true,
+								Served:  true,
+							},
+							{
+								Name:    "v2",
+								Storage: false,
+								Served:  true,
+							},
+						}},
+				},
+			},
+				CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond},
+			)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = UninstallCRDs(env.Config, CRDInstallOptions{
+				Paths: []string{filepath.Join(".", "testdata")},
+			})
+			Expect(err).NotTo(HaveOccurred())
+
+			close(done)
+		}, 10)
+	})
 })

pkg/envtest/server.go

@@ -135,10 +135,10 @@ type Environment struct {
 }
 
 // Stop stops a running server.
-// If USE_EXISTING_CLUSTER is set to true, this method is a no-op.
+// If USE_EXISTING_CLUSTER is set to true, uninstall whatever CRDs were installed with Start().
 func (te *Environment) Stop() error {
 	if te.useExistingCluster() {
-		return nil
+		return UninstallCRDs(te.Config, te.CRDInstallOptions)
 	}
 	return te.ControlPlane.Stop()
 }