Introduce a mechanism for gathering test artifacts during individual test failures (#2442)

* Introduce mechanism for gathering test artifacts during individual test failures

Update the testing e2e suite and add a mechanism for gathering test
artifacts during individual test failures. Currently, container logs are
gathered when deprovisioning upstream kind clusters, yet we lack
fine-grain ability to diagnose test failures further.

Note: test failures use the `CurrentGinkgoTestDescription.Failed` field
to determine failures. Testing artifacts are only gathered when the base
$ARTIFACTS_DIR environment variable has been specified.

Add a collect-ci-artifacts.sh bash script, responsible for gathering OLM
native resources for an individual testing namespace. This bash script
will be called when tearing down the generated testing namespace for
relevant e2e packages. Currently, the artifact gathering process is
restricted to only a single namespace - longer term, it might be
possible to instead migrate towards collecting resources that all share
a similar label selector, and utilizing that label selector to handle
multi-namespace testing scenarios.

Introduce another helper function in test/e2e/util_test.go that's
responsible for gathering test artifacts (i.e. calling this newly
introduced script) when the test case had failed, and in either case,
delete the namespace.

Signed-off-by: timflannagan <[email protected]>

* test/e2e: Refactor the bundle e2e tests to avoid using global test namespace

Signed-off-by: timflannagan <[email protected]>

Upstream-repository: operator-lifecycle-manager
Upstream-commit: 4cc28bb74ff6721eb41491ec57c55cc47bc1ad84

Author: timflannagan

staging/operator-lifecycle-manager/test/e2e/bundle_e2e_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 
 	"github.com/ghodss/yaml"
+	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -27,19 +28,23 @@ var vpaCRDRaw []byte
 
 var _ = Describe("Installing bundles with new object types", func() {
 	var (
-		kubeClient     operatorclient.ClientInterface
-		operatorClient versioned.Interface
-		dynamicClient  dynamic.Interface
+		kubeClient         operatorclient.ClientInterface
+		operatorClient     versioned.Interface
+		dynamicClient      dynamic.Interface
+		generatedNamespace corev1.Namespace
 	)
 
 	BeforeEach(func() {
 		kubeClient = ctx.Ctx().KubeClient()
 		operatorClient = ctx.Ctx().OperatorClient()
 		dynamicClient = ctx.Ctx().DynamicClient()
+
+		By("creating a test namespace")
+		generatedNamespace = SetupGeneratedTestNamespace(genName("bundle-e2e-"))
 	})
 
 	AfterEach(func() {
-		TearDown(testNamespace)
+		TeardownNamespace(generatedNamespace.GetName())
 	})
 
 	When("a bundle with a pdb, priorityclass, and VPA object is installed", func() {
@@ -66,7 +71,7 @@ var _ = Describe("Installing bundles with new object types", func() {
 			Expect(err).ToNot(HaveOccurred(), "could not convert vpa crd to unstructured")
 
 			Eventually(func() error {
-				err := ctx.Ctx().Client().Create(context.TODO(), &vpaCRD)
+				err := ctx.Ctx().Client().Create(context.Background(), &vpaCRD)
 				if err != nil {
 					if !k8serrors.IsAlreadyExists(err) {
 						return err
@@ -77,22 +82,21 @@ var _ = Describe("Installing bundles with new object types", func() {
 
 			// ensure vpa crd is established and accepted on the cluster before continuing
 			Eventually(func() (bool, error) {
-				crd, err := kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), vpaCRD.GetName(), metav1.GetOptions{})
+				crd, err := kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), vpaCRD.GetName(), metav1.GetOptions{})
 				if err != nil {
 					return false, err
 				}
 				return crdReady(&crd.Status), nil
 			}).Should(BeTrue())
 
-			var installPlanRef string
 			source := &v1alpha1.CatalogSource{
 				TypeMeta: metav1.TypeMeta{
 					Kind:       v1alpha1.CatalogSourceKind,
 					APIVersion: v1alpha1.CatalogSourceCRDAPIVersion,
 				},
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      sourceName,
-					Namespace: testNamespace,
+					Namespace: generatedNamespace.GetName(),
 					Labels:    map[string]string{"olm.catalogSource": sourceName},
 				},
 				Spec: v1alpha1.CatalogSourceSpec{
@@ -102,7 +106,7 @@ var _ = Describe("Installing bundles with new object types", func() {
 			}
 
 			Eventually(func() error {
-				source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.TODO(), source, metav1.CreateOptions{})
+				source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.Background(), source, metav1.CreateOptions{})
 				return err
 			}).Should(Succeed())
 
@@ -114,13 +118,13 @@ var _ = Describe("Installing bundles with new object types", func() {
 			_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
 
 			// Wait for the Subscription to succeed
-			sub, err := fetchSubscription(operatorClient, testNamespace, subName, subscriptionStateAtLatestChecker)
+			sub, err := fetchSubscription(operatorClient, generatedNamespace.GetName(), subName, subscriptionStateAtLatestChecker)
 			Expect(err).ToNot(HaveOccurred(), "could not get subscription at latest status")
 
-			installPlanRef = sub.Status.InstallPlanRef.Name
+			installPlanRef := sub.Status.InstallPlanRef
 
 			// Wait for the installplan to complete (5 minute timeout)
-			_, err = fetchInstallPlan(GinkgoT(), operatorClient, installPlanRef, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete))
+			_, err = fetchInstallPlanWithNamespace(GinkgoT(), operatorClient, installPlanRef.Name, installPlanRef.Namespace, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete))
 			Expect(err).ToNot(HaveOccurred(), "could not get installplan at complete phase")
 
 			ctx.Ctx().Logf("install plan %s completed", installPlanRef)
@@ -144,25 +148,25 @@ var _ = Describe("Installing bundles with new object types", func() {
 
 			// confirm extra bundle objects are installed
 			Eventually(func() error {
-				_, err := kubeClient.KubernetesInterface().SchedulingV1().PriorityClasses().Get(context.TODO(), priorityClassName, metav1.GetOptions{})
+				_, err := kubeClient.KubernetesInterface().SchedulingV1().PriorityClasses().Get(context.Background(), priorityClassName, metav1.GetOptions{})
 				return err
 			}).Should(Succeed(), "expected no error getting priorityclass object associated with CSV")
 
 			Eventually(func() error {
-				_, err := dynamicClient.Resource(resource).Namespace(testNamespace).Get(context.TODO(), vpaName, metav1.GetOptions{})
+				_, err := dynamicClient.Resource(resource).Namespace(generatedNamespace.GetName()).Get(context.Background(), vpaName, metav1.GetOptions{})
 				return err
 			}).Should(Succeed(), "expected no error finding vpa object associated with csv")
 
 			Eventually(func() error {
-				_, err := kubeClient.KubernetesInterface().PolicyV1().PodDisruptionBudgets(testNamespace).Get(context.TODO(), pdbName, metav1.GetOptions{})
+				_, err := kubeClient.KubernetesInterface().PolicyV1().PodDisruptionBudgets(generatedNamespace.GetName()).Get(context.Background(), pdbName, metav1.GetOptions{})
 				return err
 			}).Should(Succeed(), "expected no error getting pdb object associated with CSV")
 		})
 
 		AfterEach(func() {
 			By("Deleting the VPA CRD")
 			Eventually(func() error {
-				err := ctx.Ctx().Client().Delete(context.TODO(), &vpaCRD)
+				err := ctx.Ctx().Client().Delete(context.Background(), &vpaCRD)
 				if k8serrors.IsNotFound(err) {
 					return nil
 				}

staging/operator-lifecycle-manager/test/e2e/ctx/ctx.go

@@ -2,6 +2,9 @@ package ctx
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 
 	. "github.com/onsi/ginkgo"
@@ -32,6 +35,7 @@ type TestContext struct {
 	dynamicClient  dynamic.Interface
 	packageClient  pversioned.Interface
 	ssaClient      *controllerclient.ServerSideApplier
+	artifactsDir   string
 
 	scheme *runtime.Scheme
 
@@ -86,6 +90,35 @@ func (ctx TestContext) SSAClient() *controllerclient.ServerSideApplier {
 	return ctx.ssaClient
 }
 
+func (ctx TestContext) DumpNamespaceArtifacts(namespace string) error {
+	if ctx.artifactsDir == "" {
+		ctx.Logf("$ARTIFACTS_DIR is unset -- not collecting failed test case logs")
+		return nil
+	}
+	ctx.Logf("collecting logs in the %s artifacts directory", ctx.artifactsDir)
+
+	logDir := filepath.Join(ctx.artifactsDir, namespace)
+	if err := os.MkdirAll(logDir, os.ModePerm); err != nil {
+		return err
+	}
+	envvars := []string{
+		"TEST_NAMESPACE=" + namespace,
+		"TEST_ARTIFACTS_DIR=" + logDir,
+		"KUBECONFIG=" + os.Getenv("KUBECONFIG"),
+	}
+
+	// compiled test binary running e2e tests is run from the root ./bin directory
+	cmd := exec.Command("../test/e2e/collect-ci-artifacts.sh")
+	cmd.Env = append(cmd.Env, envvars...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func setDerivedFields(ctx *TestContext) error {
 	if ctx == nil {
 		return fmt.Errorf("nil test context")

staging/operator-lifecycle-manager/test/e2e/ctx/provisioner_kind.go

@@ -134,15 +134,18 @@ func Provision(ctx *TestContext) (func(), error) {
 	if err != nil {
 		return nil, fmt.Errorf("error loading kubeconfig: %s", err.Error())
 	}
-
 	ctx.restConfig = restConfig
 
+	if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" {
+		ctx.artifactsDir = artifactsDir
+	}
+
 	var once sync.Once
 	deprovision := func() {
 		once.Do(func() {
-			if artifactsDir := os.Getenv("ARTIFACTS_DIR"); artifactsDir != "" {
+			if ctx.artifactsDir != "" {
 				ctx.Logf("collecting container logs for the %s cluster", name)
-				if err := provider.CollectLogs(name, filepath.Join(artifactsDir, logDir)); err != nil {
+				if err := provider.CollectLogs(name, filepath.Join(ctx.artifactsDir, logDir)); err != nil {
 					ctx.Logf("failed to collect logs: %v", err)
 				}
 			}

staging/operator-lifecycle-manager/test/e2e/subscription_e2e_test.go

@@ -59,9 +59,7 @@ var _ = Describe("Subscription", func() {
 	})
 
 	AfterEach(func() {
-		Eventually(func() error {
-			return ctx.Ctx().Client().Delete(context.Background(), &generatedNamespace)
-		}).Should(Succeed())
+		TeardownNamespace(generatedNamespace.GetName())
 	})
 
 	When("an entry in the middle of a channel does not provide a required GVK", func() {

staging/operator-lifecycle-manager/test/e2e/util_test.go

@@ -936,3 +936,20 @@ func SetupGeneratedTestNamespace(name string) corev1.Namespace {
 
 	return ns
 }
+
+func TeardownNamespace(ns string) {
+	log := ctx.Ctx().Logf
+
+	currentTest := CurrentGinkgoTestDescription()
+	if currentTest.Failed {
+		log("collecting the %s namespace artifacts as the '%s' test case failed", ns, currentTest.TestText)
+		if err := ctx.Ctx().DumpNamespaceArtifacts(ns); err != nil {
+			log("failed to collect namespace artifacts: %v", err)
+		}
+	}
+
+	log("tearing down the %s namespace", ns)
+	Eventually(func() error {
+		return ctx.Ctx().KubeClient().KubernetesInterface().CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{})
+	}).Should(Succeed())
+}