Skip flaky tests

Signed-off-by: perdasilva <[email protected]>

Author: perdasilva

.github/workflows/e2e-tests.yml

@@ -19,7 +19,7 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: '~1.17'
-      - run: make e2e-local E2E_TEST_CHUNK=${{ matrix.parallel-id }} E2E_TEST_NUM_CHUNKS=${{ strategy.job-total }} E2E_NODES=2 ARTIFACTS_DIR=./artifacts/
+      - run: make e2e-local E2E_TEST_CHUNK=${{ matrix.parallel-id }} E2E_TEST_NUM_CHUNKS=${{ strategy.job-total }} E2E_NODES=2 ARTIFACTS_DIR=./artifacts/ SKIP='\[FLAKY\]'
       - name: Archive Test Artifacts # test results, failed or not, are always uploaded.
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

Makefile

@@ -131,7 +131,7 @@ E2E_TEST_NUM_CHUNKS ?= 4
 ifneq (all,$(E2E_TEST_CHUNK))
 TEST := $(shell go run ./test/e2e/split/... -chunks $(E2E_TEST_NUM_CHUNKS) -print-chunk $(E2E_TEST_CHUNK) ./test/e2e)
 endif
-E2E_OPTS ?= $(if $(E2E_SEED),-seed '$(E2E_SEED)') $(if $(TEST),-focus '$(TEST)') -flakeAttempts $(E2E_FLAKE_ATTEMPTS) -nodes $(E2E_NODES) -timeout $(E2E_TIMEOUT) -v -randomizeSuites -race -trace -progress
+E2E_OPTS ?= $(if $(E2E_SEED),-seed '$(E2E_SEED)') $(if $(SKIP), -skip '$(SKIP)') $(if $(TEST),-focus '$(TEST)') -flakeAttempts $(E2E_FLAKE_ATTEMPTS) -nodes $(E2E_NODES) -timeout $(E2E_TIMEOUT) -v -randomizeSuites -race -trace -progress
 E2E_INSTALL_NS ?= operator-lifecycle-manager
 E2E_TEST_NS ?= operators
 

test/e2e/crd_e2e_test.go

@@ -94,7 +94,7 @@ var _ = Describe("CRD Versions", func() {
 		Expect(fetchedInstallPlan.Status.Phase).To(Equal(operatorsv1alpha1.InstallPlanPhaseComplete))
 	})
 
-	It("blocks a CRD upgrade that could cause data loss", func() {
+	It("[FLAKY] blocks a CRD upgrade that could cause data loss", func() {
 		By("checking the storage versions in the existing CRD status and the spec of the new CRD")
 
 		c := newKubeClient()

test/e2e/csv_e2e_test.go

@@ -19,8 +19,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/apimachinery/pkg/fields"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	k8slabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/selection"
@@ -64,10 +64,10 @@ var _ = Describe("ClusterServiceVersion", func() {
 
 	When("a CustomResourceDefinition was installed alongside a ClusterServiceVersion", func() {
 		var (
-			ns  corev1.Namespace
-			crd apiextensionsv1.CustomResourceDefinition
-			og operatorsv1.OperatorGroup
-			apiname string
+			ns          corev1.Namespace
+			crd         apiextensionsv1.CustomResourceDefinition
+			og          operatorsv1.OperatorGroup
+			apiname     string
 			apifullname string
 		)
 
@@ -144,7 +144,7 @@ var _ = Describe("ClusterServiceVersion", func() {
 			}).Should(WithTransform(k8serrors.IsNotFound, BeTrue()))
 		})
 
-		It("can satisfy an associated ClusterServiceVersion's ownership requirement", func() {
+		It("[FLAKY] can satisfy an associated ClusterServiceVersion's ownership requirement", func() {
 			associated := operatorsv1alpha1.ClusterServiceVersion{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "associated-csv",
@@ -261,7 +261,7 @@ var _ = Describe("ClusterServiceVersion", func() {
 			}).Should(Succeed())
 		})
 
-		It("can satisfy an unassociated ClusterServiceVersion's non-ownership requirement", func() {
+		It("[FLAKY] can satisfy an unassociated ClusterServiceVersion's non-ownership requirement", func() {
 			unassociated := operatorsv1alpha1.ClusterServiceVersion{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "unassociated-csv",
@@ -4206,7 +4206,7 @@ var _ = Describe("ClusterServiceVersion", func() {
 
 var _ = Describe("Disabling copied CSVs", func() {
 	var (
-		ns corev1.Namespace
+		ns  corev1.Namespace
 		csv operatorsv1alpha1.ClusterServiceVersion
 	)
 
@@ -4391,7 +4391,7 @@ var _ = Describe("Disabling copied CSVs", func() {
 			}).Should(Succeed())
 		})
 
-		It("should have copied CSVs in all other Namespaces", func() {
+		It("[FLAKY] should have copied CSVs in all other Namespaces", func() {
 			Eventually(func() error {
 				// find copied csvs...
 				requirement, err := k8slabels.NewRequirement(operatorsv1alpha1.CopiedLabelKey, selection.Equals, []string{csv.GetNamespace()})

test/e2e/subscription_e2e_test.go

@@ -1022,7 +1022,7 @@ var _ = Describe("Subscription", func() {
 	// - Delete the referenced InstallPlan
 	// - Wait for sub to have status condition SubscriptionInstallPlanMissing true
 	// - Ensure original non-InstallPlan status conditions remain after InstallPlan transitions
-	It("can reconcile InstallPlan status", func() {
+	It("[FLAKY] can reconcile InstallPlan status", func() {
 		c := newKubeClient()
 		crc := newCRClient()