Add flaky e2e test stage

Signed-off-by: perdasilva <[email protected]>

Author: perdasilva

.github/workflows/flaky-e2e.yml

@@ -0,0 +1,24 @@
+name: flaky-e2e-tests
+on:
+  schedule:
+    - cron: '30 5,17 * * *' # run this every day at 5:30 and 17:30 UTC (00:30 and 12:30 ET)
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+jobs:
+  flaky-e2e-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '~1.17'
+      - run: make e2e-local E2E_NODES=1 TEST='\[FLAKY\]' ARTIFACTS_DIR=./artifacts/
+      - name: Archive Test Artifacts # test results, failed or not, are always uploaded.
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-test-output-${{(github.event.pull_request.head.sha||github.sha)}}-${{ github.run_id }}
+          path: ${{ github.workspace }}/bin/artifacts/*

test/e2e/catalog_e2e_test.go

@@ -1,4 +1,5 @@
-//  +build !bare
+//go:build !bare
+// +build !bare
 
 package e2e
 
@@ -870,7 +871,7 @@ var _ = Describe("Catalog represents a store of bundles which OLM can use to ins
 		Expect(v).Should(Equal(version.OperatorVersion{Version: busyboxVersion}), "latest version of operator not installed: catalog source update failed")
 	})
 
-	It("Dependency has correct replaces field", func() {
+	It("[FLAKY] Dependency has correct replaces field", func() {
 		// Create a CatalogSource that contains the busybox v1 and busybox-dependency v1 images
 		// Create a Subscription for busybox v1, which has a dependency on busybox-dependency v1.
 		// Wait for the busybox and busybox2 Subscriptions to succeed

test/e2e/csv_e2e_test.go

@@ -4255,7 +4255,7 @@ var _ = Describe("Disabling copied CSVs", func() {
 	})
 
 	When("an operator is installed in AllNamespace mode", func() {
-		It("should have Copied CSVs in all other namespaces", func() {
+		It("[FLAKY] should have Copied CSVs in all other namespaces", func() {
 			Eventually(func() error {
 				requirement, err := k8slabels.NewRequirement(operatorsv1alpha1.CopiedLabelKey, selection.Equals, []string{csv.GetNamespace()})
 				if err != nil {
@@ -4335,7 +4335,7 @@ var _ = Describe("Disabling copied CSVs", func() {
 			}).Should(Succeed())
 		})
 
-		It("should be reflected in the olmConfig.Status.Condition array that the expected number of copied CSVs exist", func() {
+		It("[FLAKY] should be reflected in the olmConfig.Status.Condition array that the expected number of copied CSVs exist", func() {
 			Eventually(func() error {
 				var olmConfig operatorsv1.OLMConfig
 				if err := ctx.Ctx().Client().Get(context.TODO(), apitypes.NamespacedName{Name: "cluster"}, &olmConfig); err != nil {

test/e2e/gc_e2e_test.go

@@ -646,7 +646,8 @@ var _ = Describe("Garbage collection for dependent resources", func() {
 				}).Should(BeNil())
 			})
 
-			It("should have removed the old configmap and put the new configmap in place", func() {
+			// issue: https://github.com/operator-framework/operator-lifecycle-manager/issues/2626
+			It("[FLAKY] should have removed the old configmap and put the new configmap in place", func() {
 				Eventually(func() bool {
 					_, err := kubeClient.GetConfigMap(testNamespace, configmapName)
 					return k8serrors.IsNotFound(err)

test/e2e/installplan_e2e_test.go

@@ -3190,15 +3190,15 @@ var _ = Describe("Install Plan", func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		It("should clear clear up the condition in the InstallPlan status that contains an error message when a valid OperatorGroup is created", func() {
+		It("[FLAKY] should clear clear up the condition in the InstallPlan status that contains an error message when a valid OperatorGroup is created", func() {
 
 			// first wait for a condition with a message exists
 			cond := operatorsv1alpha1.InstallPlanCondition{Type: operatorsv1alpha1.InstallPlanInstalled, Status: corev1.ConditionFalse, Reason: operatorsv1alpha1.InstallPlanReasonInstallCheckFailed,
 				Message: "no operator group found that is managing this namespace"}
 
 			Eventually(func() bool {
 				fetchedInstallPlan, err := fetchInstallPlanWithNamespace(GinkgoT(), crc, installPlanName, ns.GetName(), buildInstallPlanPhaseCheckFunc(operatorsv1alpha1.InstallPlanPhaseInstalling))
-				if err != nil || fetchedInstallPlan == nil{
+				if err != nil || fetchedInstallPlan == nil {
 					return false
 				}
 				if fetchedInstallPlan.Status.Phase != operatorsv1alpha1.InstallPlanPhaseInstalling {

test/e2e/operator_groups_e2e_test.go

@@ -1795,7 +1795,7 @@ var _ = Describe("Operator Group", func() {
 
 	// Versions of OLM at 0.14.1 and older had a bug that would place the wrong namespace annotation on copied CSVs,
 	// preventing them from being GCd. This ensures that any leftover CSVs in that state are properly cleared up.
-	It("cleanup csvs with bad owner operator groups", func() {
+	It("[FLAKY] cleanup csvs with bad owner operator groups", func() {
 
 		c := newKubeClient()
 		crc := newCRClient()
@@ -2266,7 +2266,7 @@ var _ = Describe("Operator Group", func() {
 
 			})
 
-			When("an OperatorGroup is created having matching label selector defined", func() {
+			When("[FLAKY] an OperatorGroup is created having matching label selector defined", func() {
 				var operatorGroup *v1.OperatorGroup
 
 				BeforeEach(func() {