Skip to content

Commit f0d3d26

Browse files
timflannagantimflannagan
committed
test/e2e: Wrap flake-prone garbage collection tests in eventually assertions
Update the test/e2e/gc_e2e_test.go and wrap any error-prone test blocks in eventually assertions to reduce the number of flakes that occur when attempting to create/update/delete/etc. Kubernetes resources during individual CI runs on these garbage collection specs. Signed-off-by: timflannagan <[email protected]>
1 parent 07dd3a0 commit f0d3d26

File tree

1 file changed

+119
-72
lines changed

1 file changed

+119
-72
lines changed

‎test/e2e/gc_e2e_test.go

Lines changed: 119 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,7 @@ var _ = Describe("Garbage collection for dependent resources", func() {
4747
BeforeEach(func() {
4848
group := fmt.Sprintf("%s.com", rand.String(16))
4949

50-
// Create a CustomResourceDefinition
51-
var err error
52-
crd, err = kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), &apiextensionsv1.CustomResourceDefinition{
50+
crd = &apiextensionsv1.CustomResourceDefinition{
5351
ObjectMeta: metav1.ObjectMeta{
5452
Name: fmt.Sprintf("plural.%s", group),
5553
},
@@ -73,18 +71,27 @@ var _ = Describe("Garbage collection for dependent resources", func() {
7371
ListKind: "KindList",
7472
},
7573
},
76-
}, metav1.CreateOptions{})
77-
Expect(err).NotTo(HaveOccurred())
74+
}
7875

79-
// Create a ClusterRole for the crd
80-
cr, err = kubeClient.CreateClusterRole(&rbacv1.ClusterRole{
76+
// Create a CustomResourceDefinition
77+
var err error
78+
Eventually(func() error {
79+
crd, err = kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
80+
return err
81+
}).Should(Succeed())
82+
83+
cr = &rbacv1.ClusterRole{
8184
ObjectMeta: metav1.ObjectMeta{
8285
GenerateName: "clusterrole-",
8386
OwnerReferences: []metav1.OwnerReference{ownerutil.NonBlockingOwner(crd)},
8487
},
85-
})
86-
Expect(err).NotTo(HaveOccurred())
88+
}
8789

90+
// Create a ClusterRole for the crd
91+
Eventually(func() error {
92+
cr, err = kubeClient.CreateClusterRole(cr)
93+
return err
94+
}).Should(Succeed())
8895
})
8996

9097
AfterEach(func() {
@@ -100,11 +107,13 @@ var _ = Describe("Garbage collection for dependent resources", func() {
100107

101108
BeforeEach(func() {
102109
// Delete CRD
103-
Expect(kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.GetName(), metav1.DeleteOptions{})).To(Succeed())
110+
Eventually(func() bool {
111+
err := kubeClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.GetName(), metav1.DeleteOptions{})
112+
return k8serrors.IsNotFound(err)
113+
}).Should(BeTrue())
104114
})
105115

106116
It("should delete the associated ClusterRole", func() {
107-
108117
Eventually(func() bool {
109118
_, err := kubeClient.GetClusterRole(cr.GetName())
110119
return k8serrors.IsNotFound(err)
@@ -117,16 +126,14 @@ var _ = Describe("Garbage collection for dependent resources", func() {
117126
Context("Given a ClusterRole owned by a APIService", func() {
118127

119128
var (
120-
as *apiregistrationv1.APIService
121-
cr *rbacv1.ClusterRole
129+
apiService *apiregistrationv1.APIService
130+
cr *rbacv1.ClusterRole
122131
)
123132

124133
BeforeEach(func() {
125134
group := rand.String(16)
126135

127-
// Create an API Service
128-
var err error
129-
as, err = kubeClient.CreateAPIService(&apiregistrationv1.APIService{
136+
apiService = &apiregistrationv1.APIService{
130137
ObjectMeta: metav1.ObjectMeta{
131138
Name: fmt.Sprintf("v1.%s", group),
132139
},
@@ -136,32 +143,44 @@ var _ = Describe("Garbage collection for dependent resources", func() {
136143
GroupPriorityMinimum: 1,
137144
VersionPriority: 1,
138145
},
139-
})
140-
Expect(err).NotTo(HaveOccurred())
146+
}
147+
// Create an API Service
148+
var err error
149+
Eventually(func() error {
150+
apiService, err = kubeClient.CreateAPIService(apiService)
151+
return err
152+
}).Should(Succeed())
141153

142-
// Create a ClusterRole
143-
cr, err = kubeClient.CreateClusterRole(&rbacv1.ClusterRole{
154+
cr = &rbacv1.ClusterRole{
144155
ObjectMeta: metav1.ObjectMeta{
145156
GenerateName: "clusterrole-",
146-
OwnerReferences: []metav1.OwnerReference{ownerutil.NonBlockingOwner(as)},
157+
OwnerReferences: []metav1.OwnerReference{ownerutil.NonBlockingOwner(apiService)},
147158
},
148-
})
149-
Expect(err).NotTo(HaveOccurred())
159+
}
160+
161+
Eventually(func() error {
162+
// Create a ClusterRole
163+
cr, err = kubeClient.CreateClusterRole(cr)
164+
return err
165+
}).Should(Succeed())
150166
})
151167

152168
AfterEach(func() {
153169

154170
IgnoreError(kubeClient.DeleteClusterRole(cr.GetName(), &metav1.DeleteOptions{}))
155171

156-
IgnoreError(kubeClient.DeleteAPIService(as.GetName(), &metav1.DeleteOptions{}))
172+
IgnoreError(kubeClient.DeleteAPIService(apiService.GetName(), &metav1.DeleteOptions{}))
157173

158174
})
159175

160176
When("APIService is deleted", func() {
161177

162178
BeforeEach(func() {
163179
// Delete API service
164-
Expect(kubeClient.DeleteAPIService(as.GetName(), &metav1.DeleteOptions{})).To(Succeed())
180+
Eventually(func() bool {
181+
err := kubeClient.DeleteAPIService(apiService.GetName(), &metav1.DeleteOptions{})
182+
return k8serrors.IsNotFound(err)
183+
}).Should(BeTrue())
165184
})
166185

167186
It("should delete the associated ClusterRole", func() {
@@ -193,18 +212,22 @@ var _ = Describe("Garbage collection for dependent resources", func() {
193212
propagation metav1.DeletionPropagation
194213
options metav1.DeleteOptions
195214
)
196-
197215
BeforeEach(func() {
198216

199217
ownerA = newCSV("ownera", testNamespace, "", semver.MustParse("0.0.0"), nil, nil, nil)
200218
ownerB = newCSV("ownerb", testNamespace, "", semver.MustParse("0.0.0"), nil, nil, nil)
201219

202220
// create all owners
203221
var err error
204-
fetchedA, err = operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Create(context.TODO(), &ownerA, metav1.CreateOptions{})
205-
Expect(err).NotTo(HaveOccurred())
206-
fetchedB, err = operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Create(context.TODO(), &ownerB, metav1.CreateOptions{})
207-
Expect(err).NotTo(HaveOccurred())
222+
Eventually(func() error {
223+
fetchedA, err = operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Create(context.TODO(), &ownerA, metav1.CreateOptions{})
224+
return err
225+
}).Should(Succeed())
226+
227+
Eventually(func() error {
228+
fetchedB, err = operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Create(context.TODO(), &ownerB, metav1.CreateOptions{})
229+
return err
230+
}).Should(Succeed())
208231

209232
dependent = &corev1.ConfigMap{
210233
ObjectMeta: metav1.ObjectMeta{
@@ -218,8 +241,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
218241
ownerutil.AddOwner(dependent, fetchedB, true, false)
219242

220243
// create ConfigMap dependent
221-
_, err = kubeClient.KubernetesInterface().CoreV1().ConfigMaps(testNamespace).Create(context.TODO(), dependent, metav1.CreateOptions{})
222-
Expect(err).NotTo(HaveOccurred(), "dependent could not be created")
244+
Eventually(func() error {
245+
_, err = kubeClient.KubernetesInterface().CoreV1().ConfigMaps(testNamespace).Create(context.TODO(), dependent, metav1.CreateOptions{})
246+
return err
247+
}).Should(Succeed(), "dependent could not be created")
223248

224249
propagation = metav1.DeletePropagationForeground
225250
options = metav1.DeleteOptions{PropagationPolicy: &propagation}
@@ -229,32 +254,35 @@ var _ = Describe("Garbage collection for dependent resources", func() {
229254

230255
BeforeEach(func() {
231256
// delete ownerA in the foreground (to ensure any "blocking" dependents are deleted before ownerA)
232-
err := operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), fetchedA.GetName(), options)
233-
Expect(err).NotTo(HaveOccurred())
257+
Eventually(func() bool {
258+
err := operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), fetchedA.GetName(), options)
259+
return k8serrors.IsNotFound(err)
260+
}).Should(BeTrue())
234261

235262
// wait for deletion of ownerA
236263
Eventually(func() bool {
237264
_, err := operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Get(context.TODO(), ownerA.GetName(), metav1.GetOptions{})
238265
return k8serrors.IsNotFound(err)
239266
}).Should(BeTrue())
240-
241267
})
242268

243269
It("should not have deleted the dependent since ownerB CSV is still present", func() {
244-
_, err := kubeClient.KubernetesInterface().CoreV1().ConfigMaps(testNamespace).Get(context.TODO(), dependent.GetName(), metav1.GetOptions{})
245-
Expect(err).NotTo(HaveOccurred(), "dependent deleted after one of the owner was deleted")
270+
Eventually(func() error {
271+
_, err := kubeClient.KubernetesInterface().CoreV1().ConfigMaps(testNamespace).Get(context.TODO(), dependent.GetName(), metav1.GetOptions{})
272+
return err
273+
}).Should(Succeed(), "dependent deleted after one of the owner was deleted")
246274
ctx.Ctx().Logf("dependent still exists after one owner was deleted")
247-
248275
})
249-
250276
})
251277

252278
When("removing both the owners using 'Foreground' deletion policy", func() {
253279

254280
BeforeEach(func() {
255281
// delete ownerA in the foreground (to ensure any "blocking" dependents are deleted before ownerA)
256-
err := operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), fetchedA.GetName(), options)
257-
Expect(err).NotTo(HaveOccurred())
282+
Eventually(func() bool {
283+
err := operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), fetchedA.GetName(), options)
284+
return k8serrors.IsNotFound(err)
285+
}).Should(BeTrue())
258286

259287
// wait for deletion of ownerA
260288
Eventually(func() bool {
@@ -263,8 +291,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
263291
}).Should(BeTrue())
264292

265293
// delete ownerB in the foreground (to ensure any "blocking" dependents are deleted before ownerB)
266-
err = operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), fetchedB.GetName(), options)
267-
Expect(err).NotTo(HaveOccurred())
294+
Eventually(func() bool {
295+
err := operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), fetchedB.GetName(), options)
296+
return k8serrors.IsNotFound(err)
297+
}).Should(BeTrue())
268298

269299
// wait for deletion of ownerB
270300
Eventually(func() bool {
@@ -274,9 +304,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
274304
})
275305

276306
It("should have deleted the dependent since both the owners were deleted", func() {
277-
_, err := kubeClient.KubernetesInterface().CoreV1().ConfigMaps(testNamespace).Get(context.TODO(), dependent.GetName(), metav1.GetOptions{})
278-
Expect(err).To(HaveOccurred())
279-
Expect(k8serrors.IsNotFound(err)).To(BeTrue())
307+
Eventually(func() bool {
308+
_, err := kubeClient.KubernetesInterface().CoreV1().ConfigMaps(testNamespace).Get(context.TODO(), dependent.GetName(), metav1.GetOptions{})
309+
return k8serrors.IsNotFound(err)
310+
}).Should(BeTrue(), "expected dependency configmap would be properly garabage collected")
280311
ctx.Ctx().Logf("dependent successfully garbage collected after both owners were deleted")
281312
})
282313

@@ -359,8 +390,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
359390

360391
BeforeEach(func() {
361392
// Delete subscription first
362-
err := operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Delete(context.TODO(), subName, metav1.DeleteOptions{})
363-
Expect(err).To(BeNil())
393+
Eventually(func() bool {
394+
err := operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Delete(context.TODO(), subName, metav1.DeleteOptions{})
395+
return k8serrors.IsNotFound(err)
396+
}).Should(BeTrue())
364397

365398
// wait for deletion
366399
Eventually(func() bool {
@@ -369,8 +402,10 @@ var _ = Describe("Garbage collection for dependent resources", func() {
369402
}).Should(BeTrue())
370403

371404
// Delete CSV
372-
err = operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), csvName, metav1.DeleteOptions{})
373-
Expect(err).To(BeNil())
405+
Eventually(func() bool {
406+
err := operatorClient.OperatorsV1alpha1().ClusterServiceVersions(testNamespace).Delete(context.TODO(), csvName, metav1.DeleteOptions{})
407+
return k8serrors.IsNotFound(err)
408+
}).Should(BeTrue())
374409

375410
// wait for deletion
376411
Eventually(func() bool {
@@ -427,8 +462,11 @@ var _ = Describe("Garbage collection for dependent resources", func() {
427462
},
428463
}
429464

430-
source, err := operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.TODO(), source, metav1.CreateOptions{})
431-
Expect(err).ToNot(HaveOccurred(), "could not create catalog source")
465+
var err error
466+
Eventually(func() error {
467+
source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.TODO(), source, metav1.CreateOptions{})
468+
return err
469+
}).Should(Succeed(), "could not create catalog source")
432470

433471
// Create a Subscription for package
434472
_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
@@ -457,17 +495,20 @@ var _ = Describe("Garbage collection for dependent resources", func() {
457495
var installPlanRef string
458496

459497
BeforeEach(func() {
460-
// update subscription first
461-
sub, err := operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Get(context.TODO(), subName, metav1.GetOptions{})
462-
Expect(err).ToNot(HaveOccurred(), "could not get subscription")
463-
464-
// update channel on sub
465-
sub.Spec.Channel = upgradeChannelName
466-
_, err = operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Update(context.TODO(), sub, metav1.UpdateOptions{})
467-
Expect(err).ToNot(HaveOccurred(), "could not update subscription")
498+
Eventually(func() error {
499+
// update subscription first
500+
sub, err := operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Get(context.TODO(), subName, metav1.GetOptions{})
501+
if err != nil {
502+
return fmt.Errorf("could not get subscription")
503+
}
504+
// update channel on sub
505+
sub.Spec.Channel = upgradeChannelName
506+
_, err = operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Update(context.TODO(), sub, metav1.UpdateOptions{})
507+
return err
508+
}).Should(Succeed(), "could not update subscription")
468509

469510
// Wait for the Subscription to succeed
470-
sub, err = fetchSubscription(operatorClient, testNamespace, subName, subscriptionStateAtLatestChecker)
511+
sub, err := fetchSubscription(operatorClient, testNamespace, subName, subscriptionStateAtLatestChecker)
471512
Expect(err).ToNot(HaveOccurred(), "could not get subscription at latest status")
472513

473514
installPlanRef = sub.Status.InstallPlanRef.Name
@@ -530,8 +571,11 @@ var _ = Describe("Garbage collection for dependent resources", func() {
530571
},
531572
}
532573

533-
source, err := operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.TODO(), source, metav1.CreateOptions{})
534-
Expect(err).ToNot(HaveOccurred(), "could not create catalog source")
574+
var err error
575+
Eventually(func() error {
576+
source, err = operatorClient.OperatorsV1alpha1().CatalogSources(source.GetNamespace()).Create(context.TODO(), source, metav1.CreateOptions{})
577+
return err
578+
}).Should(Succeed())
535579

536580
// Create a Subscription for package
537581
_ = createSubscriptionForCatalog(operatorClient, source.GetNamespace(), subName, source.GetName(), packageName, channelName, "", v1alpha1.ApprovalAutomatic)
@@ -561,17 +605,20 @@ var _ = Describe("Garbage collection for dependent resources", func() {
561605
var installPlanRef string
562606

563607
BeforeEach(func() {
564-
// update subscription first
565-
sub, err := operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Get(context.TODO(), subName, metav1.GetOptions{})
566-
Expect(err).ToNot(HaveOccurred(), "could not get subscription")
567-
568-
// update channel on sub
569-
sub.Spec.Channel = upgradeChannelName
570-
_, err = operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Update(context.TODO(), sub, metav1.UpdateOptions{})
571-
Expect(err).ToNot(HaveOccurred(), "could not update subscription")
608+
Eventually(func() error {
609+
// update subscription first
610+
sub, err := operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Get(context.TODO(), subName, metav1.GetOptions{})
611+
if err != nil {
612+
return fmt.Errorf("could not get subscription")
613+
}
614+
// update channel on sub
615+
sub.Spec.Channel = upgradeChannelName
616+
_, err = operatorClient.OperatorsV1alpha1().Subscriptions(testNamespace).Update(context.TODO(), sub, metav1.UpdateOptions{})
617+
return err
618+
}).Should(Succeed(), "could not update subscription")
572619

573620
// Wait for the Subscription to succeed
574-
sub, err = fetchSubscription(operatorClient, testNamespace, subName, subscriptionStateAtLatestChecker)
621+
sub, err := fetchSubscription(operatorClient, testNamespace, subName, subscriptionStateAtLatestChecker)
575622
Expect(err).ToNot(HaveOccurred(), "could not get subscription at latest status")
576623

577624
installPlanRef = sub.Status.InstallPlanRef.Name

0 commit comments

Comments
 (0)