[WIP] OCPBUGS 2782: Update service deletion to retry with timeout

staging/operator-lifecycle-manager/Makefile

@@ -29,7 +29,11 @@ GO := GO111MODULE=on GOFLAGS="$(MOD_FLAGS)" go
 GINKGO := $(GO) run github.com/onsi/ginkgo/v2/ginkgo
 BINDATA := $(GO) run github.com/go-bindata/go-bindata/v3/go-bindata
 GIT_COMMIT := $(shell git rev-parse HEAD)
-
+ifeq ($(shell arch), arm64) 
+ARCH := arm64
+else
+ARCH := 386
+endif
 # Phony prerequisite for targets that rely on the go build cache to determine staleness.
 .PHONY: build test clean vendor \
 	coverage coverage-html e2e \
@@ -82,15 +86,15 @@ build-coverage: build_cmd=test -c -covermode=count -coverpkg ./pkg/controller/..
 build-coverage: clean $(CMDS)
 
 build-linux: build_cmd=build
-build-linux: arch_flags=GOOS=linux GOARCH=386
+build-linux: arch_flags=GOOS=linux GOARCH=$(ARCH)
 build-linux: clean $(CMDS)
 
 build-wait: clean bin/wait
 
 bin/wait: FORCE
-	GOOS=linux GOARCH=386 go build $(MOD_FLAGS) -o $@ $(PKG)/test/e2e/wait
+	GOOS=linux GOARCH=$(ARCH) go build $(MOD_FLAGS) -o $@ $(PKG)/test/e2e/wait
 
-build-util-linux: arch_flags=GOOS=linux GOARCH=386
+build-util-linux: arch_flags=GOOS=linux GOARCH=$(ARCH)
 build-util-linux: build-util
 
 build-util: bin/cpb

staging/operator-lifecycle-manager/doc/install/install.md

@@ -25,7 +25,7 @@ This command starts minikube, builds the OLM containers locally with the minikub
 $ make run-local
 ```
 
-You can verify that the OLM components have been successfully deployed by running `kubectl -n local get deployments`
+You can verify that the OLM components have been successfully deployed by running `kubectl -n olm get deployments`
 
 **NOTE** It is recommended for development purposes and will use the source locally
 

staging/operator-lifecycle-manager/go.mod

@@ -22,9 +22,9 @@ require (
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/onsi/ginkgo/v2 v2.1.3
 	github.com/onsi/gomega v1.18.1
-	github.com/openshift/api v0.0.0-20220525145417-ee5b62754c68
-	github.com/openshift/client-go v0.0.0-20220525160904-9e1acff93e4a
-	github.com/operator-framework/api v0.17.1
+	github.com/openshift/api v0.0.0-20200331152225-585af27e34fd
+	github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0
+	github.com/operator-framework/api v0.16.0
 	github.com/operator-framework/operator-registry v1.17.5
 	github.com/otiai10/copy v1.2.0
 	github.com/pkg/errors v0.9.1
@@ -221,7 +221,7 @@ require (
 	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
@@ -258,6 +258,3 @@ replace (
 	go.opentelemetry.io/otel => go.opentelemetry.io/otel v0.20.0
 	go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v0.20.0
 )
-
-// downstream only
-require github.com/openshift/cluster-policy-controller v0.0.0-20220825134653-523e4104074f

staging/operator-lifecycle-manager/go.sum

@@ -1075,13 +1075,11 @@ github.com/openshift/api v0.0.0-20211014063134-be2a7fb8aa44/go.mod h1:RsQCVJu4qh
 github.com/openshift/build-machinery-go v0.0.0-20210712174854-1bb7fd1518d3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0 h1:kMiuiZXH1GdfbiMwsuAQOqGaMxlo9NCUk0wT4XAdfNM=
 github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0/go.mod h1:uUQ4LClRO+fg5MF/P6QxjMCb1C9f7Oh4RKepftDnEJE=
-github.com/openshift/cluster-policy-controller v0.0.0-20220825134653-523e4104074f h1:ll0eE7rgGHsFlrI6ksr6nXL2ur8GYBe8Jj0GwNQ/1+o=
-github.com/openshift/cluster-policy-controller v0.0.0-20220825134653-523e4104074f/go.mod h1:r9ZZT5wjwoS2heBfYR26uJhhkGYwgmFqomu9ww0y9Qw=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/operator-framework/api v0.7.1/go.mod h1:L7IvLd/ckxJEJg/t4oTTlnHKAJIP/p51AvEslW3wYdY=
-github.com/operator-framework/api v0.17.1 h1:J/6+Xj4IEV8C7hcirqUFwOiZAU3PbnJhWvB0/bB51c4=
-github.com/operator-framework/api v0.17.1/go.mod h1:kk8xJahHJR3bKqrA+A+1VIrhOTmyV76k+ARv+iV+u1Q=
+github.com/operator-framework/api v0.16.0 h1:swUOhVv7QDszxBTwYM8QAtyeqI4EQHNVAiKMS+xjakY=
+github.com/operator-framework/api v0.16.0/go.mod h1:kk8xJahHJR3bKqrA+A+1VIrhOTmyV76k+ARv+iV+u1Q=
 github.com/operator-framework/operator-registry v1.17.5 h1:LR8m1rFz5Gcyje8WK6iYt+gIhtzqo52zMRALdmTYHT0=
 github.com/operator-framework/operator-registry v1.17.5/go.mod h1:sRQIgDMZZdUcmHltzyCnM6RUoDF+WS8Arj1BQIARDS8=
 github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=
@@ -1552,9 +1550,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
-golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

staging/operator-lifecycle-manager/pkg/controller/install/certresources.go

@@ -1,6 +1,7 @@
 package install
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 	"time"
@@ -12,6 +13,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/certs"
@@ -42,6 +44,10 @@ const (
 	// olm managed label
 	OLMManagedLabelKey   = "olm.managed"
 	OLMManagedLabelValue = "true"
+
+	// service deletion timeout
+	serviceDeletionTimeout                = 3 * time.Minute
+	defaultServiceDeletionPollingInterval = 5 * time.Second
 )
 
 type certResource interface {
@@ -260,9 +266,8 @@ func (i *StrategyDeploymentInstaller) installCertRequirementsForDeployment(deplo
 		service.SetOwnerReferences(existingService.GetOwnerReferences())
 
 		// Delete the Service to replace
-		deleteErr := i.strategyClient.GetOpClient().DeleteService(service.GetNamespace(), service.GetName(), &metav1.DeleteOptions{})
-		if deleteErr != nil && !apierrors.IsNotFound(deleteErr) {
-			return nil, nil, fmt.Errorf("could not delete existing service %s", service.GetName())
+		if err := i.deleteServiceWithTimeout(service, serviceDeletionTimeout); err != nil {
+			return nil, nil, err
 		}
 	}
 
@@ -538,6 +543,40 @@ func (i *StrategyDeploymentInstaller) installCertRequirementsForDeployment(deplo
 	return &depSpec, caPEM, nil
 }
 
+func (i *StrategyDeploymentInstaller) deleteServiceWithTimeout(service *corev1.Service, deletionTimeout time.Duration) error {
+	timeout, cancelFn := context.WithTimeout(context.Background(), deletionTimeout)
+	defer cancelFn()
+
+	err := wait.PollImmediateInfiniteWithContext(timeout, i.serviceDeletionPollingInterval, func(ctx context.Context) (bool, error) {
+		// try to delete service
+		err := i.strategyClient.GetOpClient().DeleteService(service.GetNamespace(), service.GetName(), &metav1.DeleteOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, nil
+		}
+
+		// ensure it is deleted
+		_, err = i.strategyClient.GetOpClient().GetService(i.owner.GetNamespace(), service.GetName())
+		// _, err = i.strategyClient.GetOpLister().CoreV1().ServiceLister().Services(i.owner.GetNamespace()).Get(service.GetName())
+
+		// successfully deleted
+		if err != nil && apierrors.IsNotFound(err) {
+			return true, nil
+		}
+
+		// otherwise, try again...
+		return false, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("manual intervention required: unable to delete existing service %s within timeout period %s", service.GetName(), deletionTimeout)
+	}
+
+	return nil
+}
+
 func SetCAAnnotation(depSpec *appsv1.DeploymentSpec, caHash string) {
 	if len(depSpec.Template.ObjectMeta.GetAnnotations()) == 0 {
 		depSpec.Template.ObjectMeta.SetAnnotations(map[string]string{OLMCAHashAnnotationKey: caHash})