Merge pull request #502 from tmshort/OCPBUGS-13128

OCPBUGS-13128: Retry initialization error conditions (#2979)

Author: openshift-merge-robot

staging/operator-lifecycle-manager/pkg/lib/operatorstatus/monitor.go

@@ -129,9 +129,18 @@ func (m *monitor) Run(stopCh <-chan struct{}) {
 	m.logger.Infof("initializing clusteroperator resource(s) for %s", m.names)
 
 	for _, name := range m.names {
-		if err := m.init(name); err != nil {
-			m.logger.Errorf("initialization error - %v", err)
-			break
+		for {
+			if err := m.init(name); err != nil {
+				m.logger.Errorf("initialization error - %v", err)
+			} else {
+				m.logger.Infof("initialized cluster resource - %s", name)
+				break
+			}
+			select {
+			case <-time.After(defaultProbeInterval):
+			case <-stopCh:
+				return
+			}
 		}
 	}
 

staging/operator-lifecycle-manager/pkg/lib/queueinformer/queueinformer_operator.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/kubestate"
 	"github.com/pkg/errors"
@@ -13,6 +14,10 @@ import (
 	"k8s.io/client-go/tools/cache"
 )
 
+const (
+	defaultServerVersionInterval = 1 * time.Minute
+)
+
 // ExtensibleOperator describes a Reconciler that can be extended with additional informers and queue informers
 type ExtensibleOperator interface {
 	// RegisterQueueInformer registers the given QueueInformer with the Operator.
@@ -194,6 +199,16 @@ func (o *operator) start(ctx context.Context) error {
 	go func() {
 		defer close(errs)
 		v, err := o.serverVersion.ServerVersion()
+		if err == nil {
+			o.logger.Infof("connection established. cluster-version: %v", v)
+			return
+		}
+		select {
+		case <-time.After(defaultServerVersionInterval):
+		case <-ctx.Done():
+			return
+		}
+		v, err = o.serverVersion.ServerVersion()
 		if err != nil {
 			select {
 			case errs <- errors.Wrap(err, "communicating with server failed"):

staging/operator-lifecycle-manager/pkg/lib/queueinformer/queueinformer_operator_test.go

@@ -78,7 +78,7 @@ func TestOperatorRunChannelClosure(t *testing.T) {
 
 			o.Run(ctx)
 
-			timeout := time.After(time.Second)
+			timeout := time.After(2 * defaultServerVersionInterval)
 			for n, ch := range map[string]<-chan struct{}{
 				"ready": o.Ready(),
 				"done":  o.Done(),