Merge pull request #2024 from joelanford/fix/upgrade-alert

Bug 1932626: Gracefully handle service unavailable errors from kube-apiserver

Author: openshift-merge-robot

pkg/controller/operators/olm/apiservices.go

@@ -239,8 +239,8 @@ func (a *Operator) areAPIServicesAvailable(csv *v1alpha1.ClusterServiceVersion)
 			return false, nil
 		}
 
-		if err := a.isGVKRegistered(desc.Group, desc.Version, desc.Kind); err != nil {
-			return false, nil
+		if ok, err := a.isGVKRegistered(desc.Group, desc.Version, desc.Kind); !ok || err != nil {
+			return false, err
 		}
 	}
 

pkg/controller/operators/olm/operator.go

@@ -1565,15 +1565,21 @@ func (a *Operator) transitionCSVState(in v1alpha1.ClusterServiceVersion) (out *v
 			out.SetPhaseWithEvent(v1alpha1.CSVPhasePending, v1alpha1.CSVReasonNeedsReinstall, "calculated deployment install is bad", now, a.recorder)
 			return
 		}
-		if installErr := a.updateInstallStatus(out, installer, strategy, v1alpha1.CSVPhaseInstalling, v1alpha1.CSVReasonWaiting); installErr == nil {
-			logger.WithField("strategy", out.Spec.InstallStrategy.StrategyName).Infof("install strategy successful")
-		} else {
+		if installErr := a.updateInstallStatus(out, installer, strategy, v1alpha1.CSVPhaseInstalling, v1alpha1.CSVReasonWaiting); installErr != nil {
+			// Re-sync if kube-apiserver was unavailable
+			if k8serrors.IsServiceUnavailable(installErr) {
+				logger.WithError(installErr).Info("could not update install status")
+				syncError = installErr
+				return
+			}
 			// Set phase to failed if it's been a long time since the last transition (5 minutes)
 			if out.Status.LastTransitionTime != nil && a.now().Sub(out.Status.LastTransitionTime.Time) >= 5*time.Minute {
 				logger.Warn("install timed out")
 				out.SetPhaseWithEvent(v1alpha1.CSVPhaseFailed, v1alpha1.CSVReasonInstallCheckFailed, fmt.Sprintf("install timeout"), now, a.recorder)
+				return
 			}
 		}
+		logger.WithField("strategy", out.Spec.InstallStrategy.StrategyName).Infof("install strategy successful")
 
 	case v1alpha1.CSVPhaseSucceeded:
 		// Check if the current CSV is being replaced, return with replacing status if so
@@ -1622,6 +1628,12 @@ func (a *Operator) transitionCSVState(in v1alpha1.ClusterServiceVersion) (out *v
 			return
 		}
 		if installErr := a.updateInstallStatus(out, installer, strategy, v1alpha1.CSVPhaseFailed, v1alpha1.CSVReasonComponentUnhealthy); installErr != nil {
+			// Re-sync if kube-apiserver was unavailable
+			if k8serrors.IsServiceUnavailable(installErr) {
+				logger.WithError(installErr).Info("could not update install status")
+				syncError = installErr
+				return
+			}
 			logger.WithField("strategy", out.Spec.InstallStrategy.StrategyName).Warnf("unhealthy component: %s", installErr)
 			return
 		}
@@ -1704,6 +1716,12 @@ func (a *Operator) transitionCSVState(in v1alpha1.ClusterServiceVersion) (out *v
 			return
 		}
 		if installErr := a.updateInstallStatus(out, installer, strategy, v1alpha1.CSVPhasePending, v1alpha1.CSVReasonNeedsReinstall); installErr != nil {
+			// Re-sync if kube-apiserver was unavailable
+			if k8serrors.IsServiceUnavailable(installErr) {
+				logger.WithError(installErr).Info("could not update install status")
+				syncError = installErr
+				return
+			}
 			logger.WithField("strategy", out.Spec.InstallStrategy.StrategyName).Warnf("needs reinstall: %s", installErr)
 		}
 
@@ -1782,6 +1800,10 @@ func (a *Operator) updateInstallStatus(csv *v1alpha1.ClusterServiceVersion, inst
 		return nil
 	}
 
+	if err := findFirstError(k8serrors.IsServiceUnavailable, strategyErr, apiServiceErr, webhookErr); err != nil {
+		return err
+	}
+
 	// installcheck determined we can't progress (e.g. deployment failed to come up in time)
 	if install.IsErrorUnrecoverable(strategyErr) {
 		csv.SetPhaseWithEventIfChanged(v1alpha1.CSVPhaseFailed, v1alpha1.CSVReasonInstallCheckFailed, fmt.Sprintf("install failed: %s", strategyErr), now, a.recorder)
@@ -1829,6 +1851,15 @@ func (a *Operator) updateInstallStatus(csv *v1alpha1.ClusterServiceVersion, inst
 	return nil
 }
 
+func findFirstError(f func(error) bool, errs ...error) error {
+	for _, err := range errs {
+		if f(err) {
+			return err
+		}
+	}
+	return nil
+}
+
 // parseStrategiesAndUpdateStatus returns a StrategyInstaller and a Strategy for a CSV if it can, else it sets a status on the CSV and returns
 func (a *Operator) parseStrategiesAndUpdateStatus(csv *v1alpha1.ClusterServiceVersion) (install.StrategyInstaller, install.Strategy) {
 	strategy, err := a.resolver.UnmarshalStrategy(csv.Spec.InstallStrategy)

pkg/controller/operators/olm/requirements.go

@@ -11,7 +11,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/operator-framework/api/pkg/operators/v1alpha1"
-	olmErrors "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/errors"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil"
 )
@@ -156,7 +155,7 @@ func (a *Operator) requirementStatus(strategyDetailsDeployment *v1alpha1.Strateg
 		}
 
 		// Check if GVK exists
-		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
+		if ok, err := a.isGVKRegistered(r.Group, r.Version, r.Kind); !ok || err != nil {
 			status.Status = "NotPresent"
 			met = false
 			statuses = append(statuses, status)
@@ -219,7 +218,7 @@ func (a *Operator) requirementStatus(strategyDetailsDeployment *v1alpha1.Strateg
 			Name:    name,
 		}
 
-		if err := a.isGVKRegistered(r.Group, r.Version, r.Kind); err != nil {
+		if ok, err := a.isGVKRegistered(r.Group, r.Version, r.Kind); !ok || err != nil {
 			status.Status = v1alpha1.RequirementStatusReasonNotPresent
 			status.Message = "Native API does not exist"
 			met = false
@@ -388,7 +387,7 @@ func (a *Operator) requirementAndPermissionStatus(csv *v1alpha1.ClusterServiceVe
 	return met, statuses, nil
 }
 
-func (a *Operator) isGVKRegistered(group, version, kind string) error {
+func (a *Operator) isGVKRegistered(group, version, kind string) (bool, error) {
 	logger := a.logger.WithFields(logrus.Fields{
 		"group":   group,
 		"version": version,
@@ -399,15 +398,15 @@ func (a *Operator) isGVKRegistered(group, version, kind string) error {
 	resources, err := a.opClient.KubernetesInterface().Discovery().ServerResourcesForGroupVersion(gv.String())
 	if err != nil {
 		logger.WithField("err", err).Info("could not query for GVK in api discovery")
-		return err
+		return false, err
 	}
 
 	for _, r := range resources.APIResources {
 		if r.Kind == kind {
-			return nil
+			return true, nil
 		}
 	}
 
 	logger.Info("couldn't find GVK in api discovery")
-	return olmErrors.GroupVersionKindNotFoundError{group, version, kind}
+	return false, nil
 }