[installer] make sure dashboard is deployed after server and papi-server (#19042)

* [installer] make sure dashboard is deployed after server and papi-server

* fix build

* Add unit tests

* address feedback

* wait feature flag until get actual value of timed out

* default config cat client nil

* log avg fetch time

* 1

* mock feature flag hang

* Add metric

* fixup

Author: mustard-mh

components/common-go/experiments/configcat.go

@@ -20,6 +20,7 @@ const (
 	teamNameAttribute       = "team_name"
 	vscodeClientIDAttribute = "vscode_client_id"
 	gitpodHost              = "gitpod_host"
+	component               = "component"
 )
 
 func newConfigCatClient(config configcat.Config) *configCatClient {
@@ -85,6 +86,10 @@ func attributesToUser(attributes Attributes) *configcat.UserData {
 		custom[gitpodHost] = attributes.GitpodHost
 	}
 
+	if attributes.Component != "" {
+		custom[component] = attributes.Component
+	}
+
 	return &configcat.UserData{
 		Identifier: attributes.UserID,
 		Email:      attributes.UserEmail,

components/common-go/experiments/flags.go

@@ -11,6 +11,7 @@ const (
 	OIDCServiceEnabledFlag                         = "oidcServiceEnabled"
 	SupervisorPersistServerAPIChannelWhenStartFlag = "supervisor_persist_serverapi_channel_when_start"
 	SupervisorUsePublicAPIFlag                     = "supervisor_experimental_publicapi"
+	ServiceWaiterSkipComponentsFlag                = "service_waiter_skip_components"
 )
 
 func IsPersonalAccessTokensEnabled(ctx context.Context, client Client, attributes Attributes) bool {

components/common-go/experiments/types.go

@@ -33,6 +33,10 @@ type Attributes struct {
 	VSCodeClientID string
 
 	GitpodHost string
+
+	// Component is using in components/service-waiter
+	// Feature Flag key is `service_waiter_skip_component`
+	Component string
 }
 
 type ClientOpt func(o *options)
@@ -45,10 +49,25 @@ func WithGitpodProxy(gitpodHost string) ClientOpt {
 	}
 }
 
+func WithPollInterval(interval time.Duration) ClientOpt {
+	return func(o *options) {
+		o.pollInterval = interval
+	}
+}
+
+func WithDefaultClient(defaultClient Client) ClientOpt {
+	return func(o *options) {
+		o.defaultClient = defaultClient
+		o.hasDefaultClient = true
+	}
+}
+
 type options struct {
-	pollInterval time.Duration
-	baseURL      string
-	sdkKey       string
+	pollInterval     time.Duration
+	baseURL          string
+	sdkKey           string
+	defaultClient    Client
+	hasDefaultClient bool
 }
 
 // NewClient constructs a new experiments.Client. This is NOT A SINGLETON.
@@ -66,6 +85,9 @@ func NewClient(opts ...ClientOpt) Client {
 	}
 
 	if opt.sdkKey == "" {
+		if opt.hasDefaultClient {
+			return opt.defaultClient
+		}
 		return NewAlwaysReturningDefaultValueClient()
 	}
 	logger := log.Log.Dup()

components/service-waiter/cmd/component.go

@@ -7,24 +7,31 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"github.com/gitpod-io/gitpod/common-go/experiments"
 	"github.com/gitpod-io/gitpod/common-go/log"
+	"github.com/gitpod-io/gitpod/service-waiter/pkg/metrics"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
 
 var componentCmdOpt struct {
-	image     string
-	namespace string
-	component string
-	labels    string
+	image          string
+	namespace      string
+	component      string
+	labels         string
+	ideMetricsHost string
+	gitpodHost     string
+
+	featureFlagTimeout time.Duration
 }
 
 var componentCmd = &cobra.Command{
@@ -46,6 +53,8 @@ var componentCmd = &cobra.Command{
 		ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
 		defer cancel()
 
+		go startWaitFeatureFlag(ctx, componentCmdOpt.featureFlagTimeout)
+
 		err := waitPodsImage(ctx)
 
 		if err != nil {
@@ -56,6 +65,8 @@ var componentCmd = &cobra.Command{
 	},
 }
 
+var shouldSkipComponentWaiter bool = false
+
 func checkPodsImage(ctx context.Context, k8sClient *kubernetes.Clientset) (bool, error) {
 	pods, err := k8sClient.CoreV1().Pods(componentCmdOpt.namespace).List(ctx, metav1.ListOptions{
 		LabelSelector: componentCmdOpt.labels,
@@ -107,6 +118,10 @@ func waitPodsImage(ctx context.Context) error {
 			}
 			return ctx.Err()
 		default:
+			if shouldSkipComponentWaiter {
+				log.Infof("skip component waiter %s with Feature Flag", componentCmdOpt.component)
+				return nil
+			}
 			ok, err := checkPodsImage(ctx, k8sClient)
 			if err != nil {
 				log.WithError(err).Error("image check failed")
@@ -127,8 +142,75 @@ func init() {
 	componentCmd.Flags().StringVar(&componentCmdOpt.namespace, "namespace", "", "The namespace of deployment")
 	componentCmd.Flags().StringVar(&componentCmdOpt.component, "component", "", "Component name of deployment")
 	componentCmd.Flags().StringVar(&componentCmdOpt.labels, "labels", "", "Labels of deployment")
+	componentCmd.Flags().StringVar(&componentCmdOpt.ideMetricsHost, "ide-metrics-host", "", "Host of ide metrics")
+	componentCmd.Flags().DurationVar(&componentCmdOpt.featureFlagTimeout, "feature-flag-timeout", 3*time.Minute, "The maximum time to wait for feature flag")
+	componentCmd.Flags().StringVar(&componentCmdOpt.gitpodHost, "gitpod-host", "", "Domain of Gitpod installation")
 
 	_ = componentCmd.MarkFlagRequired("namespace")
 	_ = componentCmd.MarkFlagRequired("component")
 	_ = componentCmd.MarkFlagRequired("labels")
 }
+
+func startWaitFeatureFlag(ctx context.Context, timeout time.Duration) {
+	featureFlagCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	client := experiments.NewClient(experiments.WithDefaultClient(nil), experiments.WithPollInterval(time.Second*3))
+	defaultSkip := true
+	if client == nil {
+		log.Error("failed to create experiments client, skip immediately")
+		shouldSkipComponentWaiter = defaultSkip
+		metrics.AddSkipComponentsCounter(componentCmdOpt.ideMetricsHost, strconv.FormatBool(shouldSkipComponentWaiter), false)
+		return
+	}
+	getShouldSkipComponentWaiter := func() {
+		startTime := time.Now()
+		value, isActualValue, fetchTimes := ActualWaitFeatureFlag(featureFlagCtx, client, defaultSkip)
+		avgTime := time.Duration(0)
+		if fetchTimes > 0 {
+			avgTime = time.Since(startTime) / time.Duration(fetchTimes)
+		}
+		log.WithField("fetchTimes", fetchTimes).WithField("avgTime", avgTime).WithField("isActualValue", isActualValue).WithField("value", value).Info("get final value of feature flag")
+		shouldSkipComponentWaiter = value
+		metrics.AddSkipComponentsCounter(componentCmdOpt.ideMetricsHost, strconv.FormatBool(shouldSkipComponentWaiter), isActualValue)
+	}
+	for !shouldSkipComponentWaiter {
+		if featureFlagCtx.Err() != nil {
+			break
+		}
+		getShouldSkipComponentWaiter()
+		time.Sleep(1 * time.Second)
+	}
+}
+
+var FeatureSleepDuration = 1 * time.Second
+
+func ActualWaitFeatureFlag(ctx context.Context, client experiments.Client, defaultValue bool) (flagValue bool, ok bool, fetchTimes int) {
+	if client == nil {
+		return defaultValue, false, fetchTimes
+	}
+	for {
+		select {
+		case <-ctx.Done():
+			log.WithField("defaultValue", defaultValue).Info("feature flag timeout with no expected value, fallback")
+			return defaultValue, false, fetchTimes
+		default:
+			stringValue := client.GetStringValue(ctx, experiments.ServiceWaiterSkipComponentsFlag, "NONE", experiments.Attributes{
+				GitpodHost: componentCmdOpt.gitpodHost,
+				Component:  componentCmdOpt.component,
+			})
+			fetchTimes++
+			if stringValue == "true" {
+				return true, true, fetchTimes
+			}
+			if stringValue == "false" {
+				return false, true, fetchTimes
+			}
+			if stringValue == "NONE" {
+				time.Sleep(FeatureSleepDuration)
+				continue
+			}
+			log.WithField("value", stringValue).Warn("unexpected value from feature flag")
+			time.Sleep(FeatureSleepDuration)
+		}
+	}
+}