Skip to content

Commit 1ec7186

Browse files
authored
Make sure dashboard is deployed after server and public api server (#18995)
* [service-waiter] add cli to wait deployments server and public-api-server * [installer] make dashboard to wait server and papi * Implement TODOs * fixup * [installer] change service account name * fix build * fix rate limit hit * improve * remove debug cli env * Skip with empty image and change attempt duration * fixup
1 parent 18e04fb commit 1ec7186

File tree

10 files changed

+381
-14
lines changed

10 files changed

+381
-14
lines changed
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License.AGPL.txt in the project root for license information.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"fmt"
10+
"time"
11+
12+
"github.com/sirupsen/logrus"
13+
"github.com/spf13/cobra"
14+
"github.com/spf13/viper"
15+
16+
"github.com/gitpod-io/gitpod/common-go/log"
17+
corev1 "k8s.io/api/core/v1"
18+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19+
"k8s.io/client-go/kubernetes"
20+
"k8s.io/client-go/rest"
21+
)
22+
23+
var componentCmdOpt struct {
24+
image string
25+
namespace string
26+
component string
27+
labels string
28+
}
29+
30+
var componentCmd = &cobra.Command{
31+
Use: "component",
32+
Short: "waits for component to become latest build of current installer build",
33+
PreRun: func(cmd *cobra.Command, args []string) {
34+
err := viper.BindPFlags(cmd.Flags())
35+
if err != nil {
36+
log.WithError(err).Fatal("cannot bind Viper to pflags")
37+
}
38+
},
39+
Run: func(cmd *cobra.Command, args []string) {
40+
if componentCmdOpt.image == "" {
41+
log.Errorf("target image is empty, skip service waiter %s", componentCmdOpt.component)
42+
return
43+
}
44+
timeout := getTimeout()
45+
log.WithField("timeout", timeout.String()).WithFields(logrus.Fields{"image": componentCmdOpt.image, "component": componentCmdOpt.component, "namespace": componentCmdOpt.namespace, "labels": componentCmdOpt.labels}).Info("start to wait component")
46+
ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
47+
defer cancel()
48+
49+
err := waitPodsImage(ctx)
50+
51+
if err != nil {
52+
log.WithError(err).Fatal("failed to wait service")
53+
} else {
54+
log.Info("service is ready")
55+
}
56+
},
57+
}
58+
59+
func checkPodsImage(ctx context.Context, k8sClient *kubernetes.Clientset) (bool, error) {
60+
pods, err := k8sClient.CoreV1().Pods(componentCmdOpt.namespace).List(ctx, metav1.ListOptions{
61+
LabelSelector: componentCmdOpt.labels,
62+
})
63+
if err != nil {
64+
return false, fmt.Errorf("cannot get pod list: %w", err)
65+
}
66+
if len(pods.Items) == 0 {
67+
return false, fmt.Errorf("no pods found")
68+
}
69+
readyCount := 0
70+
for _, pod := range pods.Items {
71+
for _, container := range pod.Spec.Containers {
72+
if container.Name == componentCmdOpt.component {
73+
if container.Image != componentCmdOpt.image {
74+
return false, fmt.Errorf("image is not the same: %s != %s", container.Image, componentCmdOpt.image)
75+
}
76+
for _, condition := range pod.Status.Conditions {
77+
if condition.Type == corev1.PodReady {
78+
if condition.Status == corev1.ConditionTrue {
79+
readyCount += 1
80+
} else {
81+
return false, fmt.Errorf("pod is not ready")
82+
}
83+
}
84+
}
85+
}
86+
}
87+
}
88+
log.Infof("ready pods: %d/%d", readyCount, len(pods.Items))
89+
return readyCount == len(pods.Items), nil
90+
}
91+
92+
func waitPodsImage(ctx context.Context) error {
93+
k8sCfg, err := rest.InClusterConfig()
94+
if err != nil {
95+
return fmt.Errorf("cannot get in cluster config: %w", err)
96+
}
97+
k8sClient, err := kubernetes.NewForConfig(k8sCfg)
98+
if err != nil {
99+
return fmt.Errorf("cannot create k8s client: %w", err)
100+
}
101+
ok := false
102+
for {
103+
select {
104+
case <-ctx.Done():
105+
if ok {
106+
return nil
107+
}
108+
return ctx.Err()
109+
default:
110+
ok, err := checkPodsImage(ctx, k8sClient)
111+
if err != nil {
112+
log.WithError(err).Error("image check failed")
113+
time.Sleep(1 * time.Second)
114+
continue
115+
}
116+
if ok {
117+
return nil
118+
}
119+
time.Sleep(1 * time.Second)
120+
}
121+
}
122+
}
123+
124+
func init() {
125+
rootCmd.AddCommand(componentCmd)
126+
componentCmd.Flags().StringVar(&componentCmdOpt.image, "image", "", "The latest image of current installer build")
127+
componentCmd.Flags().StringVar(&componentCmdOpt.namespace, "namespace", "", "The namespace of deployment")
128+
componentCmd.Flags().StringVar(&componentCmdOpt.component, "component", "", "Component name of deployment")
129+
componentCmd.Flags().StringVar(&componentCmdOpt.labels, "labels", "", "Labels of deployment")
130+
131+
_ = componentCmd.MarkFlagRequired("namespace")
132+
_ = componentCmd.MarkFlagRequired("component")
133+
_ = componentCmd.MarkFlagRequired("labels")
134+
}

components/service-waiter/go.mod

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,37 +7,70 @@ require (
77
github.com/go-sql-driver/mysql v1.6.0
88
github.com/mitchellh/go-homedir v1.1.0
99
github.com/redis/go-redis/v9 v9.0.5
10+
github.com/sirupsen/logrus v1.9.3
1011
github.com/spf13/cobra v1.4.0
1112
github.com/spf13/viper v1.7.0
13+
k8s.io/api v0.27.3
14+
k8s.io/apimachinery v0.27.3
15+
k8s.io/client-go v0.0.0-00010101000000-000000000000
1216
)
1317

1418
require (
1519
github.com/cespare/xxhash/v2 v2.2.0 // indirect
20+
github.com/davecgh/go-spew v1.1.1 // indirect
1621
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
22+
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
1723
github.com/fsnotify/fsnotify v1.4.9 // indirect
1824
github.com/gitpod-io/gitpod/components/scrubber v0.0.0-00010101000000-000000000000 // indirect
25+
github.com/go-logr/logr v1.2.3 // indirect
26+
github.com/go-openapi/jsonpointer v0.19.6 // indirect
27+
github.com/go-openapi/jsonreference v0.20.1 // indirect
28+
github.com/go-openapi/swag v0.22.3 // indirect
29+
github.com/gogo/protobuf v1.3.2 // indirect
1930
github.com/golang/protobuf v1.5.3 // indirect
31+
github.com/google/gnostic v0.5.7-v3refs // indirect
32+
github.com/google/go-cmp v0.5.9 // indirect
33+
github.com/google/gofuzz v1.1.0 // indirect
34+
github.com/google/uuid v1.3.0 // indirect
2035
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
2136
github.com/hashicorp/hcl v1.0.0 // indirect
2237
github.com/inconshreveable/mousetrap v1.0.0 // indirect
38+
github.com/josharian/intern v1.0.0 // indirect
39+
github.com/json-iterator/go v1.1.12 // indirect
2340
github.com/magiconair/properties v1.8.1 // indirect
41+
github.com/mailru/easyjson v0.7.7 // indirect
2442
github.com/mitchellh/mapstructure v1.1.2 // indirect
2543
github.com/mitchellh/reflectwalk v1.0.2 // indirect
44+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
45+
github.com/modern-go/reflect2 v1.0.2 // indirect
46+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
2647
github.com/pelletier/go-toml v1.2.0 // indirect
27-
github.com/sirupsen/logrus v1.9.3 // indirect
48+
github.com/rogpeppe/go-internal v1.11.0 // indirect
2849
github.com/spf13/afero v1.1.2 // indirect
2950
github.com/spf13/cast v1.3.0 // indirect
3051
github.com/spf13/jwalterweatherman v1.0.0 // indirect
3152
github.com/spf13/pflag v1.0.5 // indirect
3253
github.com/subosito/gotenv v1.2.0 // indirect
3354
golang.org/x/net v0.10.0 // indirect
55+
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
3456
golang.org/x/sys v0.11.0 // indirect
57+
golang.org/x/term v0.8.0 // indirect
3558
golang.org/x/text v0.9.0 // indirect
59+
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
60+
google.golang.org/appengine v1.6.7 // indirect
3661
google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect
3762
google.golang.org/grpc v1.52.3 // indirect
3863
google.golang.org/protobuf v1.30.0 // indirect
64+
gopkg.in/inf.v0 v0.9.1 // indirect
3965
gopkg.in/ini.v1 v1.51.0 // indirect
4066
gopkg.in/yaml.v2 v2.4.0 // indirect
67+
gopkg.in/yaml.v3 v3.0.1 // indirect
68+
k8s.io/klog/v2 v2.90.1 // indirect
69+
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
70+
k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
71+
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
72+
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
73+
sigs.k8s.io/yaml v1.3.0 // indirect
4174
)
4275

4376
replace github.com/gitpod-io/gitpod/common-go => ../common-go // leeway

0 commit comments

Comments
 (0)