Skip to content

Commit 700b605

Browse files
mustard-mhmustard-mh
authored
Make db waiter to wait latest migration (#18455)
* Make db waiter to wait latest migration * Move generate out of build * Remove conn require * Allow to ignore migration check * Use shell * Use fail and remote useless log * Generate service-waiter txt when create migration * Revert "Generate service-waiter txt when create migration" This reverts commit 11c12b8. * Add log
1 parent d868852 commit 700b605

File tree

12 files changed

+174
-42
lines changed

12 files changed

+174
-42
lines changed

‎components/gitpod-db/BUILD.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,19 @@ packages:
1414
yarnLock: ${coreYarnLockBase}/yarn.lock
1515
tsconfig: tsconfig.json
1616
dontTest: true
17+
- name: latest-migration
18+
type: generic
19+
srcs:
20+
- "src/typeorm/migration/*.ts"
21+
- "scripts/generate-latest-migration.sh"
22+
config:
23+
dontTest: false
24+
commands:
25+
- ["sh", "-c", "scripts/generate-latest-migration.sh > latest-migration.txt"]
26+
test:
27+
- ["scripts/generate-latest-migration.sh", "test"]
28+
- ["sh", "-c", "rm -rf src"]
29+
- ["sh", "-c", "rm -rf scripts"]
1730
- name: migrations
1831
type: yarn
1932
srcs:

‎components/gitpod-db/scripts/generate-latest-migration.sh

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023 Gitpod GmbH. All rights reserved.
3+
# Licensed under the GNU Affero General Public License (AGPL).
4+
# See License.AGPL.txt in the project root for license information.
5+
6+
get_latest_migration() {
7+
# List all migrations and sort
8+
migrations=$(find ./src/typeorm/migration/*.ts | sort -n | sed 's/\.\/src\/typeorm\/migration\///g' | sed 's/\.ts//g')
9+
10+
# Get the latest migration and format its name {ts}-{name} into {name}{ts}
11+
# To align to the generated TypeORM class name which used as migration name
12+
latest_migration=$(echo "$migrations" | tail -n 1 | sed 's/\([0-9]\{13\}\)-\(.*\)/\2\1/g')
13+
14+
# Echo the latest migration
15+
echo "$latest_migration"
16+
}
17+
18+
test() {
19+
if [ -z "$(get_latest_migration)" ]; then
20+
echo "Error: get_latest_migration() should not be empty" 1>&2;
21+
exit 1
22+
fi
23+
}
24+
25+
if [ "$1" == "test" ]; then
26+
test
27+
else
28+
get_latest_migration
29+
fi

‎components/service-waiter/BUILD.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ packages:
77
- "go.sum"
88
deps:
99
- components/common-go:lib
10+
- components/gitpod-db:latest-migration
11+
prep:
12+
- ["sh", "-c", "mkdir -p cmd/resources"]
13+
- ["sh", "-c", "cat _deps/components-gitpod-db--latest-migration/latest-migration.txt > cmd/resources/latest-migration.txt"]
1014
env:
1115
- CGO_ENABLED=0
1216
- GOOS=linux

‎components/service-waiter/cmd/database.go

Lines changed: 78 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,13 @@
55
package cmd
66

77
import (
8+
"context"
89
"crypto/tls"
910
"crypto/x509"
1011
"database/sql"
12+
_ "embed"
13+
"errors"
14+
"fmt"
1115
"net"
1216
"os"
1317
"strings"
@@ -20,6 +24,11 @@ import (
2024
"github.com/gitpod-io/gitpod/common-go/log"
2125
)
2226

27+
const migrationTableName = "migrations"
28+
29+
//go:embed resources/latest-migration.txt
30+
var latestMigrationName string
31+
2332
// databaseCmd represents the database command
2433
var databaseCmd = &cobra.Command{
2534
Use: "database",
@@ -34,11 +43,19 @@ DB_CA_CERT and DB_USER(=gitpod)`,
3443
}
3544
},
3645
Run: func(cmd *cobra.Command, args []string) {
46+
timeout := getTimeout()
47+
ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
48+
defer cancel()
49+
3750
cfg := mysql.NewConfig()
3851
cfg.Addr = net.JoinHostPort(viper.GetString("host"), viper.GetString("port"))
3952
cfg.Net = "tcp"
4053
cfg.User = viper.GetString("username")
4154
cfg.Passwd = viper.GetString("password")
55+
56+
// Must be "gitpod"
57+
// Align to https://github.com/gitpod-io/gitpod/blob/884d922e8e33d8b936ec18d7fe3c8dcffde42b5a/components/gitpod-db/go/conn.go#L37
58+
cfg.DBName = "gitpod"
4259
cfg.Timeout = 1 * time.Second
4360

4461
dsn := cfg.FormatDSN()
@@ -51,7 +68,7 @@ DB_CA_CERT and DB_USER(=gitpod)`,
5168
if caCert != "" {
5269
rootCertPool := x509.NewCertPool()
5370
if ok := rootCertPool.AppendCertsFromPEM([]byte(caCert)); !ok {
54-
log.Fatal("Failed to append DB CA cert.")
71+
fail("Failed to append DB CA cert.")
5572
}
5673

5774
tlsConfigName := "custom"
@@ -60,51 +77,77 @@ DB_CA_CERT and DB_USER(=gitpod)`,
6077
MinVersion: tls.VersionTLS12, // semgrep finding: set lower boundary to exclude insecure TLS1.0
6178
})
6279
if err != nil {
63-
log.WithError(err).Fatal("Failed to register DB CA cert")
80+
fail(fmt.Sprintf("Failed to register DB CA cert: %+v", err))
6481
}
6582
cfg.TLSConfig = tlsConfigName
6683
}
6784

68-
timeout := getTimeout()
69-
done := make(chan bool)
70-
go func() {
71-
log.WithField("timeout", timeout.String()).WithField("dsn", censoredDSN).Info("attempting to connect to DB")
72-
for {
73-
db, err := sql.Open("mysql", cfg.FormatDSN())
74-
if err != nil {
75-
continue
76-
}
77-
err = db.Ping()
78-
if err != nil {
79-
if strings.Contains(err.Error(), "Access denied") {
80-
fail("Invalid credentials for the database. Check DB_USERNAME and DB_PASSWORD.")
81-
}
82-
83-
log.WithError(err).Debug("retry")
84-
<-time.After(time.Second)
85-
continue
86-
}
87-
88-
err = db.Close()
89-
if err != nil {
90-
log.WithError(err).Warn("cannot close DB connection used for probing")
85+
migrationName := GetLatestMigrationName()
86+
migrationCheck := viper.GetBool("migration-check")
87+
log.WithField("timeout", timeout.String()).WithField("dsn", censoredDSN).WithField("migration", migrationName).WithField("migrationCheck", migrationCheck).Info("waiting for database")
88+
var lastErr error
89+
log.Info("attempting to check if database is available")
90+
for ctx.Err() == nil {
91+
if err := checkDbAvailable(ctx, cfg, migrationName, migrationCheck); err != nil {
92+
if lastErr == nil || (lastErr != nil && err.Error() != lastErr.Error()) {
93+
// log if error is new or changed
94+
log.WithError(err).Error("database not available")
95+
log.Info("attempting to check if database is available")
96+
} else {
97+
log.WithError(err).Debug("database not available, attempting to check again")
9198
}
99+
lastErr = err
100+
<-time.After(time.Second)
101+
} else {
92102
break
93103
}
104+
}
94105

95-
done <- true
96-
}()
97-
98-
select {
99-
case <-done:
106+
if ctx.Err() != nil {
107+
log.WithError(ctx.Err()).WithError(lastErr).Error("database did not become available in time")
108+
fail(fmt.Sprintf("database did not become available in time(%s): %+v", timeout.String(), lastErr))
109+
} else {
100110
log.Info("database became available")
101-
return
102-
case <-time.After(timeout):
103-
log.WithField("timeout", timeout.String()).Fatal("database did not become available in time")
104111
}
105112
},
106113
}
107114

115+
func GetLatestMigrationName() string {
116+
return strings.TrimSpace(latestMigrationName)
117+
}
118+
119+
// checkDbAvailable will connect and check if database is connectable
120+
// with migrations and migrationCheck set, it will also check if the latest migration has been applied
121+
func checkDbAvailable(ctx context.Context, cfg *mysql.Config, migration string, migrationCheck bool) (err error) {
122+
db, err := sql.Open("mysql", cfg.FormatDSN())
123+
if err != nil {
124+
return err
125+
}
126+
// ignore error
127+
defer db.Close()
128+
129+
// if migration name is not set, just ping the database
130+
if migration == "" || !migrationCheck {
131+
return db.PingContext(ctx)
132+
}
133+
134+
row := db.QueryRowContext(ctx, "SELECT name FROM "+migrationTableName+" ORDER BY `timestamp` DESC LIMIT 1")
135+
var dbLatest string
136+
if err := row.Scan(&dbLatest); err != nil {
137+
if errors.Is(err, sql.ErrNoRows) {
138+
return fmt.Errorf("failed to check migrations: no row found")
139+
}
140+
return fmt.Errorf("failed to check migrations: %w", err)
141+
}
142+
if dbLatest != migration {
143+
return fmt.Errorf("expected migration %s, but found %s", migration, dbLatest)
144+
}
145+
146+
log.WithField("want", migration).WithField("got", dbLatest).Info("migrated")
147+
148+
return nil
149+
}
150+
108151
func init() {
109152
rootCmd.AddCommand(databaseCmd)
110153

@@ -113,4 +156,6 @@ func init() {
113156
databaseCmd.Flags().StringP("password", "P", os.Getenv("DB_PASSWORD"), "Password to use when connecting")
114157
databaseCmd.Flags().StringP("username", "u", envOrDefault("DB_USERNAME", "gitpod"), "Username to use when connected")
115158
databaseCmd.Flags().StringP("caCert", "", os.Getenv("DB_CA_CERT"), "Custom CA cert (chain) to use when connected")
159+
160+
databaseCmd.Flags().BoolP("migration-check", "", false, "Enable to check if the latest migration has been applied")
116161
}

‎components/service-waiter/cmd/database_test.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License.AGPL.txt in the project root for license information.
4+
5+
package cmd
6+
7+
import (
8+
_ "embed"
9+
"os"
10+
"strings"
11+
"testing"
12+
)
13+
14+
func TestGetLatestMigrationName(t *testing.T) {
15+
t.Run("should have latest migration name", func(t *testing.T) {
16+
path, err := os.Getwd()
17+
if err != nil {
18+
t.Errorf("failed to get current work dir: %v", err)
19+
return
20+
}
21+
if !strings.Contains(path, "components-service-waiter--app") {
22+
t.Skip("skipping test; not running in leeway build")
23+
return
24+
}
25+
if GetLatestMigrationName() == "" {
26+
t.Errorf("migration name should not be empty")
27+
}
28+
})
29+
}

‎components/service-waiter/cmd/resources/latest-migration.txt

Whitespace-only changes.

‎install/installer/pkg/common/common.go

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -447,13 +447,25 @@ func ConfigcatProxyEnv(ctx *RenderContext) []corev1.EnvVar {
447447
}
448448

449449
func DatabaseWaiterContainer(ctx *RenderContext) *corev1.Container {
450+
return databaseWaiterContainer(ctx, false)
451+
}
452+
453+
func DatabaseMigrationWaiterContainer(ctx *RenderContext) *corev1.Container {
454+
return databaseWaiterContainer(ctx, true)
455+
}
456+
457+
func databaseWaiterContainer(ctx *RenderContext, doMigrationCheck bool) *corev1.Container {
458+
args := []string{
459+
"-v",
460+
"database",
461+
}
462+
if doMigrationCheck {
463+
args = append(args, "--migration-check", "true")
464+
}
450465
return &corev1.Container{
451466
Name: "database-waiter",
452467
Image: ctx.ImageName(ctx.Config.Repository, "service-waiter", ctx.VersionManifest.Components.ServiceWaiter.Version),
453-
Args: []string{
454-
"-v",
455-
"database",
456-
},
468+
Args: args,
457469
SecurityContext: &corev1.SecurityContext{
458470
Privileged: pointer.Bool(false),
459471
AllowPrivilegeEscalation: pointer.Bool(false),

‎install/installer/pkg/components/public-api-server/deployment.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
119119
RestartPolicy: corev1.RestartPolicyAlways,
120120
TerminationGracePeriodSeconds: pointer.Int64(30),
121121
InitContainers: []corev1.Container{
122-
*common.DatabaseWaiterContainer(ctx),
122+
*common.DatabaseMigrationWaiterContainer(ctx),
123123
*common.RedisWaiterContainer(ctx),
124124
},
125125
Containers: []corev1.Container{

‎install/installer/pkg/components/server/deployment.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
308308
volumes...,
309309
),
310310
InitContainers: []corev1.Container{
311-
*common.DatabaseWaiterContainer(ctx),
311+
*common.DatabaseMigrationWaiterContainer(ctx),
312312
*common.RedisWaiterContainer(ctx),
313313
},
314314
Containers: []corev1.Container{{

‎install/installer/pkg/components/spicedb/deployment.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ func dbEnvVars(ctx *common.RenderContext) []corev1.EnvVar {
177177
}
178178

179179
func dbWaiter(ctx *common.RenderContext) v1.Container {
180-
databaseWaiter := common.DatabaseWaiterContainer(ctx)
180+
databaseWaiter := common.DatabaseMigrationWaiterContainer(ctx)
181181
// Use updated env-vars, which in the case cloud-sql-proxy override default db conf
182182

183183
databaseWaiter.Env = dbEnvVars(ctx)

‎install/installer/pkg/components/usage/deployment.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
101101
DNSPolicy: corev1.DNSClusterFirst,
102102
RestartPolicy: corev1.RestartPolicyAlways,
103103
TerminationGracePeriodSeconds: pointer.Int64(30),
104-
InitContainers: []corev1.Container{*common.DatabaseWaiterContainer(ctx)},
104+
InitContainers: []corev1.Container{*common.DatabaseMigrationWaiterContainer(ctx)},
105105
Volumes: volumes,
106106
Containers: []corev1.Container{{
107107
Name: Component,

‎install/installer/pkg/components/ws-manager-bridge/deployment.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
127127
volumes...,
128128
),
129129
InitContainers: []corev1.Container{
130-
*common.DatabaseWaiterContainer(ctx),
130+
*common.DatabaseMigrationWaiterContainer(ctx),
131131
*common.RedisWaiterContainer(ctx),
132132
},
133133
Containers: []corev1.Container{{

0 commit comments

Comments
 (0)