Make db waiter to wait latest migration

components/gitpod-db/BUILD.yaml

@@ -14,6 +14,19 @@ packages:
       yarnLock: ${coreYarnLockBase}/yarn.lock
       tsconfig: tsconfig.json
       dontTest: true
+  - name: latest-migration
+    type: generic
+    srcs:
+      - "src/typeorm/migration/*.ts"
+      - "scripts/generate-latest-migration.sh"
+    config:
+      dontTest: false
+      commands:
+        - ["sh", "-c", "scripts/generate-latest-migration.sh > latest-migration.txt"]
+      test:
+        - ["scripts/generate-latest-migration.sh", "test"]
+        - ["sh", "-c", "rm -rf src"]
+        - ["sh", "-c", "rm -rf scripts"]
   - name: migrations
     type: yarn
     srcs:

components/gitpod-db/scripts/generate-latest-migration.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+# Licensed under the GNU Affero General Public License (AGPL).
+# See License.AGPL.txt in the project root for license information.
+
+get_latest_migration() {
+    # List all migrations and sort
+    migrations=$(find ./src/typeorm/migration/*.ts | sort -n | sed 's/\.\/src\/typeorm\/migration\///g' | sed 's/\.ts//g')
+
+    # Get the latest migration and format its name {ts}-{name} into {name}{ts}
+    # To align to the generated TypeORM class name which used as migration name
+    latest_migration=$(echo "$migrations" | tail -n 1 | sed 's/\([0-9]\{13\}\)-\(.*\)/\2\1/g')
+
+    # Echo the latest migration
+    echo "$latest_migration"
+}
+
+test() {
+    if [ -z "$(get_latest_migration)" ]; then
+        echo "Error: get_latest_migration() should not be empty" 1>&2;
+        exit 1
+    fi
+}
+
+if [ "$1" == "test" ]; then
+    test
+else
+    get_latest_migration
+fi

components/service-waiter/BUILD.yaml

@@ -7,6 +7,10 @@ packages:
       - "go.sum"
     deps:
       - components/common-go:lib
+      - components/gitpod-db:latest-migration
+    prep:
+      - ["sh", "-c", "mkdir -p cmd/resources"]
+      - ["sh", "-c", "cat _deps/components-gitpod-db--latest-migration/latest-migration.txt > cmd/resources/latest-migration.txt"]
     env:
       - CGO_ENABLED=0
       - GOOS=linux

components/service-waiter/cmd/database.go

@@ -5,9 +5,13 @@
 package cmd
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"database/sql"
+	_ "embed"
+	"errors"
+	"fmt"
 	"net"
 	"os"
 	"strings"
@@ -20,6 +24,11 @@ import (
 	"github.com/gitpod-io/gitpod/common-go/log"
 )
 
+const migrationTableName = "migrations"
+
+//go:embed resources/latest-migration.txt
+var latestMigrationName string
+
 // databaseCmd represents the database command
 var databaseCmd = &cobra.Command{
 	Use:   "database",
@@ -34,11 +43,19 @@ DB_CA_CERT and DB_USER(=gitpod)`,
 		}
 	},
 	Run: func(cmd *cobra.Command, args []string) {
+		timeout := getTimeout()
+		ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
+		defer cancel()
+
 		cfg := mysql.NewConfig()
 		cfg.Addr = net.JoinHostPort(viper.GetString("host"), viper.GetString("port"))
 		cfg.Net = "tcp"
 		cfg.User = viper.GetString("username")
 		cfg.Passwd = viper.GetString("password")
+
+		// Must be "gitpod"
+		// Align to https://github.com/gitpod-io/gitpod/blob/884d922e8e33d8b936ec18d7fe3c8dcffde42b5a/components/gitpod-db/go/conn.go#L37
+		cfg.DBName = "gitpod"
 		cfg.Timeout = 1 * time.Second
 
 		dsn := cfg.FormatDSN()
@@ -51,7 +68,7 @@ DB_CA_CERT and DB_USER(=gitpod)`,
 		if caCert != "" {
 			rootCertPool := x509.NewCertPool()
 			if ok := rootCertPool.AppendCertsFromPEM([]byte(caCert)); !ok {
-				log.Fatal("Failed to append DB CA cert.")
+				fail("Failed to append DB CA cert.")
 			}
 
 			tlsConfigName := "custom"
@@ -60,51 +77,77 @@ DB_CA_CERT and DB_USER(=gitpod)`,
 				MinVersion: tls.VersionTLS12, // semgrep finding: set lower boundary to exclude insecure TLS1.0
 			})
 			if err != nil {
-				log.WithError(err).Fatal("Failed to register DB CA cert")
+				fail(fmt.Sprintf("Failed to register DB CA cert: %+v", err))
 			}
 			cfg.TLSConfig = tlsConfigName
 		}
 
-		timeout := getTimeout()
-		done := make(chan bool)
-		go func() {
-			log.WithField("timeout", timeout.String()).WithField("dsn", censoredDSN).Info("attempting to connect to DB")
-			for {
-				db, err := sql.Open("mysql", cfg.FormatDSN())
-				if err != nil {
-					continue
-				}
-				err = db.Ping()
-				if err != nil {
-					if strings.Contains(err.Error(), "Access denied") {
-						fail("Invalid credentials for the database. Check DB_USERNAME and DB_PASSWORD.")
-					}
-
-					log.WithError(err).Debug("retry")
-					<-time.After(time.Second)
-					continue
-				}
-
-				err = db.Close()
-				if err != nil {
-					log.WithError(err).Warn("cannot close DB connection used for probing")
+		migrationName := GetLatestMigrationName()
+		migrationCheck := viper.GetBool("migration-check")
+		log.WithField("timeout", timeout.String()).WithField("dsn", censoredDSN).WithField("migration", migrationName).WithField("migrationCheck", migrationCheck).Info("waiting for database")
+		var lastErr error
+		log.Info("attempting to check if database is available")
+		for ctx.Err() == nil {
+			if err := checkDbAvailable(ctx, cfg, migrationName, migrationCheck); err != nil {
+				if lastErr == nil || (lastErr != nil && err.Error() != lastErr.Error()) {
+					// log if error is new or changed
+					log.WithError(err).Error("database not available")
+					log.Info("attempting to check if database is available")
+				} else {
+					log.WithError(err).Debug("database not available, attempting to check again")
 				}
+				lastErr = err
+				<-time.After(time.Second)
+			} else {
 				break
 			}
+		}
 
-			done <- true
-		}()
-
-		select {
-		case <-done:
+		if ctx.Err() != nil {
+			log.WithError(ctx.Err()).WithError(lastErr).Error("database did not become available in time")
+			fail(fmt.Sprintf("database did not become available in time(%s): %+v", timeout.String(), lastErr))
+		} else {
 			log.Info("database became available")
-			return
-		case <-time.After(timeout):
-			log.WithField("timeout", timeout.String()).Fatal("database did not become available in time")
 		}
 	},
 }
 
+func GetLatestMigrationName() string {
+	return strings.TrimSpace(latestMigrationName)
+}
+
+// checkDbAvailable will connect and check if database is connectable
+// with migrations and migrationCheck set, it will also check if the latest migration has been applied
+func checkDbAvailable(ctx context.Context, cfg *mysql.Config, migration string, migrationCheck bool) (err error) {
+	db, err := sql.Open("mysql", cfg.FormatDSN())
+	if err != nil {
+		return err
+	}
+	// ignore error
+	defer db.Close()
+
+	// if migration name is not set, just ping the database
+	if migration == "" || !migrationCheck {
+		return db.PingContext(ctx)
+	}
+
+	row := db.QueryRowContext(ctx, "SELECT name FROM "+migrationTableName+" ORDER BY `timestamp` DESC LIMIT 1")
+	var dbLatest string
+	if err := row.Scan(&dbLatest); err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return fmt.Errorf("failed to check migrations: no row found")
+		}
+		return fmt.Errorf("failed to check migrations: %w", err)
+	}
+	if dbLatest != migration {
+		return fmt.Errorf("expected migration %s, but found %s", migration, dbLatest)
+	}
+
+	log.WithField("want", migration).WithField("got", dbLatest).Info("migrated")
+
+	return nil
+}
+
 func init() {
 	rootCmd.AddCommand(databaseCmd)
 
@@ -113,4 +156,6 @@ func init() {
 	databaseCmd.Flags().StringP("password", "P", os.Getenv("DB_PASSWORD"), "Password to use when connecting")
 	databaseCmd.Flags().StringP("username", "u", envOrDefault("DB_USERNAME", "gitpod"), "Username to use when connected")
 	databaseCmd.Flags().StringP("caCert", "", os.Getenv("DB_CA_CERT"), "Custom CA cert (chain) to use when connected")
+
+	databaseCmd.Flags().BoolP("migration-check", "", false, "Enable to check if the latest migration has been applied")
 }

components/service-waiter/cmd/database_test.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package cmd
+
+import (
+	_ "embed"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestGetLatestMigrationName(t *testing.T) {
+	t.Run("should have latest migration name", func(t *testing.T) {
+		path, err := os.Getwd()
+		if err != nil {
+			t.Errorf("failed to get current work dir: %v", err)
+			return
+		}
+		if !strings.Contains(path, "components-service-waiter--app") {
+			t.Skip("skipping test; not running in leeway build")
+			return
+		}
+		if GetLatestMigrationName() == "" {
+			t.Errorf("migration name should not be empty")
+		}
+	})
+}

install/installer/pkg/common/common.go

@@ -447,13 +447,25 @@ func ConfigcatProxyEnv(ctx *RenderContext) []corev1.EnvVar {
 }
 
 func DatabaseWaiterContainer(ctx *RenderContext) *corev1.Container {
+	return databaseWaiterContainer(ctx, false)
+}
+
+func DatabaseMigrationWaiterContainer(ctx *RenderContext) *corev1.Container {
+	return databaseWaiterContainer(ctx, true)
+}
+
+func databaseWaiterContainer(ctx *RenderContext, doMigrationCheck bool) *corev1.Container {
+	args := []string{
+		"-v",
+		"database",
+	}
+	if doMigrationCheck {
+		args = append(args, "--migration-check", "true")
+	}
 	return &corev1.Container{
 		Name:  "database-waiter",
 		Image: ctx.ImageName(ctx.Config.Repository, "service-waiter", ctx.VersionManifest.Components.ServiceWaiter.Version),
-		Args: []string{
-			"-v",
-			"database",
-		},
+		Args:  args,
 		SecurityContext: &corev1.SecurityContext{
 			Privileged:               pointer.Bool(false),
 			AllowPrivilegeEscalation: pointer.Bool(false),

install/installer/pkg/components/public-api-server/deployment.go

@@ -119,7 +119,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						RestartPolicy:                 corev1.RestartPolicyAlways,
 						TerminationGracePeriodSeconds: pointer.Int64(30),
 						InitContainers: []corev1.Container{
-							*common.DatabaseWaiterContainer(ctx),
+							*common.DatabaseMigrationWaiterContainer(ctx),
 							*common.RedisWaiterContainer(ctx),
 						},
 						Containers: []corev1.Container{

install/installer/pkg/components/server/deployment.go

@@ -308,7 +308,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							volumes...,
 						),
 						InitContainers: []corev1.Container{
-							*common.DatabaseWaiterContainer(ctx),
+							*common.DatabaseMigrationWaiterContainer(ctx),
 							*common.RedisWaiterContainer(ctx),
 						},
 						Containers: []corev1.Container{{

install/installer/pkg/components/spicedb/deployment.go

@@ -177,7 +177,7 @@ func dbEnvVars(ctx *common.RenderContext) []corev1.EnvVar {
 }
 
 func dbWaiter(ctx *common.RenderContext) v1.Container {
-	databaseWaiter := common.DatabaseWaiterContainer(ctx)
+	databaseWaiter := common.DatabaseMigrationWaiterContainer(ctx)
 	// Use updated env-vars, which in the case cloud-sql-proxy override default db conf
 
 	databaseWaiter.Env = dbEnvVars(ctx)

install/installer/pkg/components/usage/deployment.go

@@ -101,7 +101,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						DNSPolicy:                     corev1.DNSClusterFirst,
 						RestartPolicy:                 corev1.RestartPolicyAlways,
 						TerminationGracePeriodSeconds: pointer.Int64(30),
-						InitContainers:                []corev1.Container{*common.DatabaseWaiterContainer(ctx)},
+						InitContainers:                []corev1.Container{*common.DatabaseMigrationWaiterContainer(ctx)},
 						Volumes:                       volumes,
 						Containers: []corev1.Container{{
 							Name:            Component,

install/installer/pkg/components/ws-manager-bridge/deployment.go

@@ -127,7 +127,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 							volumes...,
 						),
 						InitContainers: []corev1.Container{
-							*common.DatabaseWaiterContainer(ctx),
+							*common.DatabaseMigrationWaiterContainer(ctx),
 							*common.RedisWaiterContainer(ctx),
 						},
 						Containers: []corev1.Container{{