Make db waiter to wait latest migration

Author: mustard-mh

components/gitpod-db/BUILD.yaml

@@ -96,6 +96,14 @@ packages:
       image:
         - ${imageRepoBase}/db-migrations:${version}
         - ${imageRepoBase}/db-migrations:commit-${__git_commit}
+  - name: latest-migration
+    type: generic
+    deps:
+      - :lib
+    config:
+      commands:
+        - ["sh", "-c", "node components-gitpod-db--lib/package/lib/typeorm/print-migration.js > latest-migration.json"]
+        - ["sh", "-c", "rm -r components*"]
 scripts:
 - name: init-testdb
   description: "Starts a properly initialized MySQL instance to run tests against. Usage: '. $(leeway run components/gitpod-db:init-testdb)'"

components/gitpod-db/src/typeorm/print-migration.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import * as fs from "node:fs";
+
+// list migration folder files and find the latest one
+const files = fs.readdirSync(__dirname + "/migration");
+const latest = files
+    .filter((f) => f.endsWith(".js") && f.includes("-"))
+    .sort()
+    .pop();
+
+if (!latest) {
+    console.log(JSON.stringify({ name: "", timestamp: "" }));
+} else {
+    const [timestamp, name] = latest.split("-");
+    console.log(JSON.stringify({ name, timestamp }));
+}

components/service-waiter/cmd/database.go

@@ -5,9 +5,11 @@
 package cmd
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"database/sql"
+	"fmt"
 	"net"
 	"os"
 	"strings"
@@ -20,6 +22,13 @@ import (
 	"github.com/gitpod-io/gitpod/common-go/log"
 )
 
+const migrationTableName = "migrations"
+
+type migrationInfo struct {
+	Name      string
+	Timestamp string
+}
+
 // databaseCmd represents the database command
 var databaseCmd = &cobra.Command{
 	Use:   "database",
@@ -34,6 +43,10 @@ DB_CA_CERT and DB_USER(=gitpod)`,
 		}
 	},
 	Run: func(cmd *cobra.Command, args []string) {
+		timeout := getTimeout()
+		ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
+		defer cancel()
+
 		cfg := mysql.NewConfig()
 		cfg.Addr = net.JoinHostPort(viper.GetString("host"), viper.GetString("port"))
 		cfg.Net = "tcp"
@@ -65,52 +78,71 @@ DB_CA_CERT and DB_USER(=gitpod)`,
 			cfg.TLSConfig = tlsConfigName
 		}
 
-		timeout := getTimeout()
-		done := make(chan bool)
-		go func() {
-			log.WithField("timeout", timeout.String()).WithField("dsn", censoredDSN).Info("attempting to connect to DB")
-			for {
-				db, err := sql.Open("mysql", cfg.FormatDSN())
-				if err != nil {
-					continue
-				}
-				err = db.Ping()
-				if err != nil {
-					if strings.Contains(err.Error(), "Access denied") {
-						fail("Invalid credentials for the database. Check DB_USERNAME and DB_PASSWORD.")
-					}
-
-					log.WithError(err).Debug("retry")
-					<-time.After(time.Second)
-					continue
-				}
-
-				err = db.Close()
-				if err != nil {
-					log.WithError(err).Warn("cannot close DB connection used for probing")
-				}
+		migration := migrationInfo{
+			Name:      viper.GetString("migration-name"),
+			Timestamp: viper.GetString("migration-timestamp"),
+		}
+
+		log.WithField("timeout", timeout.String()).WithField("dsn", censoredDSN).WithField("migration", migration).Info("waiting for database")
+		for {
+			log.Info("attempting to check if database is available")
+			if err := checkDbAvailable(ctx, cfg, &migration); err != nil {
+				log.WithError(err).Debug("retry")
+				<-time.After(time.Second)
+			} else {
 				break
 			}
-
-			done <- true
-		}()
+		}
 
 		select {
-		case <-done:
-			log.Info("database became available")
-			return
-		case <-time.After(timeout):
+		case <-ctx.Done():
 			log.WithField("timeout", timeout.String()).Fatal("database did not become available in time")
+		default:
+			log.Info("database became available")
 		}
 	},
 }
 
+// checkDbAvailable will connect and check if migrations table contains the latest migration
+func checkDbAvailable(ctx context.Context, cfg *mysql.Config, migration *migrationInfo) error {
+	db, err := sql.Open("mysql", cfg.FormatDSN())
+	if err != nil {
+		return err
+	}
+	// ignore error
+	defer db.Close()
+
+	// if migration info is not set, just ping the database
+	if migration.Name == "" || migration.Timestamp == "" {
+		return db.PingContext(ctx)
+	}
+
+	log.Info("checking if database is migrated")
+	row := db.QueryRowContext(ctx, "SELECT name FROM "+migrationTableName+" WHERE 1690915807191 = ?", migration.Timestamp)
+	var name string
+	if err := row.Scan(&name); err != nil {
+		return err
+	}
+	// migration name should be `{migrationName}{timestamp}`. i.e. AddGitStatusColumnToWorkspaceInstance1690915807191
+	if name != fmt.Sprintf("%s%s", migration.Name, migration.Timestamp) {
+		log.WithError(err).WithField("latest", name).WithField("checking", migration).Error("migrations table does not contain the latest migration")
+		return err
+	}
+
+	return nil
+}
+
 func init() {
 	rootCmd.AddCommand(databaseCmd)
 
+	// Database config
 	databaseCmd.Flags().StringP("host", "H", os.Getenv("DB_HOST"), "Host to try and connect to")
 	databaseCmd.Flags().StringP("port", "p", envOrDefault("DB_PORT", "3306"), "Port to connect on")
 	databaseCmd.Flags().StringP("password", "P", os.Getenv("DB_PASSWORD"), "Password to use when connecting")
 	databaseCmd.Flags().StringP("username", "u", envOrDefault("DB_USERNAME", "gitpod"), "Username to use when connected")
 	databaseCmd.Flags().StringP("caCert", "", os.Getenv("DB_CA_CERT"), "Custom CA cert (chain) to use when connected")
+
+	// Database migration config
+	databaseCmd.Flags().StringP("migration-name", "", os.Getenv("DB_MIGRATION_NAME"), "The latest migration name")
+	databaseCmd.Flags().StringP("migration-timestamp", "", os.Getenv("DB_MIGRATION_TIMESTAMP"), "The latest migration timestamp")
 }

components/service-waiter/cmd/root.go

@@ -100,22 +100,6 @@ func getTimeout() time.Duration {
 	return timeout
 }
 
-// fail ends the waiting process propagating the message on its way out
-func fail(message string) {
-	terminationLog := "/dev/termination-log"
-
-	log.WithField("message", message).Warn("failed to wait for a service")
-
-	if _, err := os.Stat(terminationLog); !os.IsNotExist(err) {
-		err := os.WriteFile(terminationLog, []byte(message), 0600)
-		if err != nil {
-			log.WithError(err).Error("cannot write termination log")
-		}
-	}
-
-	os.Exit(1)
-}
-
 func envOrDefault(env, def string) (res string) {
 	res = os.Getenv(env)
 	if res == "" {

install/installer/BUILD.yaml

@@ -19,6 +19,7 @@ packages:
       - components/blobserve:lib
       - components/common-go:lib
       - components/gitpod-db/go:lib
+      - components/gitpod-db:latest-migration
       - components/content-service-api/go:lib
       - components/content-service:lib
       - components/ee/agent-smith:lib
@@ -46,6 +47,7 @@ packages:
       - ["sh", "-c", "ls -d third_party/charts/*/ | while read f; do echo \"cd $f && helm dep up && cd -\"; done | sh"]
       - ["mv", "_deps/components-ws-manager-mk2--crd/workspace.gitpod.io_workspaces.yaml", "pkg/components/ws-manager-mk2/crd.yaml"]
       - ["sh", "-c", "cat _deps/components-ws-manager-mk2--crd/workspace.gitpod.io_snapshots.yaml >> pkg/components/ws-manager-mk2/crd.yaml"]
+      - ["sh", "-c", "cat _deps/components-gitpod-db--latest-migration/latest-migration.txt > pkg/common/latest-migration.txt"]
     config:
       packaging: app
       buildCommand: ["go", "build", "-trimpath", "-ldflags", "-buildid= -w -s -X 'github.com/gitpod-io/gitpod/installer/cmd.Version=commit-${__git_commit}'"]

install/installer/pkg/common/common.go

@@ -6,6 +6,7 @@ package common
 
 import (
 	"crypto/sha256"
+	_ "embed"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -446,13 +447,29 @@ func ConfigcatProxyEnv(ctx *RenderContext) []corev1.EnvVar {
 	return envs
 }
 
+//go:embed resources/latest-migration.json
+var LatestMigrationInfo []byte
+
 func DatabaseWaiterContainer(ctx *RenderContext) *corev1.Container {
+	type migrationInfo struct {
+		Name      string `json:"name,omitempty"`
+		Timestamp string `json:"timestamp,omitempty"`
+	}
+	var migration migrationInfo
+	if err := json.Unmarshal(LatestMigrationInfo, &migration); err != nil {
+		panic(fmt.Sprintf("cannot unmarshal latest migration info: %v", err))
+	}
+
 	return &corev1.Container{
 		Name:  "database-waiter",
 		Image: ctx.ImageName(ctx.Config.Repository, "service-waiter", ctx.VersionManifest.Components.ServiceWaiter.Version),
 		Args: []string{
 			"-v",
 			"database",
+			"--migration-name",
+			migration.Name,
+			"--migration-timestamp",
+			migration.Timestamp,
 		},
 		SecurityContext: &corev1.SecurityContext{
 			Privileged:               pointer.Bool(false),

install/installer/pkg/common/common_test.go

@@ -5,6 +5,7 @@
 package common_test
 
 import (
+	"encoding/json"
 	"fmt"
 	"testing"
 
@@ -46,3 +47,20 @@ func TestKubeRBACProxyContainerWithConfig(t *testing.T) {
 		{Name: baseserver.BuiltinMetricsPortName, ContainerPort: baseserver.BuiltinMetricsPort},
 	}, container.Ports)
 }
+
+func TestDatabaseWaiterContainer2(t *testing.T) {
+	t.Run("helloaaaaaaaa", func(t *testing.T) {
+		type data struct {
+			Name      string `json:"name,omitempty"`
+			Timestamp string `json:"timestamp,omitempty"`
+		}
+		d := data{}
+		json.Unmarshal(common.LatestMigrationInfo, &d)
+		if d.Name != "AddGitStatusColumnToWorkspaceInstance" {
+			t.Errorf("DatabaseWaiterContainer() = %v, want %v", d.Name, "AddGitStatusColumnToWorkspaceInstance")
+		}
+		if d.Timestamp != "1690915807191" {
+			t.Errorf("DatabaseWaiterContainer() = %v, want %v", d.Timestamp, "1690915807191")
+		}
+	})
+}

install/installer/pkg/common/resources/latest-migration.json

@@ -0,0 +1 @@
+{}