[e2e] Support and fixes for mk2 e2e tests

Author: WVerlaek

.github/workflows/workspace-integration-tests.yml

@@ -16,6 +16,10 @@ on:
                 required: false
                 type: boolean
                 description: "Skip delete preview environment (debug only)"
+            wsman_mk2:
+                required: false
+                type: boolean
+                description: "Run tests against ws-manager-mk2"
     schedule:
         - cron: "0 3,12 * * *"
 jobs:
@@ -99,7 +103,7 @@ jobs:
               env:
                   SLACK_WEBHOOK: ${{ steps.secrets.outputs.WORKSPACE_SLACK_WEBHOOK }}
                   SLACK_COLOR: ${{ job.status }}
-                  SLACK_MESSAGE: Workspace Integration Tests failed
+                  SLACK_MESSAGE: "Workspace Integration Tests failed (used mk2: ${{ github.event.inputs.wsman_mk2 }})"
 
     infrastructure:
       needs: [ configuration ]
@@ -200,6 +204,8 @@ jobs:
                   WS_MANAGER_TESTS="$BASE_TESTS_DIR/components/ws-manager"
                   WORKSPACE_TESTS="$BASE_TESTS_DIR/workspace"
 
+                  WS_MANAGER_MK2="${{ github.event.inputs.wsman_mk2 }}"
+
                   go install github.com/jstemmer/go-junit-report/v2@latest
 
                   FAILURE_COUNT=0

components/ws-manager-mk2/service/manager.go

@@ -366,7 +366,8 @@ func (wsm *WorkspaceManagerServer) StopWorkspace(ctx context.Context, req *wsman
 		ws.Status.SetCondition(workspacev1.NewWorkspaceConditionStoppedByRequest(gracePeriod.String()))
 		return nil
 	})
-	if err != nil {
+	// Ignore NotFound errors, workspace has already been stopped.
+	if err != nil && status.Code(err) != codes.NotFound {
 		return nil, err
 	}
 	return &wsmanapi.StopWorkspaceResponse{}, nil

test/pkg/integration/apis.go

@@ -708,11 +708,16 @@ func (c *ComponentAPI) WorkspaceManager() (wsmanapi.WorkspaceManagerClient, erro
 		return c.wsmanStatus.Client, nil
 	}
 
+	var wsman = ComponentWorkspaceManager
+	if UseWsmanMk2() {
+		wsman = ComponentWorkspaceManagerMK2
+	}
+
 	if c.wsmanStatus.Port == 0 {
 		c.wsmanStatusMu.Lock()
 		defer c.wsmanStatusMu.Unlock()
 
-		pod, _, err := selectPod(ComponentWorkspaceManager, selectPodOptions{}, c.namespace, c.client)
+		pod, _, err := selectPod(wsman, selectPodOptions{}, c.namespace, c.client)
 		if err != nil {
 			return nil, err
 		}
@@ -733,7 +738,7 @@ func (c *ComponentAPI) WorkspaceManager() (wsmanapi.WorkspaceManagerClient, erro
 		c.wsmanStatus.Port = localPort
 	}
 
-	secretName := "ws-manager-client-tls"
+	secretName := fmt.Sprintf("%s-client-tls", wsman)
 	ctx, cancel := context.WithCancel(context.Background())
 
 	c.appendCloser(func() error { cancel(); return nil })
@@ -759,7 +764,7 @@ func (c *ComponentAPI) WorkspaceManager() (wsmanapi.WorkspaceManagerClient, erro
 	creds := credentials.NewTLS(&tls.Config{
 		Certificates: []tls.Certificate{cert},
 		RootCAs:      certPool,
-		ServerName:   "ws-manager",
+		ServerName:   string(wsman),
 	})
 	dialOption := grpc.WithTransportCredentials(creds)
 
@@ -1084,12 +1089,17 @@ func (c *ComponentAPI) ImageBuilder(opts ...APIImageBuilderOpt) (imgbldr.ImageBu
 		return c.imgbldStatus.Client, nil
 	}
 
+	imgbuilder := ComponentImageBuilderMK3
+	if UseWsmanMk2() {
+		imgbuilder = ComponentImageBuilderMK3Wsman
+	}
+
 	err := func() error {
 		if c.imgbldStatus.Port == 0 {
 			c.imgbldStatusMu.Lock()
 			defer c.imgbldStatusMu.Unlock()
 
-			pod, _, err := selectPod(ComponentImageBuilderMK3, selectPodOptions{}, c.namespace, c.client)
+			pod, _, err := selectPod(imgbuilder, selectPodOptions{}, c.namespace, c.client)
 			if err != nil {
 				return err
 			}

test/pkg/integration/integration.go

@@ -586,12 +586,16 @@ const (
 	ComponentWorkspaceDaemon ComponentType = "ws-daemon"
 	// ComponentWorkspaceManager points to the workspace manager
 	ComponentWorkspaceManager ComponentType = "ws-manager"
+	// ComponentWorkspaceManagerMK2 points to the MK2 workspace manager
+	ComponentWorkspaceManagerMK2 ComponentType = "ws-manager-mk2"
 	// ComponentContentService points to the content service
 	ComponentContentService ComponentType = "content-service"
 	// ComponentWorkspace points to a workspace
 	ComponentWorkspace ComponentType = "workspace"
 	// ComponentImageBuilderMK3 points to the image-builder-mk3
 	ComponentImageBuilderMK3 ComponentType = "image-builder-mk3"
+	// ComponentImageBuilderMK3Wsman points to ws-manager-mk2's image-builder-mk3
+	ComponentImageBuilderMK3Wsman ComponentType = "image-builder-mk3-wsman"
 )
 
 func waitForPodRunningReady(c kubernetes.Interface, podName string, namespace string, timeout time.Duration) error {

test/pkg/integration/setup.go

@@ -142,17 +142,24 @@ func waitOnGitpodRunning(namespace string, waitTimeout time.Duration) env.Func {
 	klog.V(2).Info("Checking status of Gitpod components...")
 
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+		imgBuilder := "image-builder-mk3"
+		wsman := "ws-manager"
+		if UseWsmanMk2() {
+			imgBuilder = "image-builder-mk3-wsman"
+			wsman = "ws-manager-mk2"
+		}
+
 		components := []string{
 			"agent-smith",
 			"blobserve",
 			"content-service",
 			"dashboard",
-			"image-builder-mk3",
+			imgBuilder,
 			"proxy",
 			"registry-facade",
 			"server",
 			"ws-daemon",
-			"ws-manager",
+			wsman,
 			"ws-manager-bridge",
 			"ws-proxy",
 		}
@@ -221,3 +228,7 @@ func getNamespace(path string) (string, error) {
 
 	return namespace, nil
 }
+
+func UseWsmanMk2() bool {
+	return os.Getenv("WS_MANAGER_MK2") == "true"
+}

test/tests/components/ws-manager/prebuild_test.go

@@ -552,27 +552,31 @@ func TestOpenWorkspaceFromOutdatedPrebuild(t *testing.T) {
 					})
 
 					// create a prebuild
-					ws, prebuildStopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error {
-						req.Type = wsmanapi.WorkspaceType_PREBUILD
-						req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{
-							Name:  "GITPOD_TASKS",
-							Value: `[{ "init": "./init.sh" }]`,
-						})
-						req.Spec.FeatureFlags = test.FF
-						req.Spec.Initializer = &csapi.WorkspaceInitializer{
-							Spec: &csapi.WorkspaceInitializer_Git{
-								Git: &csapi.GitInitializer{
-									RemoteUri:        test.RemoteUri,
-									TargetMode:       csapi.CloneTargetMode_REMOTE_BRANCH,
-									CloneTaget:       test.CloneTargetForPrebuild,
-									CheckoutLocation: test.CheckoutLocation,
-									Config:           &csapi.GitConfig{},
+					ws, prebuildStopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api,
+						integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error {
+							req.Type = wsmanapi.WorkspaceType_PREBUILD
+							req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{
+								Name:  "GITPOD_TASKS",
+								Value: `[{ "init": "./init.sh" }]`,
+							})
+							req.Spec.FeatureFlags = test.FF
+							req.Spec.Initializer = &csapi.WorkspaceInitializer{
+								Spec: &csapi.WorkspaceInitializer_Git{
+									Git: &csapi.GitInitializer{
+										RemoteUri:        test.RemoteUri,
+										TargetMode:       csapi.CloneTargetMode_REMOTE_BRANCH,
+										CloneTaget:       test.CloneTargetForPrebuild,
+										CheckoutLocation: test.CheckoutLocation,
+										Config:           &csapi.GitConfig{},
+									},
 								},
-							},
-						}
-						req.Spec.WorkspaceLocation = test.CheckoutLocation
-						return nil
-					}))
+							}
+							req.Spec.WorkspaceLocation = test.CheckoutLocation
+							return nil
+						}),
+						// The init task only runs for a short duration, so it's possible we miss the Running state, as it quickly transitions to Stopping.
+						// Therefore ignore if we can't see the Running state.
+						integration.WithWaitWorkspaceForOpts(integration.WorkspaceCanFail))
 					if err != nil {
 						t.Fatalf("cannot launch a workspace: %q", err)
 					}
@@ -586,6 +590,9 @@ func TestOpenWorkspaceFromOutdatedPrebuild(t *testing.T) {
 					if err != nil {
 						t.Fatalf("stop workspace and find snapshot error: %v", err)
 					}
+					if prebuildSnapshot == "" {
+						t.Fatalf("prebuild snapshot is empty")
+					}
 
 					t.Logf("prebuild snapshot: %s", prebuildSnapshot)
 
@@ -1050,7 +1057,7 @@ func watchStopWorkspaceAndFindSnapshot(t *testing.T, ctx context.Context, instan
 	if lastStatus.Conditions.HeadlessTaskFailed != "" {
 		return "", nil, errors.New("unexpected HeadlessTaskFailed condition")
 	}
-	if lastStatus == nil || lastStatus.Conditions == nil || lastStatus.Conditions.VolumeSnapshot == nil {
+	if lastStatus == nil || lastStatus.Conditions == nil {
 		return "", nil, nil
 	}
 	return lastStatus.Conditions.Snapshot, lastStatus.Conditions.VolumeSnapshot, nil

test/tests/components/ws-manager/protected_secrets_test.go

@@ -120,8 +120,13 @@ func assertEnvSuppliedBySecret(t *testing.T, wsPod *corev1.Pod, secretEnv string
 					t.Fatalf("environment variable value is not supplied by secret")
 				}
 
-				if env.ValueFrom.SecretKeyRef.Name != wsPod.Name {
-					t.Fatalf("expected environment variable values are not supplied by secret %s", wsPod.Name)
+				expectedName := wsPod.Name
+				if integration.UseWsmanMk2() {
+					expectedName = fmt.Sprintf("%s-env", strings.TrimPrefix(wsPod.Name, "ws-"))
+				}
+
+				if env.ValueFrom.SecretKeyRef.Name != expectedName {
+					t.Fatalf("expected environment variable values are not supplied by secret %s", expectedName)
 				}
 			}
 		}