[e2e] Support and fixes for mk2 e2e tests

.github/actions/deploy-gitpod/entrypoint.sh

@@ -29,7 +29,7 @@ previewctl get-credentials --gcp-service-account "${PREVIEW_ENV_DEV_SA_KEY_PATH}
 PREVIEW_NAME="$(previewctl get-name --branch "${INPUT_NAME}")"
 export PREVIEW_NAME
 
-for var in WITH_WS_MANAGER_MK2 WITH_DEDICATED_EMU WITH_EE_LICENSE ANALYTICS WORKSPACE_FEATURE_FLAGS; do
+for var in WSMANAGER_MK2 WITH_DEDICATED_EMU WITH_EE_LICENSE ANALYTICS WORKSPACE_FEATURE_FLAGS; do
   input_var="INPUT_${var}"
   if [[ -n "${!input_var:-}" ]];then
     export GITPOD_${var}=${!input_var}

.github/actions/deploy-gitpod/metadata.yml

@@ -13,7 +13,7 @@ inputs:
     previewctl_hash:
         description: "The Leeway hash of the dev/preview/previewctl:docker package to be used when downloading previewclt"
         required: false
-    with_ws_manager_mk2:
+    wsmanager_mk2:
         description: "Use WS Manager MK2"
         required: false
     with_dedicated_emu:

.github/workflows/build.yml

@@ -296,7 +296,7 @@ jobs:
           version: ${{needs.configuration.outputs.version}}
           sa_key: ${{ secrets.GCP_CREDENTIALS }}
           previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }}
-          with_ws_manager_mk2: ${{needs.configuration.outputs.with_ws_manager_mk2}}
+          wsmanager_mk2: ${{needs.configuration.outputs.with_ws_manager_mk2}}
           with_dedicated_emulation: ${{needs.configuration.outputs.with_dedicated_emulation}}
           with_ee_license: ${{needs.configuration.outputs.with_ee_license}}
           analytics: ${{needs.configuration.outputs.analytics}}

.github/workflows/workspace-integration-tests.yml

@@ -16,6 +16,10 @@ on:
                 required: false
                 type: boolean
                 description: "Skip delete preview environment (debug only)"
+            wsman_mk2:
+                required: false
+                type: boolean
+                description: "Run tests against ws-manager-mk2"
     schedule:
         - cron: "0 3,12 * * *"
 jobs:
@@ -99,7 +103,7 @@ jobs:
               env:
                   SLACK_WEBHOOK: ${{ steps.secrets.outputs.WORKSPACE_SLACK_WEBHOOK }}
                   SLACK_COLOR: ${{ job.status }}
-                  SLACK_MESSAGE: Workspace Integration Tests failed
+                  SLACK_MESSAGE: "Workspace Integration Tests failed"
 
     infrastructure:
       needs: [ configuration ]
@@ -126,6 +130,7 @@ jobs:
             sa_key: ${{ secrets.GCP_CREDENTIALS }}
             version: ${{ needs.configuration.outputs.version}}
             previewctl_hash: ${{needs.configuration.outputs.previewctl_hash }}
+            wsmanager_mk2: ${{ github.event.inputs.wsman_mk2 }}
 
     check:
         name: Check for regressions
@@ -200,6 +205,8 @@ jobs:
                   WS_MANAGER_TESTS="$BASE_TESTS_DIR/components/ws-manager"
                   WORKSPACE_TESTS="$BASE_TESTS_DIR/workspace"
 
+                  export WS_MANAGER_MK2="${{ github.event.inputs.wsman_mk2 }}"
+
                   go install github.com/jstemmer/go-junit-report/v2@latest
 
                   FAILURE_COUNT=0
@@ -243,7 +250,7 @@ jobs:
               env:
                   SLACK_WEBHOOK: ${{ steps.secrets.outputs.WORKSPACE_SLACK_WEBHOOK }}
                   SLACK_COLOR: ${{ job.status }}
-                  SLACK_MESSAGE: ${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} tests passed
+                  SLACK_MESSAGE: "${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} tests passed (mk2: ${{ github.event.inputs.wsman_mk2 }})"
 
     delete:
       name: Delete preview environment

components/ws-manager-mk2/service/manager.go

@@ -366,7 +366,8 @@ func (wsm *WorkspaceManagerServer) StopWorkspace(ctx context.Context, req *wsman
 		ws.Status.SetCondition(workspacev1.NewWorkspaceConditionStoppedByRequest(gracePeriod.String()))
 		return nil
 	})
-	if err != nil {
+	// Ignore NotFound errors, workspace has already been stopped.
+	if err != nil && status.Code(err) != codes.NotFound {
 		return nil, err
 	}
 	return &wsmanapi.StopWorkspaceResponse{}, nil

dev/preview/workflow/preview/deploy-gitpod.sh

@@ -634,4 +634,10 @@ leeway run components:add-smith-token \
   -DPREVIEW_K3S_KUBE_CONTEXT="${PREVIEW_K3S_KUBE_CONTEXT}" \
   -DPREVIEW_NAMESPACE="${PREVIEW_NAMESPACE}"
 
+# Add experimental node label if ws-manager-mk2 is enabled.
+# Remove once mk2 workspaces no longer run on experimental nodes.
+if [[ "${GITPOD_WSMANAGER_MK2}" == "true" ]]; then
+  kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" --namespace="${PREVIEW_NAMESPACE}" label nodes "${PREVIEW_K3S_KUBE_CONTEXT}" gitpod.io/experimental="true" --overwrite
+fi
+
 log_success "Installation is happy: https://${DOMAIN}/workspaces"

test/pkg/integration/apis.go

@@ -708,11 +708,16 @@ func (c *ComponentAPI) WorkspaceManager() (wsmanapi.WorkspaceManagerClient, erro
 		return c.wsmanStatus.Client, nil
 	}
 
+	var wsman = ComponentWorkspaceManager
+	if UseWsmanMk2() {
+		wsman = ComponentWorkspaceManagerMK2
+	}
+
 	if c.wsmanStatus.Port == 0 {
 		c.wsmanStatusMu.Lock()
 		defer c.wsmanStatusMu.Unlock()
 
-		pod, _, err := selectPod(ComponentWorkspaceManager, selectPodOptions{}, c.namespace, c.client)
+		pod, _, err := selectPod(wsman, selectPodOptions{}, c.namespace, c.client)
 		if err != nil {
 			return nil, err
 		}
@@ -733,7 +738,7 @@ func (c *ComponentAPI) WorkspaceManager() (wsmanapi.WorkspaceManagerClient, erro
 		c.wsmanStatus.Port = localPort
 	}
 
-	secretName := "ws-manager-client-tls"
+	secretName := fmt.Sprintf("%s-client-tls", wsman)
 	ctx, cancel := context.WithCancel(context.Background())
 
 	c.appendCloser(func() error { cancel(); return nil })
@@ -759,7 +764,7 @@ func (c *ComponentAPI) WorkspaceManager() (wsmanapi.WorkspaceManagerClient, erro
 	creds := credentials.NewTLS(&tls.Config{
 		Certificates: []tls.Certificate{cert},
 		RootCAs:      certPool,
-		ServerName:   "ws-manager",
+		ServerName:   string(wsman),
 	})
 	dialOption := grpc.WithTransportCredentials(creds)
 
@@ -1084,12 +1089,17 @@ func (c *ComponentAPI) ImageBuilder(opts ...APIImageBuilderOpt) (imgbldr.ImageBu
 		return c.imgbldStatus.Client, nil
 	}
 
+	imgbuilder := ComponentImageBuilderMK3
+	if UseWsmanMk2() {
+		imgbuilder = ComponentImageBuilderMK3Wsman
+	}
+
 	err := func() error {
 		if c.imgbldStatus.Port == 0 {
 			c.imgbldStatusMu.Lock()
 			defer c.imgbldStatusMu.Unlock()
 
-			pod, _, err := selectPod(ComponentImageBuilderMK3, selectPodOptions{}, c.namespace, c.client)
+			pod, _, err := selectPod(imgbuilder, selectPodOptions{}, c.namespace, c.client)
 			if err != nil {
 				return err
 			}

test/pkg/integration/integration.go

@@ -586,12 +586,16 @@ const (
 	ComponentWorkspaceDaemon ComponentType = "ws-daemon"
 	// ComponentWorkspaceManager points to the workspace manager
 	ComponentWorkspaceManager ComponentType = "ws-manager"
+	// ComponentWorkspaceManagerMK2 points to the MK2 workspace manager
+	ComponentWorkspaceManagerMK2 ComponentType = "ws-manager-mk2"
 	// ComponentContentService points to the content service
 	ComponentContentService ComponentType = "content-service"
 	// ComponentWorkspace points to a workspace
 	ComponentWorkspace ComponentType = "workspace"
 	// ComponentImageBuilderMK3 points to the image-builder-mk3
 	ComponentImageBuilderMK3 ComponentType = "image-builder-mk3"
+	// ComponentImageBuilderMK3Wsman points to ws-manager-mk2's image-builder-mk3
+	ComponentImageBuilderMK3Wsman ComponentType = "image-builder-mk3-wsman"
 )
 
 func waitForPodRunningReady(c kubernetes.Interface, podName string, namespace string, timeout time.Duration) error {

test/pkg/integration/setup.go

@@ -142,17 +142,24 @@ func waitOnGitpodRunning(namespace string, waitTimeout time.Duration) env.Func {
 	klog.V(2).Info("Checking status of Gitpod components...")
 
 	return func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+		imgBuilder := "image-builder-mk3"
+		wsman := "ws-manager"
+		if UseWsmanMk2() {
+			imgBuilder = "image-builder-mk3-wsman"
+			wsman = "ws-manager-mk2"
+		}
+
 		components := []string{
 			"agent-smith",
 			"blobserve",
 			"content-service",
 			"dashboard",
-			"image-builder-mk3",
+			imgBuilder,
 			"proxy",
 			"registry-facade",
 			"server",
 			"ws-daemon",
-			"ws-manager",
+			wsman,
 			"ws-manager-bridge",
 			"ws-proxy",
 		}
@@ -221,3 +228,7 @@ func getNamespace(path string) (string, error) {
 
 	return namespace, nil
 }
+
+func UseWsmanMk2() bool {
+	return os.Getenv("WS_MANAGER_MK2") == "true"
+}

test/tests/components/ws-manager/prebuild_test.go

@@ -552,27 +552,31 @@ func TestOpenWorkspaceFromOutdatedPrebuild(t *testing.T) {
 					})
 
 					// create a prebuild
-					ws, prebuildStopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error {
-						req.Type = wsmanapi.WorkspaceType_PREBUILD
-						req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{
-							Name:  "GITPOD_TASKS",
-							Value: `[{ "init": "./init.sh" }]`,
-						})
-						req.Spec.FeatureFlags = test.FF
-						req.Spec.Initializer = &csapi.WorkspaceInitializer{
-							Spec: &csapi.WorkspaceInitializer_Git{
-								Git: &csapi.GitInitializer{
-									RemoteUri:        test.RemoteUri,
-									TargetMode:       csapi.CloneTargetMode_REMOTE_BRANCH,
-									CloneTaget:       test.CloneTargetForPrebuild,
-									CheckoutLocation: test.CheckoutLocation,
-									Config:           &csapi.GitConfig{},
+					ws, prebuildStopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api,
+						integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error {
+							req.Type = wsmanapi.WorkspaceType_PREBUILD
+							req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{
+								Name:  "GITPOD_TASKS",
+								Value: `[{ "init": "./init.sh" }]`,
+							})
+							req.Spec.FeatureFlags = test.FF
+							req.Spec.Initializer = &csapi.WorkspaceInitializer{
+								Spec: &csapi.WorkspaceInitializer_Git{
+									Git: &csapi.GitInitializer{
+										RemoteUri:        test.RemoteUri,
+										TargetMode:       csapi.CloneTargetMode_REMOTE_BRANCH,
+										CloneTaget:       test.CloneTargetForPrebuild,
+										CheckoutLocation: test.CheckoutLocation,
+										Config:           &csapi.GitConfig{},
+									},
 								},
-							},
-						}
-						req.Spec.WorkspaceLocation = test.CheckoutLocation
-						return nil
-					}))
+							}
+							req.Spec.WorkspaceLocation = test.CheckoutLocation
+							return nil
+						}),
+						// The init task only runs for a short duration, so it's possible we miss the Running state, as it quickly transitions to Stopping.
+						// Therefore ignore if we can't see the Running state.
+						integration.WithWaitWorkspaceForOpts(integration.WorkspaceCanFail))
 					if err != nil {
 						t.Fatalf("cannot launch a workspace: %q", err)
 					}
@@ -586,6 +590,9 @@ func TestOpenWorkspaceFromOutdatedPrebuild(t *testing.T) {
 					if err != nil {
 						t.Fatalf("stop workspace and find snapshot error: %v", err)
 					}
+					if prebuildSnapshot == "" {
+						t.Fatalf("prebuild snapshot is empty")
+					}
 
 					t.Logf("prebuild snapshot: %s", prebuildSnapshot)
 
@@ -1050,7 +1057,7 @@ func watchStopWorkspaceAndFindSnapshot(t *testing.T, ctx context.Context, instan
 	if lastStatus.Conditions.HeadlessTaskFailed != "" {
 		return "", nil, errors.New("unexpected HeadlessTaskFailed condition")
 	}
-	if lastStatus == nil || lastStatus.Conditions == nil || lastStatus.Conditions.VolumeSnapshot == nil {
+	if lastStatus == nil || lastStatus.Conditions == nil {
 		return "", nil, nil
 	}
 	return lastStatus.Conditions.Snapshot, lastStatus.Conditions.VolumeSnapshot, nil

test/tests/components/ws-manager/protected_secrets_test.go

@@ -120,8 +120,13 @@ func assertEnvSuppliedBySecret(t *testing.T, wsPod *corev1.Pod, secretEnv string
 					t.Fatalf("environment variable value is not supplied by secret")
 				}
 
-				if env.ValueFrom.SecretKeyRef.Name != wsPod.Name {
-					t.Fatalf("expected environment variable values are not supplied by secret %s", wsPod.Name)
+				expectedName := wsPod.Name
+				if integration.UseWsmanMk2() {
+					expectedName = fmt.Sprintf("%s-env", strings.TrimPrefix(wsPod.Name, "ws-"))
+				}
+
+				if env.ValueFrom.SecretKeyRef.Name != expectedName {
+					t.Fatalf("expected environment variable values are not supplied by secret %s", expectedName)
 				}
 			}
 		}