[WorspaceStarter] Don't fail on temporary gRPC errors

Author: geropl

components/gitpod-protocol/src/util/grpc.ts

@@ -105,3 +105,7 @@ export function createClientCallMetricsInterceptor(metrics: IClientCallMetrics):
         return new grpc.InterceptingCall(nextCall(options), requester);
     };
 }
+
+export function isGrpcError(err: any): err is grpc.StatusObject {
+    return err.code && err.details;
+}

components/server/src/workspace/workspace-service.ts

@@ -60,6 +60,7 @@ import { goDurationToHumanReadable } from "@gitpod/gitpod-protocol/lib/util/time
 import { HeadlessLogEndpoint, HeadlessLogService } from "./headless-log-service";
 import { Deferred } from "@gitpod/gitpod-protocol/lib/util/deferred";
 import { OrganizationService } from "../orgs/organization-service";
+import { isGrpcError } from "@gitpod/gitpod-protocol/lib/util/grpc";
 
 export interface StartWorkspaceOptions extends StarterStartWorkspaceOptions {
     /**
@@ -901,10 +902,6 @@ export class WorkspaceService {
 
 // TODO(gpl) Make private after FGA rollout
 export function mapGrpcError(err: Error): Error {
-    function isGrpcError(err: any): err is grpc.StatusObject {
-        return err.code && err.details;
-    }
-
     if (!isGrpcError(err)) {
         return err;
     }

components/server/src/workspace/workspace-starter.ts

@@ -131,6 +131,7 @@ import { SYSTEM_USER } from "../authorization/authorizer";
 import { EnvVarService, ResolvedEnvVars } from "../user/env-var-service";
 import { RedlockAbortSignal } from "redlock";
 import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
+import { isGrpcError } from "@gitpod/gitpod-protocol/lib/util/grpc";
 
 export interface StartWorkspaceOptions extends GitpodServer.StartWorkspaceOptions {
     excludeFeatureFlags?: NamedWorkspaceFeatureFlag[];
@@ -658,7 +659,9 @@ export class WorkspaceStarter {
                 });
             }
         } catch (err) {
-            if (!(err instanceof StartInstanceError)) {
+            if (isGrpcError(err) && (err.code === grpc.status.UNAVAILABLE || err.code === grpc.status.ALREADY_EXISTS)) {
+                // fall-through: we don't want to fail but retry/wait for future updates to resolve this
+            } else if (!(err instanceof StartInstanceError)) {
                 // fallback in case we did not already handle this error
                 await this.failInstanceStart({ span }, err, workspace, instance, abortSignal);
                 err = new StartInstanceError("other", err); // don't throw because there's nobody catching it. We just want to log/trace it.