[spicedb] fix stale connection handling

components/content-service-api/typescript/package.json

@@ -11,7 +11,7 @@
     "lib"
   ],
   "dependencies": {
-    "@grpc/grpc-js": "1.8.8",
+    "@grpc/grpc-js": "1.9.1",
     "google-protobuf": "^3.19.1",
     "inversify": "^6.0.1",
     "opentracing": "^0.14.4"

components/gitpod-protocol/package.json

@@ -10,7 +10,7 @@
         "src"
     ],
     "devDependencies": {
-        "@grpc/grpc-js": "1.8.8",
+        "@grpc/grpc-js": "1.9.1",
         "@testdeck/mocha": "^0.3.3",
         "@types/analytics-node": "^3.1.9",
         "@types/chai-subset": "^1.3.3",

components/ide-metrics-api/typescript-grpc/package.json

@@ -9,7 +9,7 @@
     "lib"
   ],
   "dependencies": {
-    "@grpc/grpc-js": "1.8.8",
+    "@grpc/grpc-js": "1.9.1",
     "google-protobuf": "^3.19.1"
   },
   "devDependencies": {

components/image-builder-api/typescript/package.json

@@ -13,7 +13,7 @@
   "dependencies": {
     "@gitpod/content-service": "0.1.5",
     "@gitpod/gitpod-protocol": "0.1.5",
-    "@grpc/grpc-js": "1.8.8",
+    "@grpc/grpc-js": "1.9.1",
     "google-protobuf": "^3.19.1",
     "inversify": "^6.0.1",
     "opentracing": "^0.14.4"

components/server/debug.sh

@@ -26,6 +26,11 @@ kubectl scale deployment $deploymentName --replicas=1
 echo "Wait for the pod to be ready"
 kubectl wait --for=condition=ready pod -l component=$deploymentName
 
+while [[ $(kubectl get pods -l component=$deploymentName -o 'jsonpath={..status.phase}' | grep -o 'Running' | wc -w) -ne 1 ]]; do
+  echo "Waiting for scale down operation to complete..."
+  sleep 1
+done
+
 podName=$(kubectl get pods -l component=$deploymentName -o=jsonpath='{.items[0].metadata.name}')
 echo "Forward $podName port $debugPort to localhost. Waiting for a debugger to attach ..."
 kubectl port-forward pod/"$podName" $debugPort

components/server/package.json

@@ -136,5 +136,8 @@
     "ts-node": "^10.4.0",
     "typescript": "~4.4.2",
     "watch": "^1.0.2"
+  },
+  "resolutions": {
+    "@authzed/authzed-node/**/@grpc/grpc-js": "1.9.1"
   }
 }

components/server/src/authorization/spicedb-authorizer.ts

@@ -21,6 +21,10 @@ export class SpiceDBAuthorizer {
         private readonly clientProvider: SpiceDBClientProvider,
     ) {}
 
+    private get client(): v1.ZedPromiseClientInterface {
+        return this.clientProvider.getClient();
+    }
+
     async check(
         req: v1.CheckPermissionRequest,
         experimentsFields: {
@@ -39,8 +43,7 @@ export class SpiceDBAuthorizer {
         const timer = spicedbClientLatency.startTimer();
         let error: Error | undefined;
         try {
-            const client = this.clientProvider.getClient();
-            const response = await client.checkPermission(req, this.callOptions);
+            const response = await this.client.checkPermission(req, this.callOptions);
             const permitted = response.permissionship === v1.CheckPermissionResponse_Permissionship.HAS_PERMISSION;
 
             return permitted;
@@ -59,8 +62,7 @@ export class SpiceDBAuthorizer {
         const timer = spicedbClientLatency.startTimer();
         let error: Error | undefined;
         try {
-            const client = this.clientProvider.getClient();
-            const response = await client.writeRelationships(
+            const response = await this.client.writeRelationships(
                 v1.WriteRelationshipsRequest.create({
                     updates,
                 }),
@@ -81,11 +83,16 @@ export class SpiceDBAuthorizer {
         const timer = spicedbClientLatency.startTimer();
         let error: Error | undefined;
         try {
-            const client = this.clientProvider.getClient();
-            const existing = await client.readRelationships(v1.ReadRelationshipsRequest.create(req), this.callOptions);
+            const existing = await this.client.readRelationships(
+                v1.ReadRelationshipsRequest.create(req),
+                this.callOptions,
+            );
             if (existing.length > 0) {
-                const response = await client.deleteRelationships(req, this.callOptions);
-                const after = await client.readRelationships(v1.ReadRelationshipsRequest.create(req), this.callOptions);
+                const response = await this.client.deleteRelationships(req, this.callOptions);
+                const after = await this.client.readRelationships(
+                    v1.ReadRelationshipsRequest.create(req),
+                    this.callOptions,
+                );
                 if (after.length > 0) {
                     log.error("[spicedb] Failed to delete relationships.", { existing, after, request: req });
                 }
@@ -108,11 +115,7 @@ export class SpiceDBAuthorizer {
     }
 
     async readRelationships(req: v1.ReadRelationshipsRequest): Promise<v1.ReadRelationshipsResponse[]> {
-        const client = this.clientProvider.getClient();
-        if (!client) {
-            return [];
-        }
-        return client.readRelationships(req, this.callOptions);
+        return this.client.readRelationships(req, this.callOptions);
     }
 
     /**

components/server/src/authorization/spicedb.ts

@@ -5,22 +5,16 @@
  */
 
 import { v1 } from "@authzed/authzed-node";
-import { IClientCallMetrics, createClientCallMetricsInterceptor } from "@gitpod/gitpod-protocol/lib/util/grpc";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
 import * as grpc from "@grpc/grpc-js";
 
-export const SpiceDBClientProvider = Symbol("SpiceDBClientProvider");
-
 export interface SpiceDBClientConfig {
     address: string;
     token: string;
 }
 
 export type SpiceDBClient = v1.ZedPromiseClientInterface;
 type Client = v1.ZedClientInterface & grpc.Client;
-export interface SpiceDBClientProvider {
-    getClient(): SpiceDBClient;
-}
 
 export function spiceDBConfigFromEnv(): SpiceDBClientConfig | undefined {
     const token = process.env["SPICEDB_PRESHARED_KEY"];
@@ -40,63 +34,48 @@ export function spiceDBConfigFromEnv(): SpiceDBClientConfig | undefined {
     };
 }
 
-function spicedbClientFromConfig(config: SpiceDBClientConfig): Client {
-    const clientOptions: grpc.ClientOptions = {
-        "grpc.client_idle_timeout_ms": 10000, // this ensures a connection is not stuck in the "READY" state for too long. Required for the reconnect logic below.
-        "grpc.max_reconnect_backoff_ms": 5000, // default: 12000
-    };
-
-    return v1.NewClient(
-        config.token,
-        config.address,
-        v1.ClientSecurity.INSECURE_PLAINTEXT_CREDENTIALS,
-        undefined,
-        clientOptions,
-    ) as Client;
-}
-
-export class CachingSpiceDBClientProvider implements SpiceDBClientProvider {
+export class SpiceDBClientProvider {
     private client: Client | undefined;
 
-    private readonly interceptors: grpc.Interceptor[] = [];
-
     constructor(
-        private readonly _clientConfig: SpiceDBClientConfig,
-        private readonly clientCallMetrics?: IClientCallMetrics | undefined,
-    ) {
-        if (this.clientCallMetrics) {
-            this.interceptors.push(createClientCallMetricsInterceptor(this.clientCallMetrics));
-        }
-    }
+        private readonly clientConfig: SpiceDBClientConfig,
+        private readonly interceptors: grpc.Interceptor[] = [],
+    ) {}
 
     getClient(): SpiceDBClient {
-        let client = this.client;
-        if (!client) {
-            client = spicedbClientFromConfig(this.clientConfig);
-        } else if (client.getChannel().getConnectivityState(true) !== grpc.connectivityState.READY) {
-            // (gpl): We need this check and explicit re-connect because we observe a ~120s connection timeout without it.
-            // It's not entirely clear where that timeout comes from, but likely from the underlying transport, that is not exposed by grpc/grpc-js
-            client.close();
-
-            log.warn("[spicedb] Lost connection to SpiceDB - reconnecting...");
-
-            client = spicedbClientFromConfig(this.clientConfig);
-        }
-        this.client = client;
-
-        return client.promises;
-    }
-
-    protected get clientConfig() {
-        const config = this._clientConfig;
-        if (this.interceptors) {
-            return {
-                ...config,
-                options: {
-                    interceptors: [...this.interceptors],
+        if (!this.client) {
+            this.client = v1.NewClient(
+                this.clientConfig.token,
+                this.clientConfig.address,
+                v1.ClientSecurity.INSECURE_PLAINTEXT_CREDENTIALS,
+                undefined, //
+                {
+                    // we ping frequently to check if the connection is still alive
+                    "grpc.keepalive_time_ms": 1000,
+                    "grpc.keepalive_timeout_ms": 1000,
+
+                    "grpc.max_reconnect_backoff_ms": 5000,
+                    "grpc.initial_reconnect_backoff_ms": 500,
+                    "grpc.service_config": JSON.stringify({
+                        methodConfig: [
+                            {
+                                name: [{}],
+                                retryPolicy: {
+                                    maxAttempts: 10,
+                                    initialBackoff: "0.1s",
+                                    maxBackoff: "5s",
+                                    backoffMultiplier: 2.0,
+                                    retryableStatusCodes: ["UNAVAILABLE", "DEADLINE_EXCEEDED"],
+                                },
+                            },
+                        ],
+                    }),
+
+                    "grpc.enable_retries": 1, //TODO enabled by default
+                    interceptors: this.interceptors,
                 },
-            };
+            ) as Client;
         }
-        return config;
+        return this.client.promises;
     }
 }

components/server/src/container-module.ts

@@ -11,7 +11,11 @@ import { GitpodFileParser } from "@gitpod/gitpod-protocol/lib/gitpod-file-parser
 import { PrometheusClientCallMetrics } from "@gitpod/gitpod-protocol/lib/messaging/client-call-metrics";
 import { newAnalyticsWriterFromEnv } from "@gitpod/gitpod-protocol/lib/util/analytics";
 import { DebugApp } from "@gitpod/gitpod-protocol/lib/util/debug-app";
-import { IClientCallMetrics, defaultGRPCOptions } from "@gitpod/gitpod-protocol/lib/util/grpc";
+import {
+    IClientCallMetrics,
+    createClientCallMetricsInterceptor,
+    defaultGRPCOptions,
+} from "@gitpod/gitpod-protocol/lib/util/grpc";
 import { prometheusClientMiddleware } from "@gitpod/gitpod-protocol/lib/util/nice-grpc";
 import { IDEServiceClient, IDEServiceDefinition } from "@gitpod/ide-service-api/lib/ide.pb";
 import { ImageBuilderClientCallMetrics, ImageBuilderClientProvider } from "@gitpod/image-builder/lib";
@@ -46,7 +50,7 @@ import { AuthJWT, SignInJWT } from "./auth/jwt";
 import { LoginCompletionHandler } from "./auth/login-completion-handler";
 import { VerificationService } from "./auth/verification-service";
 import { Authorizer, createInitializingAuthorizer } from "./authorization/authorizer";
-import { CachingSpiceDBClientProvider, SpiceDBClientProvider, spiceDBConfigFromEnv } from "./authorization/spicedb";
+import { SpiceDBClientProvider, spiceDBConfigFromEnv } from "./authorization/spicedb";
 import { BillingModes } from "./billing/billing-mode";
 import { EntitlementService, EntitlementServiceImpl } from "./billing/entitlement-service";
 import { EntitlementServiceUBP } from "./billing/entitlement-service-ubp";
@@ -309,7 +313,10 @@ export const productionContainerModule = new ContainerModule(
                     throw new Error("[spicedb] Missing configuration expected in env vars!");
                 }
                 const clientCallMetrics = ctx.container.get<IClientCallMetrics>(IClientCallMetrics);
-                return new CachingSpiceDBClientProvider(config, clientCallMetrics);
+                return new SpiceDBClientProvider(
+                    config, //
+                    [createClientCallMetricsInterceptor(clientCallMetrics)],
+                );
             })
             .inSingletonScope();
         bind(SpiceDBAuthorizer).toSelf().inSingletonScope();

components/server/src/test/service-testing-container-module.ts

@@ -13,7 +13,7 @@ import { v4 } from "uuid";
 import { AuthProviderParams } from "../auth/auth-provider";
 import { HostContextProvider, HostContextProviderFactory } from "../auth/host-context-provider";
 import { HostContextProviderImpl } from "../auth/host-context-provider-impl";
-import { CachingSpiceDBClientProvider, SpiceDBClientProvider } from "../authorization/spicedb";
+import { SpiceDBClientProvider } from "../authorization/spicedb";
 import { Config } from "../config";
 import { StorageClient } from "../storage/storage-client";
 import { testContainer } from "@gitpod/gitpod-db/lib";
@@ -194,7 +194,7 @@ const mockApplyingContainerModule = new ContainerModule((bind, unbound, isbound,
                 token: v4(),
                 address: "localhost:50051",
             };
-            return new CachingSpiceDBClientProvider(config);
+            return new SpiceDBClientProvider(config);
         })
         .inSingletonScope();
 });

components/supervisor-api/typescript-grpc/package.json

@@ -9,7 +9,7 @@
     "lib"
   ],
   "dependencies": {
-    "@grpc/grpc-js": "1.8.8",
+    "@grpc/grpc-js": "1.9.1",
     "google-protobuf": "^3.19.1"
   },
   "devDependencies": {

components/usage-api/typescript/package.json

@@ -18,6 +18,7 @@
     "ts-proto": "^1.153.0"
   },
   "devDependencies": {
+    "@types/long": "4.0.0",
     "grpc-tools": "^1.12.4",
     "typescript": "~4.4.2",
     "typescript-formatter": "^7.2.2"

components/ws-daemon-api/typescript/package.json

@@ -13,7 +13,7 @@
   ],
   "dependencies": {
     "@gitpod/content-service": "0.1.5",
-    "@grpc/grpc-js": "1.8.8",
+    "@grpc/grpc-js": "1.9.1",
     "google-protobuf": "^3.19.1",
     "inversify": "^6.0.1",
     "opentracing": "^0.14.4"

components/ws-manager-api/typescript/package.json

@@ -16,7 +16,7 @@
   "dependencies": {
     "@gitpod/content-service": "0.1.5",
     "@gitpod/gitpod-protocol": "0.1.5",
-    "@grpc/grpc-js": "1.8.8",
+    "@grpc/grpc-js": "1.9.1",
     "google-protobuf": "^3.19.1",
     "inversify": "^6.0.1",
     "opentracing": "^0.14.4"

components/ws-manager-bridge-api/typescript/package.json

@@ -11,7 +11,7 @@
     "lib"
   ],
   "dependencies": {
-    "@grpc/grpc-js": "1.8.8",
+    "@grpc/grpc-js": "1.9.1",
     "google-protobuf": "^3.19.1",
     "inversify": "^6.0.1",
     "opentracing": "^0.14.4"

install/installer/pkg/components/spicedb/deployment.go

@@ -154,13 +154,13 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 											Command: []string{"grpc_health_probe", "-v", fmt.Sprintf("-addr=localhost:%d", ContainerGRPCPort)},
 										},
 									},
-									InitialDelaySeconds: 5,
-									// try again every 10 seconds
-									PeriodSeconds: 10,
-									// fail after 6 * 10 + 5 = 65 seconds
-									FailureThreshold: 6,
+									InitialDelaySeconds: 1,
+									// try again every 2 seconds
+									PeriodSeconds: 2,
+									// fail after 30 * 2 + 1 = 61
+									FailureThreshold: 30,
 									SuccessThreshold: 1,
-									TimeoutSeconds:   3,
+									TimeoutSeconds:   1,
 								},
 								VolumeMounts: []v1.VolumeMount{
 									bootstrapVolumeMount,