[fga] retry write relationships (#18683)

Author: svenefftinge

components/server/src/authorization/spicedb-authorizer.ts

@@ -55,23 +55,40 @@ export class SpiceDBAuthorizer {
     }
 
     async writeRelationships(...updates: v1.RelationshipUpdate[]): Promise<v1.WriteRelationshipsResponse | undefined> {
-        const timer = spicedbClientLatency.startTimer();
-        let error: Error | undefined;
-        try {
-            const response = await this.client.writeRelationships(
-                v1.WriteRelationshipsRequest.create({
-                    updates,
-                }),
-                this.callOptions,
-            );
-            log.info("[spicedb] Successfully wrote relationships.", { response, updates });
+        let tries = 0;
+        // we do sometimes see INTERNAL errors from SpiceDB, so we retry a few times
+        // last time we checked it was 15 times per day (check logs)
+        while (tries++ < 3) {
+            const timer = spicedbClientLatency.startTimer();
+            let error: Error | undefined;
+            try {
+                const response = await this.client.writeRelationships(
+                    v1.WriteRelationshipsRequest.create({
+                        updates,
+                    }),
+                    this.callOptions,
+                );
+                log.info("[spicedb] Successfully wrote relationships.", { response, updates, tries });
 
-            return response;
-        } catch (err) {
-            error = err;
-            log.error("[spicedb] Failed to write relationships.", err, { updates: new TrustedValue(updates) });
-        } finally {
-            observeSpicedbClientLatency("write", error, timer());
+                return response;
+            } catch (err) {
+                error = err;
+                if (err.code === grpc.status.INTERNAL && tries < 3) {
+                    log.warn("[spicedb] Failed to write relationships.", err, {
+                        updates: new TrustedValue(updates),
+                        tries,
+                    });
+                } else {
+                    log.error("[spicedb] Failed to write relationships.", err, {
+                        updates: new TrustedValue(updates),
+                        tries,
+                    });
+                    // we don't try again on other errors
+                    return;
+                }
+            } finally {
+                observeSpicedbClientLatency("write", error, timer());
+            }
         }
     }
 

components/server/src/orgs/usage-service.ts

@@ -17,6 +17,7 @@ import { inject, injectable } from "inversify";
 import { Authorizer } from "../authorization/authorizer";
 import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error";
 import { CostCenterJSON, ListUsageRequest, ListUsageResponse } from "@gitpod/gitpod-protocol/lib/usage";
+import { TrustedValue } from "@gitpod/gitpod-protocol/lib/util/scrubbing";
 
 @injectable()
 export class UsageService {
@@ -149,19 +150,19 @@ export class UsageService {
         const currentInvoiceCredits = creditBalance.usedCredits;
         const usageLimit = creditBalance.usageLimit;
         if (currentInvoiceCredits >= usageLimit) {
-            log.info({ userId }, "Usage limit reached", {
+            log.info({ userId, organizationId }, "Usage limit reached", {
                 attributionId,
-                currentInvoiceCredits,
+                currentInvoiceCredits: new TrustedValue(currentInvoiceCredits),
                 usageLimit,
             });
             return {
                 reached: true,
                 attributionId,
             };
         } else if (currentInvoiceCredits > usageLimit * 0.8) {
-            log.info({ userId }, "Usage limit almost reached", {
+            log.info({ userId, organizationId }, "Usage limit almost reached", {
                 attributionId,
-                currentInvoiceCredits,
+                currentInvoiceCredits: new TrustedValue(currentInvoiceCredits),
                 usageLimit,
             });
             return {