[ts] Add metric gitpod_logs_total(level) for TS components

Author: geropl

components/gitpod-protocol/src/util/logging-backend.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import * as prometheusClient from "prom-client";
+import { LogReporter } from "./logging";
+
+const logsCounter = new prometheusClient.Counter({
+    name: "gitpod_logs_total",
+    help: "Total number of logs by level",
+    labelNames: ["level"],
+    registers: [prometheusClient.register],
+});
+
+export function reportLogCount(level: string) {
+    logsCounter.inc({ level });
+}
+
+export function installLogCountReporter() {
+    LogReporter.setReporter((item) => {
+        reportLogCount(item.severity);
+    });
+}

components/gitpod-protocol/src/util/logging.ts

@@ -44,6 +44,14 @@ export namespace LogContext {
     }
 }
 
+let logItemReporter: LogReporter.Reporter | undefined = undefined;
+export namespace LogReporter {
+    export type Reporter = (item: LogItem) => void;
+    export function setReporter(reporter: Reporter): void {
+        logItemReporter = reporter;
+    }
+}
+
 export interface LogPayload {
     // placeholder to indicate that only dictionary-style objects should be passed as payload
 }
@@ -333,7 +341,7 @@ function makeLogItem(
     payloadArgs = payloadArgs.map((arg) => scrubPayload(arg, plainLogging));
     const payload: unknown =
         payloadArgs.length == 0 ? undefined : payloadArgs.length == 1 ? payloadArgs[0] : payloadArgs;
-    const logItem = {
+    const logItem: LogItem = {
         // undefined fields get eliminated in JSON.stringify()
         ...reportedErrorEvent,
         component,
@@ -345,6 +353,11 @@ function makeLogItem(
         payload,
         loggedViaConsole: calledViaConsole ? true : undefined,
     };
+    if (logItemReporter) {
+        try {
+            logItemReporter(logItem);
+        } catch (err) {}
+    }
     if (plainLogging) {
         return `[${logItem.severity}] [${logItem.component}] ${logItem.message}
             ${JSON.stringify(payload || "", undefined, "              ")}
@@ -401,15 +414,17 @@ function makeReportedErrorEvent(error: Error | undefined): {} {
 
 type LogItem = {
     component?: string;
-    severity?: string;
+    severity: string;
     time?: string;
+    context?: LogContext;
     environment?: string;
     region?: string;
     message?: string;
     messageStub?: string;
     errorStub?: string;
     error?: unknown;
     payload?: unknown;
+    loggedViaConsole?: boolean;
 };
 
 function makeLogItemStub(logItem: LogItem): LogItem {

components/server/src/init.ts

@@ -55,6 +55,7 @@ import express from "express";
 import { Container } from "inversify";
 import { Server } from "./server";
 import { log, LogrusLogLevel } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { installLogCountReporter } from "@gitpod/gitpod-protocol/lib/util/logging-backend";
 import { TracingManager } from "@gitpod/gitpod-protocol/lib/util/tracing";
 import { TypeORM } from "@gitpod/gitpod-db/lib";
 import { dbConnectionsEnqueued, dbConnectionsFree, dbConnectionsTotal } from "./prometheus-metrics";
@@ -66,6 +67,7 @@ if (process.env.NODE_ENV === "development") {
 
 log.enableJSONLogging("server", process.env.VERSION, LogrusLogLevel.getFromEnv());
 installCtxLogAugmenter();
+installLogCountReporter();
 
 // eslint-disable-next-line @typescript-eslint/no-floating-promises
 (async () => {

components/ws-manager-bridge/src/main.ts

@@ -8,6 +8,7 @@ import { Container } from "inversify";
 import * as express from "express";
 import * as prometheusClient from "prom-client";
 import { log, LogrusLogLevel } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { installLogCountReporter } from "@gitpod/gitpod-protocol/lib/util/logging-backend";
 import { DebugApp } from "@gitpod/gitpod-protocol/lib/util/debug-app";
 import { TypeORM } from "@gitpod/gitpod-db/lib/typeorm/typeorm";
 import { TracingManager } from "@gitpod/gitpod-protocol/lib/util/tracing";
@@ -17,6 +18,7 @@ import { AppClusterWorkspaceInstancesController } from "./app-cluster-instance-c
 import { redisMetricsRegistry } from "@gitpod/gitpod-db/lib";
 
 log.enableJSONLogging("ws-manager-bridge", undefined, LogrusLogLevel.getFromEnv());
+installLogCountReporter();
 
 export const start = async (container: Container) => {
     process.on("uncaughtException", function (err) {