[ts] Add metric gitpod_logs_total(level) for TS components

Author: geropl

components/gitpod-protocol/src/util/logging-node.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import * as prometheusClient from "prom-client";
+import { LogHook } from "./logging";
+
+const logsCounter = new prometheusClient.Counter({
+    name: "gitpod_logs_total",
+    help: "Total number of logs by level",
+    labelNames: ["level"],
+    registers: [prometheusClient.register],
+});
+
+export function reportLogCount(level: string) {
+    logsCounter.inc({ level });
+}
+
+export function installLogCountMetric() {
+    LogHook.setHook((item) => {
+        reportLogCount((item.severity || "").toLowerCase());
+    });
+}

components/gitpod-protocol/src/util/logging.ts

@@ -5,7 +5,6 @@
  */
 
 import { scrubber } from "./scrubbing";
-import * as prometheusClient from "prom-client";
 
 const inspect: (object: unknown) => string = require("util").inspect; // undefined in frontend
 
@@ -45,6 +44,14 @@ export namespace LogContext {
     }
 }
 
+let logItemHook: LogHook.Hook | undefined = undefined;
+export namespace LogHook {
+    export type Hook = (item: LogItem) => void;
+    export function setHook(hook: Hook): void {
+        logItemHook = hook;
+    }
+}
+
 export interface LogPayload {
     // placeholder to indicate that only dictionary-style objects should be passed as payload
 }
@@ -248,16 +255,7 @@ namespace GoogleLogSeverity {
     };
 }
 
-const logsCounter = new prometheusClient.Counter({
-    name: "gitpod_logs_total",
-    help: "Total number of logs by level",
-    labelNames: ["level"],
-    registers: [prometheusClient.register],
-});
-
 function doLog(calledViaConsole: boolean, consoleLog: ConsoleLog, severity: GoogleLogSeverity, args: unknown[]): void {
-    logsCounter.labels(severity).inc();
-
     if (!jsonLogging) {
         // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
         consoleLog(...args);
@@ -343,7 +341,7 @@ function makeLogItem(
     payloadArgs = payloadArgs.map((arg) => scrubPayload(arg, plainLogging));
     const payload: unknown =
         payloadArgs.length == 0 ? undefined : payloadArgs.length == 1 ? payloadArgs[0] : payloadArgs;
-    const logItem = {
+    const logItem: LogItem = {
         // undefined fields get eliminated in JSON.stringify()
         ...reportedErrorEvent,
         component,
@@ -355,6 +353,11 @@ function makeLogItem(
         payload,
         loggedViaConsole: calledViaConsole ? true : undefined,
     };
+    if (logItemHook) {
+        try {
+            logItemHook(logItem);
+        } catch (err) {}
+    }
     if (plainLogging) {
         return `[${logItem.severity}] [${logItem.component}] ${logItem.message}
             ${JSON.stringify(payload || "", undefined, "              ")}
@@ -411,15 +414,17 @@ function makeReportedErrorEvent(error: Error | undefined): {} {
 
 type LogItem = {
     component?: string;
-    severity?: string;
+    severity: string;
     time?: string;
+    context?: LogContext;
     environment?: string;
     region?: string;
     message?: string;
     messageStub?: string;
     errorStub?: string;
     error?: unknown;
     payload?: unknown;
+    loggedViaConsole?: boolean;
 };
 
 function makeLogItemStub(logItem: LogItem): LogItem {

components/server/src/init.ts

@@ -55,6 +55,7 @@ import express from "express";
 import { Container } from "inversify";
 import { Server } from "./server";
 import { log, LogrusLogLevel } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { installLogCountMetric } from "@gitpod/gitpod-protocol/lib/util/logging-node";
 import { TracingManager } from "@gitpod/gitpod-protocol/lib/util/tracing";
 import { TypeORM } from "@gitpod/gitpod-db/lib";
 import { dbConnectionsEnqueued, dbConnectionsFree, dbConnectionsTotal } from "./prometheus-metrics";
@@ -66,6 +67,7 @@ if (process.env.NODE_ENV === "development") {
 
 log.enableJSONLogging("server", process.env.VERSION, LogrusLogLevel.getFromEnv());
 installCtxLogAugmenter();
+installLogCountMetric();
 
 // eslint-disable-next-line @typescript-eslint/no-floating-promises
 (async () => {

components/ws-manager-bridge/src/main.ts

@@ -8,6 +8,7 @@ import { Container } from "inversify";
 import * as express from "express";
 import * as prometheusClient from "prom-client";
 import { log, LogrusLogLevel } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { installLogCountMetric } from "@gitpod/gitpod-protocol/lib/util/logging-node";
 import { DebugApp } from "@gitpod/gitpod-protocol/lib/util/debug-app";
 import { TypeORM } from "@gitpod/gitpod-db/lib/typeorm/typeorm";
 import { TracingManager } from "@gitpod/gitpod-protocol/lib/util/tracing";
@@ -17,6 +18,7 @@ import { AppClusterWorkspaceInstancesController } from "./app-cluster-instance-c
 import { redisMetricsRegistry } from "@gitpod/gitpod-db/lib";
 
 log.enableJSONLogging("ws-manager-bridge", undefined, LogrusLogLevel.getFromEnv());
+installLogCountMetric();
 
 export const start = async (container: Container) => {
     process.on("uncaughtException", function (err) {