[typeorm] increase/monitor connection pool (#18457)

Author: svenefftinge

components/gitpod-db/src/typeorm/typeorm.ts

@@ -46,7 +46,7 @@ export class TypeORM {
             namingStrategy: new DefaultNamingStrategy(),
             extra: {
                 // default is 10 (see https://github.com/mysqljs/mysql#pool-options), which is too low for our use case
-                connectionLimit: 20,
+                connectionLimit: 40,
             },
         };
     }

components/server/src/init.ts

@@ -56,6 +56,8 @@ import { Container } from "inversify";
 import { Server } from "./server";
 import { log, LogrusLogLevel } from "@gitpod/gitpod-protocol/lib/util/logging";
 import { TracingManager } from "@gitpod/gitpod-protocol/lib/util/tracing";
+import { TypeORM } from "@gitpod/gitpod-db/lib";
+import { dbConnectionsFree, dbConnectionsTotal } from "./prometheus-metrics";
 if (process.env.NODE_ENV === "development") {
     require("longjohn");
 }
@@ -76,9 +78,24 @@ export async function start(container: Container) {
         }
     });
 
+    const interval = setInterval(async () => {
+        try {
+            const connection = await container.get(TypeORM).getConnection();
+            const pool: any = (connection.driver as any).pool;
+            const activeConnections = pool._allConnections.length;
+            const freeConnections = pool._freeConnections.length;
+
+            dbConnectionsTotal.set(activeConnections);
+            dbConnectionsFree.set(freeConnections);
+        } catch (error) {
+            log.error("Error updating TypeORM metrics", error);
+        }
+    }, 5000);
+
     process.on("SIGTERM", async () => {
         log.info("SIGTERM received, stopping");
         await server.stop();
+        clearInterval(interval);
     });
 
     const tracing = container.get(TracingManager);

components/server/src/prometheus-metrics.ts

@@ -33,8 +33,20 @@ export function registerServerMetrics(registry: prometheusClient.Registry) {
     registry.registerMetric(redisUpdatesReceived);
     registry.registerMetric(redisUpdatesCompletedTotal);
     registry.registerMetric(updateSubscribersRegistered);
+    registry.registerMetric(dbConnectionsTotal);
+    registry.registerMetric(dbConnectionsFree);
 }
 
+export const dbConnectionsTotal = new prometheusClient.Gauge({
+    name: "gitpod_typeorm_total_connections",
+    help: "Total number of connections in TypeORM pool",
+});
+
+export const dbConnectionsFree = new prometheusClient.Gauge({
+    name: "gitpod_typeorm_free_connections",
+    help: "Number of free connections in TypeORM pool",
+});
+
 const loginCompletedTotal = new prometheusClient.Counter({
     name: "gitpod_login_completed_total",
     help: "Total number of logins completed into gitpod, by status",