Allow creating and monitoring run replication services with different settings (#2055)

* Allow creating and monitoring run replication services with different settings

* Fix test

Author: ericallam

apps/webapp/app/env.server.ts

@@ -771,6 +771,7 @@ const EnvironmentSchema = z.object({
   RUN_REPLICATION_WAIT_FOR_ASYNC_INSERT: z.string().default("0"),
   RUN_REPLICATION_KEEP_ALIVE_ENABLED: z.string().default("1"),
   RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().default(9_000),
+  RUN_REPLICATION_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/routes/admin.api.v1.runs-replication.create.ts

@@ -0,0 +1,122 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { z } from "zod";
+import { ClickHouse } from "@internal/clickhouse";
+import { env } from "~/env.server";
+import { RunsReplicationService } from "~/services/runsReplicationService.server";
+import {
+  getRunsReplicationGlobal,
+  setRunsReplicationGlobal,
+} from "~/services/runsReplicationGlobal.server";
+
+const CreateRunReplicationServiceParams = z.object({
+  name: z.string(),
+  keepAliveEnabled: z.boolean(),
+  keepAliveIdleSocketTtl: z.number(),
+  maxOpenConnections: z.number(),
+  maxFlushConcurrency: z.number(),
+  flushIntervalMs: z.number(),
+  flushBatchSize: z.number(),
+  leaderLockTimeoutMs: z.number(),
+  leaderLockExtendIntervalMs: z.number(),
+  leaderLockAcquireAdditionalTimeMs: z.number(),
+  leaderLockRetryIntervalMs: z.number(),
+  ackIntervalSeconds: z.number(),
+  waitForAsyncInsert: z.boolean(),
+});
+
+type CreateRunReplicationServiceParams = z.infer<typeof CreateRunReplicationServiceParams>;
+
+export async function action({ request }: ActionFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  try {
+    const globalService = getRunsReplicationGlobal();
+
+    if (globalService) {
+      return json(
+        { error: "Global runs replication service already exists. Stop it first." },
+        { status: 400 }
+      );
+    }
+
+    const params = CreateRunReplicationServiceParams.parse(await request.json());
+
+    const service = createRunReplicationService(params);
+
+    setRunsReplicationGlobal(service);
+
+    await service.start();
+
+    return json({
+      success: true,
+    });
+  } catch (error) {
+    return json({ error: error instanceof Error ? error.message : error }, { status: 400 });
+  }
+}
+
+function createRunReplicationService(params: CreateRunReplicationServiceParams) {
+  const clickhouse = new ClickHouse({
+    url: env.RUN_REPLICATION_CLICKHOUSE_URL,
+    name: params.name,
+    keepAlive: {
+      enabled: params.keepAliveEnabled,
+      idleSocketTtl: params.keepAliveIdleSocketTtl,
+    },
+    logLevel: "debug",
+    compression: {
+      request: true,
+    },
+    maxOpenConnections: params.maxOpenConnections,
+  });
+
+  const service = new RunsReplicationService({
+    clickhouse: clickhouse,
+    pgConnectionUrl: env.DATABASE_URL,
+    serviceName: params.name,
+    slotName: env.RUN_REPLICATION_SLOT_NAME,
+    publicationName: env.RUN_REPLICATION_PUBLICATION_NAME,
+    redisOptions: {
+      keyPrefix: "runs-replication:",
+      port: env.RUN_REPLICATION_REDIS_PORT ?? undefined,
+      host: env.RUN_REPLICATION_REDIS_HOST ?? undefined,
+      username: env.RUN_REPLICATION_REDIS_USERNAME ?? undefined,
+      password: env.RUN_REPLICATION_REDIS_PASSWORD ?? undefined,
+      enableAutoPipelining: true,
+      ...(env.RUN_REPLICATION_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }),
+    },
+    maxFlushConcurrency: params.maxFlushConcurrency,
+    flushIntervalMs: params.flushIntervalMs,
+    flushBatchSize: params.flushBatchSize,
+    leaderLockTimeoutMs: params.leaderLockTimeoutMs,
+    leaderLockExtendIntervalMs: params.leaderLockExtendIntervalMs,
+    leaderLockAcquireAdditionalTimeMs: params.leaderLockAcquireAdditionalTimeMs,
+    leaderLockRetryIntervalMs: params.leaderLockRetryIntervalMs,
+    ackIntervalSeconds: params.ackIntervalSeconds,
+    logLevel: "debug",
+    waitForAsyncInsert: params.waitForAsyncInsert,
+  });
+
+  return service;
+}

apps/webapp/app/routes/admin.api.v1.runs-replication.start-monitor.ts

@@ -0,0 +1,59 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { startTcpBufferMonitor } from "~/services/monitorTcpBuffers.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { getTcpMonitorGlobal, setTcpMonitorGlobal } from "~/services/runsReplicationGlobal.server";
+
+const schema = z.object({
+  intervalMs: z.number().min(1000).max(60_000).default(5_000),
+});
+
+export async function action({ request }: ActionFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  try {
+    const body = await request.json();
+    const { intervalMs } = schema.parse(body);
+
+    const globalMonitor = getTcpMonitorGlobal();
+
+    if (globalMonitor) {
+      return json(
+        {
+          error: "Tcp buffer monitor already running, you must stop it before starting a new one",
+        },
+        {
+          status: 400,
+        }
+      );
+    }
+
+    setTcpMonitorGlobal(startTcpBufferMonitor(intervalMs));
+
+    return json({
+      success: true,
+    });
+  } catch (error) {
+    return json({ error: error instanceof Error ? error.message : error }, { status: 400 });
+  }
+}

apps/webapp/app/routes/admin.api.v1.runs-replication.start.ts

@@ -1,6 +1,7 @@
 import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
 import { prisma } from "~/db.server";
 import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { getRunsReplicationGlobal } from "~/services/runsReplicationGlobal.server";
 import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
 
 export async function action({ request }: ActionFunctionArgs) {
@@ -26,7 +27,13 @@ export async function action({ request }: ActionFunctionArgs) {
   }
 
   try {
-    await runsReplicationInstance?.start();
+    const globalService = getRunsReplicationGlobal();
+
+    if (globalService) {
+      await globalService.start();
+    } else {
+      await runsReplicationInstance?.start();
+    }
 
     return json({
       success: true,

apps/webapp/app/routes/admin.api.v1.runs-replication.stop-monitor.ts

@@ -0,0 +1,47 @@
+import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import {
+  getTcpMonitorGlobal,
+  unregisterTcpMonitorGlobal,
+} from "~/services/runsReplicationGlobal.server";
+
+export async function action({ request }: ActionFunctionArgs) {
+  // Next authenticate the request
+  const authenticationResult = await authenticateApiRequestWithPersonalAccessToken(request);
+
+  if (!authenticationResult) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: {
+      id: authenticationResult.userId,
+    },
+  });
+
+  if (!user) {
+    return json({ error: "Invalid or Missing API key" }, { status: 401 });
+  }
+
+  if (!user.admin) {
+    return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+  }
+
+  try {
+    const globalMonitor = getTcpMonitorGlobal();
+
+    if (!globalMonitor) {
+      return json({ error: "Tcp buffer monitor not running" }, { status: 400 });
+    }
+
+    clearInterval(globalMonitor);
+    unregisterTcpMonitorGlobal();
+
+    return json({
+      success: true,
+    });
+  } catch (error) {
+    return json({ error: error instanceof Error ? error.message : error }, { status: 400 });
+  }
+}

apps/webapp/app/routes/admin.api.v1.runs-replication.stop.ts

@@ -1,6 +1,7 @@
 import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
 import { prisma } from "~/db.server";
 import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { getRunsReplicationGlobal } from "~/services/runsReplicationGlobal.server";
 import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
 
 export async function action({ request }: ActionFunctionArgs) {
@@ -26,7 +27,13 @@ export async function action({ request }: ActionFunctionArgs) {
   }
 
   try {
-    await runsReplicationInstance?.stop();
+    const globalService = getRunsReplicationGlobal();
+
+    if (globalService) {
+      await globalService.stop();
+    } else {
+      await runsReplicationInstance?.stop();
+    }
 
     return json({
       success: true,

apps/webapp/app/routes/admin.api.v1.runs-replication.teardown.ts

@@ -1,6 +1,10 @@
 import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
 import { prisma } from "~/db.server";
 import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import {
+  getRunsReplicationGlobal,
+  unregisterRunsReplicationGlobal,
+} from "~/services/runsReplicationGlobal.server";
 import { runsReplicationInstance } from "~/services/runsReplicationInstance.server";
 
 export async function action({ request }: ActionFunctionArgs) {
@@ -26,7 +30,14 @@ export async function action({ request }: ActionFunctionArgs) {
   }
 
   try {
-    await runsReplicationInstance?.teardown();
+    const globalService = getRunsReplicationGlobal();
+
+    if (globalService) {
+      await globalService.teardown();
+      unregisterRunsReplicationGlobal();
+    } else {
+      await runsReplicationInstance?.teardown();
+    }
 
     return json({
       success: true,

apps/webapp/app/services/monitorTcpBuffers.server.ts

@@ -0,0 +1,57 @@
+// monitorTcpBuffers.ts
+import fs from "fs/promises";
+import os from "os";
+import { logger } from "./logger.server";
+
+/**
+ * Parse /proc/net/sockstat and /proc/sys/net/* every `intervalMs`
+ * and log the numbers. You can pivot these logs into CloudWatch
+ * metrics with a filter pattern if you like.
+ */
+export function startTcpBufferMonitor(intervalMs = 5_000) {
+  async function sampleOnce() {
+    try {
+      const [sockstat, wmemMax, tcpMem] = await Promise.all([
+        fs.readFile("/proc/net/sockstat", "utf8"),
+        fs.readFile("/proc/sys/net/core/wmem_max", "utf8"),
+        fs.readFile("/proc/sys/net/ipv4/tcp_mem", "utf8"),
+      ]);
+
+      logger.debug("tcp-buffer-monitor", {
+        sockstat,
+        wmemMax,
+        tcpMem,
+      });
+
+      // /proc/net/sockstat has lines like:
+      // TCP: inuse 5 orphan 0 tw 0 alloc 6 mem 409
+      const tcpLine = sockstat.split("\n").find((l) => l.startsWith("TCP:")) ?? "";
+      const fields = tcpLine.trim().split(/\s+/);
+      const inUse = Number(fields[2]); // open sockets
+      const alloc = Number(fields[8]); // total sockets with buffers
+      const memPages = Number(fields[10]); // pages (4 kB each)
+      const memBytes = memPages * 4096;
+
+      const wmemMaxBytes = Number(wmemMax.trim());
+      const [low, pressure, high] = tcpMem
+        .trim()
+        .split(/\s+/)
+        .map((n) => Number(n) * 4096); // pages → bytes
+
+      logger.debug("tcp-buffer-monitor", {
+        t: Date.now(),
+        host: os.hostname(),
+        sockets_in_use: inUse,
+        sockets_alloc: alloc,
+        tcp_mem_bytes: memBytes,
+        tcp_mem_high: high,
+        wmem_max: wmemMaxBytes,
+      });
+    } catch (err) {
+      // Log and keep going; most errors are “file disappeared for a moment”
+      console.error("tcp-buffer-monitor error", err);
+    }
+  }
+
+  return setInterval(sampleOnce, intervalMs);
+}