Improve trace view performance

Author: ericallam

apps/webapp/app/components/primitives/TreeView/TreeView.tsx

@@ -528,7 +528,7 @@ export type Tree<TData> = {
 /** A tree but flattened so it can easily be used for DOM elements */
 export type FlatTreeItem<TData> = {
   id: string;
-  parentId: string | undefined;
+  parentId?: string | undefined;
   children: string[];
   hasChildren: boolean;
   /** The indentation level, the root is 0 */

apps/webapp/app/entry.server.tsx

@@ -186,4 +186,9 @@ export { apiRateLimiter } from "./services/apiRateLimit.server";
 export { socketIo } from "./v3/handleSocketIo.server";
 export { wss } from "./v3/handleWebsockets.server";
 export { registryProxy } from "./v3/registryProxy.server";
-export { eventLoopMonitor } from "./eventLoopMonitor.server";
+import { eventLoopMonitor } from "./eventLoopMonitor.server";
+import { env } from "./env.server";
+
+if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
+  eventLoopMonitor.enable();
+}

apps/webapp/app/env.server.ts

@@ -204,6 +204,7 @@ const EnvironmentSchema = z.object({
 
   USAGE_OPEN_METER_API_KEY: z.string().optional(),
   USAGE_OPEN_METER_BASE_URL: z.string().optional(),
+  EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/eventLoopMonitor.server.ts

@@ -1,38 +1,77 @@
 import { createHook } from "node:async_hooks";
-import { logger } from "./services/logger.server";
 import { singleton } from "./utils/singleton";
+import { tracer } from "./v3/tracer.server";
 
 const THRESHOLD_NS = 1e8; // 100ms
 
-const cache = new Map<number, [number, number]>();
+const cache = new Map<number, { type: string; start?: [number, number] }>();
+
+function init(asyncId: number, type: string, triggerAsyncId: number, resource: any) {
+  cache.set(asyncId, {
+    type,
+  });
+}
+
+function destroy(asyncId: number) {
+  cache.delete(asyncId);
+}
 
 function before(asyncId: number) {
-  cache.set(asyncId, process.hrtime());
+  const cached = cache.get(asyncId);
+
+  if (!cached) {
+    return;
+  }
+
+  cache.set(asyncId, {
+    ...cached,
+    start: process.hrtime(),
+  });
 }
 
 function after(asyncId: number) {
   const cached = cache.get(asyncId);
-  if (cached == null) {
+
+  if (!cached) {
     return;
   }
+
   cache.delete(asyncId);
 
-  const diff = process.hrtime(cached);
+  if (!cached.start) {
+    return;
+  }
+
+  const diff = process.hrtime(cached.start);
   const diffNs = diff[0] * 1e9 + diff[1];
   if (diffNs > THRESHOLD_NS) {
     const time = diffNs / 1e6; // in ms
 
-    logger.error(`Event loop was blocked for ${time}ms`, {
-      label: "EventLoopMonitor",
+    const newSpan = tracer.startSpan("event-loop-blocked", {
       startTime: new Date(new Date().getTime() - time),
-      durationMs: time,
+      attributes: {
+        asyncType: cached.type,
+        label: "EventLoopMonitor",
+      },
     });
+
+    newSpan.end();
   }
 }
 
 export const eventLoopMonitor = singleton("eventLoopMonitor", () => {
-  // console.log("🥸 Initializing event loop monitor");
-  // const asyncHook = createHook({ before, after });
-  // asyncHook.enable();
-  // return asyncHook;
+  const hook = createHook({ init, before, after, destroy });
+
+  return {
+    enable: () => {
+      console.log("🥸  Initializing event loop monitor");
+
+      hook.enable();
+    },
+    disable: () => {
+      console.log("🥸  Disabling event loop monitor");
+
+      hook.disable();
+    },
+  };
 });

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam/route.tsx

@@ -6,8 +6,8 @@ import {
   MagnifyingGlassPlusIcon,
 } from "@heroicons/react/20/solid";
 import type { Location } from "@remix-run/react";
-import { useParams, useRevalidator } from "@remix-run/react";
-import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { useLoaderData, useParams, useRevalidator } from "@remix-run/react";
+import { LoaderFunctionArgs, SerializeFrom } from "@remix-run/server-runtime";
 import { Virtualizer } from "@tanstack/react-virtual";
 import {
   formatDurationMilliseconds,
@@ -18,10 +18,10 @@ import { RuntimeEnvironmentType } from "@trigger.dev/database";
 import { motion } from "framer-motion";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
-import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ShowParentIcon, ShowParentIconSelected } from "~/assets/icons/ShowParentIcon";
 import tileBgPath from "~/assets/images/[email protected]";
 import { BlankstateInstructions } from "~/components/BlankstateInstructions";
+import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { InlineCode } from "~/components/code/InlineCode";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { MainCenteredContainer, PageBody } from "~/components/layout/AppLayout";
@@ -33,6 +33,7 @@ import { Input } from "~/components/primitives/Input";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
 import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover";
+import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import {
   ResizableHandle,
   ResizablePanel,
@@ -55,7 +56,7 @@ import { useProject } from "~/hooks/useProject";
 import { useReplaceLocation } from "~/hooks/useReplaceLocation";
 import { Shortcut, useShortcutKeys } from "~/hooks/useShortcutKeys";
 import { useUser } from "~/hooks/useUser";
-import { RunEvent, RunPresenter } from "~/presenters/v3/RunPresenter.server";
+import { RunPresenter } from "~/presenters/v3/RunPresenter.server";
 import { getResizableRunSettings, setResizableRunSettings } from "~/services/resizablePanel";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
@@ -68,12 +69,12 @@ import {
   v3RunsPath,
 } from "~/utils/pathBuilder";
 import { SpanView } from "../resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route";
-import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
-import { Property, PropertyTable } from "~/components/primitives/PropertyTable";
 import { SimpleTooltip } from "~/components/primitives/Tooltip";
 
 const MAX_LIVE_RELOADING_EVENTS = 500;
 
+type TraceEvent = NonNullable<SerializeFrom<typeof loader>["trace"]>["events"][0];
+
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
   const { projectParam, organizationSlug, runParam } = v3RunParamsSchema.parse(params);
@@ -89,10 +90,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   //resizable settings
   const resizeSettings = await getResizableRunSettings(request);
 
-  return typedjson({
-    ...result,
+  return {
+    run: result.run,
+    trace: result.trace,
     resizeSettings,
-  });
+  };
 };
 
 function getSpanId(location: Location<any>): string | undefined {
@@ -101,7 +103,7 @@ function getSpanId(location: Location<any>): string | undefined {
 }
 
 export default function Page() {
-  const { run, trace, resizeSettings } = useTypedLoaderData<typeof loader>();
+  const { run, trace, resizeSettings } = useLoaderData<typeof loader>();
   const organization = useOrganization();
   const project = useProject();
   const user = useUser();
@@ -258,7 +260,7 @@ export default function Page() {
                 }}
                 totalDuration={duration}
                 rootSpanStatus={rootSpanStatus}
-                rootStartedAt={rootStartedAt}
+                rootStartedAt={rootStartedAt ? new Date(rootStartedAt) : undefined}
                 environmentType={run.environment.type}
                 shouldLiveReload={shouldLiveReload}
               />
@@ -281,7 +283,7 @@ export default function Page() {
 }
 
 type TasksTreeViewProps = {
-  events: RunEvent[];
+  events: TraceEvent[];
   selectedId?: string;
   parentRunFriendlyId?: string;
   onSelectedIdChanged: (selectedId: string | undefined) => void;
@@ -762,7 +764,7 @@ function TimelineView({
   );
 }
 
-function NodeText({ node }: { node: RunEvent }) {
+function NodeText({ node }: { node: TraceEvent }) {
   const className = "truncate";
   return (
     <Paragraph variant="small" className={cn(className)}>
@@ -771,7 +773,7 @@ function NodeText({ node }: { node: RunEvent }) {
   );
 }
 
-function NodeStatusIcon({ node }: { node: RunEvent }) {
+function NodeStatusIcon({ node }: { node: TraceEvent }) {
   if (node.data.level !== "TRACE") return null;
   if (node.data.style.variant !== "primary") return null;
 
@@ -896,7 +898,7 @@ function SpanWithDuration({
   showDuration,
   node,
   ...props
-}: Timeline.SpanProps & { node: RunEvent; showDuration: boolean }) {
+}: Timeline.SpanProps & { node: TraceEvent; showDuration: boolean }) {
   return (
     <Timeline.Span {...props}>
       <motion.div