Faster jobs page (#879)

* Optional next/previous with the list pagination

* Defer the loading of the runs table

* Only return the latest run in a separate query, then do a join in code

* Added a composite index to speed up getting the latest run from a job id

Author: matt-aitken

apps/webapp/app/components/runs/RunsTable.tsx

@@ -77,11 +77,11 @@ export function RunsTable({
       <TableBody>
         {total === 0 && !hasFilters ? (
           <TableBlankRow colSpan={showJob ? 10 : 9}>
-            <NoRuns title="No runs found" />
+            {!isLoading && <NoRuns title="No runs found" />}
           </TableBlankRow>
         ) : runs.length === 0 ? (
           <TableBlankRow colSpan={showJob ? 10 : 9}>
-            <NoRuns title="No runs match your filters" />
+            {!isLoading && <NoRuns title="No runs match your filters" />}
           </TableBlankRow>
         ) : (
           runs.map((run) => {

apps/webapp/app/presenters/JobListPresenter.server.ts

@@ -9,6 +9,7 @@ import { Project } from "~/models/project.server";
 import { User } from "~/models/user.server";
 import { z } from "zod";
 import { projectPath } from "~/utils/pathBuilder";
+import { JobRunStatus } from "@trigger.dev/database";
 
 export type ProjectJob = Awaited<ReturnType<JobListPresenter["call"]>>[0];
 
@@ -43,14 +44,6 @@ export class JobListPresenter {
         id: true,
         slug: true,
         title: true,
-        runs: {
-          select: {
-            createdAt: true,
-            status: true,
-          },
-          take: 1,
-          orderBy: [{ createdAt: "desc" }],
-        },
         integrations: {
           select: {
             key: true,
@@ -105,6 +98,28 @@ export class JobListPresenter {
       orderBy: [{ title: "asc" }],
     });
 
+    const latestRuns = await this.#prismaClient.$queryRaw<
+      {
+        createdAt: Date;
+        status: JobRunStatus;
+        jobId: string;
+        rn: BigInt;
+      }[]
+    >`
+    SELECT * FROM (
+      SELECT 
+          "id", 
+          "createdAt", 
+          "status", 
+          "jobId",
+          ROW_NUMBER() OVER(PARTITION BY "jobId" ORDER BY "createdAt" DESC) as rn
+      FROM 
+          "public"."JobRun" 
+      WHERE 
+          "jobId" IN (${Prisma.join(jobs.map((j) => j.id))})
+  ) t
+  WHERE rn = 1;`;
+
     return jobs
       .flatMap((job) => {
         const version = job.versions.at(0);
@@ -132,6 +147,8 @@ export class JobListPresenter {
           properties = [...properties, ...versionProperties];
         }
 
+        const latestRun = latestRuns.find((r) => r.jobId === job.id);
+
         return [
           {
             id: job.id,
@@ -155,7 +172,7 @@ export class JobListPresenter {
               (i) => i.setupStatus === "MISSING_FIELDS"
             ),
             environment: version.environment,
-            lastRun: job.runs.at(0),
+            lastRun: latestRun,
             properties,
             projectSlug: job.project.slug,
           },

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/ListPagination.tsx

@@ -5,8 +5,8 @@ import { cn } from "~/utils/cn";
 
 type List = {
   pagination: {
-    next: string | undefined;
-    previous: string | undefined;
+    next?: string | undefined;
+    previous?: string | undefined;
   };
 };
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.runs/route.tsx

@@ -1,5 +1,5 @@
-import { useLocation, useNavigate, useNavigation } from "@remix-run/react";
-import { LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { Await, useLoaderData, useLocation, useNavigate, useNavigation } from "@remix-run/react";
+import { LoaderFunctionArgs, defer } from "@remix-run/server-runtime";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { LinkButton } from "~/components/primitives/Buttons";
@@ -20,6 +20,8 @@ import { ProjectParamSchema, docsPath, projectPath } from "~/utils/pathBuilder";
 import { ListPagination } from "../_app.orgs.$organizationSlug.projects.$projectParam.jobs.$jobParam._index/ListPagination";
 import { RunListSearchSchema } from "~/components/runs/RunStatuses";
 import { RunsFilters } from "~/components/runs/RunFilters";
+import { Suspense } from "react";
+import { Spinner } from "~/components/primitives/Spinner";
 
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
@@ -31,7 +33,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
   const presenter = new RunListPresenter();
 
-  const list = await presenter.call({
+  const list = presenter.call({
     userId,
     filterEnvironment: searchParams.environment,
     filterStatus: searchParams.status,
@@ -44,13 +46,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     to: searchParams.to,
   });
 
-  return typedjson({
+  return defer({
     list,
   });
 };
 
 export default function Page() {
-  const { list } = useTypedLoaderData<typeof loader>();
+  const { list } = useLoaderData<typeof loader>();
   const navigation = useNavigation();
   const isLoading = navigation.state !== "idle";
   const organization = useOrganization();
@@ -79,18 +81,49 @@ export default function Page() {
         <div className="h-full overflow-y-auto p-4 scrollbar-thin scrollbar-track-transparent scrollbar-thumb-slate-700">
           <div className="mb-2 flex items-center justify-between gap-x-2">
             <RunsFilters />
-            <ListPagination list={list} />
+            <Suspense fallback={<></>}>
+              <Await resolve={list}>{(data) => <ListPagination list={data} />}</Await>
+            </Suspense>
           </div>
-          <RunsTable
-            total={list.runs.length}
-            hasFilters={false}
-            showJob={true}
-            runs={list.runs}
-            isLoading={isLoading}
-            runsParentPath={projectPath(organization, project)}
-            currentUser={user}
-          />
-          <ListPagination list={list} className="mt-2 justify-end" />
+          <Suspense
+            fallback={
+              <RunsTable
+                total={0}
+                hasFilters={false}
+                showJob={true}
+                runs={[]}
+                isLoading={true}
+                runsParentPath={projectPath(organization, project)}
+                currentUser={user}
+              />
+            }
+          >
+            <Await resolve={list}>
+              {(data) => {
+                const runs = data.runs.map((run) => ({
+                  ...run,
+                  startedAt: run.startedAt ? new Date(run.startedAt) : null,
+                  completedAt: run.completedAt ? new Date(run.completedAt) : null,
+                  createdAt: new Date(run.createdAt),
+                }));
+
+                return (
+                  <>
+                    <RunsTable
+                      total={data.runs.length}
+                      hasFilters={false}
+                      showJob={true}
+                      runs={runs}
+                      isLoading={isLoading}
+                      runsParentPath={projectPath(organization, project)}
+                      currentUser={user}
+                    />
+                    <ListPagination list={data} className="mt-2 justify-end" />
+                  </>
+                );
+              }}
+            </Await>
+          </Suspense>
         </div>
       </PageBody>
     </PageContainer>

packages/database/prisma/migrations/20240130165343_add_composite_index_to_job_run_for_job_id_and_created_at/migration.sql

@@ -0,0 +1 @@
+CREATE INDEX idx_jobrun_jobId_createdAt ON "public"."JobRun" ("jobId", "createdAt" DESC);