Keep cross check with build imports to reduce bundle dependencies number

Author: codenem

packages/cli-v3/e2e/compile.ts

@@ -2,7 +2,7 @@ import { esbuildDecorators } from "@anatine/esbuild-decorators";
 import { build } from "esbuild";
 import { readFileSync } from "node:fs";
 import { writeFile } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { join, posix, relative, resolve, sep } from "node:path";
 import invariant from "tiny-invariant";
 
 import {
@@ -163,6 +163,27 @@ export async function compile(options: CompileOptions) {
 
   logger.debug(`Writing compiled files to ${tempDir}`);
 
+  // Get the metaOutput for the result build
+  const pathsToProjectDir = relative(
+    join(process.cwd(), "e2e", "fixtures"),
+    config.projectDir
+  ).split(sep);
+
+  const metaOutput =
+    result.metafile!.outputs[
+      posix.join("e2e", "fixtures", ...pathsToProjectDir, "out", "stdin.js")
+    ];
+
+  invariant(metaOutput, "Meta output for the result build is missing");
+
+  // Get the metaOutput for the entryPoint build
+  const entryPointMetaOutput =
+    entryPointResult.metafile!.outputs[
+      posix.join("e2e", "fixtures", ...pathsToProjectDir, "out", "stdin.js")
+    ];
+
+  invariant(entryPointMetaOutput, "Meta output for the entryPoint build is missing");
+
   // Get the outputFile and the sourceMapFile for the result build
   const workerOutputFile = result.outputFiles.find(
     (file) => file.path === join(config.projectDir, "out", "stdin.js")
@@ -195,6 +216,8 @@ export async function compile(options: CompileOptions) {
   await writeFile(join(tempDir, "index.js"), entryPointOutputFile.text);
 
   return {
+    entryPointMetaOutput,
+    workerMetaOutput: metaOutput,
     directDependenciesMeta,
     workerOutputFile,
     entryPointOutputFile,

packages/cli-v3/e2e/handleDependencies.ts

@@ -16,8 +16,11 @@ import { logger } from "../src/utilities/logger.js";
 import { cliLink } from "../src/utilities/cliOutput.js";
 import { E2EJavascriptProject } from "./javascriptProject.js";
 import { DependencyMeta } from "../src/utilities/javascriptProject.js";
+import { Metafile } from "esbuild";
 
 type HandleDependenciesOptions = {
+  entryPointMetaOutput: Metafile["outputs"]["out/stdin.js"];
+  metaOutput: Metafile["outputs"]["out/stdin.js"];
   directDependenciesMeta: Record<string, DependencyMeta>;
   packageManager: PackageManager;
   resolvedConfig: ReadConfigResult;
@@ -29,15 +32,29 @@ export async function handleDependencies(options: HandleDependenciesOptions) {
     throw new Error("cannot resolve config");
   }
   const {
+    entryPointMetaOutput,
+    metaOutput,
     directDependenciesMeta,
     packageManager,
     resolvedConfig: { config },
     tempDir,
   } = options;
 
+  logger.debug("Getting the imports for the worker and entryPoint builds", {
+    workerImports: metaOutput.imports,
+    entryPointImports: entryPointMetaOutput.imports,
+  });
+
+  // Get all the required dependencies from the metaOutputs and save them to /tmp/dir/package.json
+  const allImports = [...metaOutput.imports, ...entryPointMetaOutput.imports];
+
   const javascriptProject = new E2EJavascriptProject(config.projectDir, packageManager);
 
-  const dependencies = await resolveRequiredDependencies(directDependenciesMeta, config);
+  const dependencies = await resolveRequiredDependencies(
+    directDependenciesMeta,
+    allImports,
+    config
+  );
 
   logger.debug("gatherRequiredDependencies()", { dependencies });
 

packages/cli-v3/e2e/index.test.ts

@@ -18,7 +18,7 @@ import { createDeployHash } from "./createDeployHash";
 import { handleDependencies } from "./handleDependencies";
 import { E2EOptions, E2EOptionsSchema } from "./schemas";
 import { fixturesConfig, TestCase } from "./fixtures.config";
-import { OutputFile } from "esbuild";
+import { Metafile, OutputFile } from "esbuild";
 import { findUpMultiple } from "find-up";
 import { DependencyMeta } from "../src/utilities/javascriptProject";
 
@@ -164,8 +164,10 @@ if (testCases.length > 0) {
             ).resolves.not.toThrowError();
           }
 
+          let entryPointMetaOutput: Metafile["outputs"]["out/stdin.js"];
           let directDependenciesMeta: Record<string, DependencyMeta>;
           let entryPointOutputFile: OutputFile;
+          let workerMetaOutput: Metafile["outputs"]["out/stdin.js"];
           let workerOutputFile: OutputFile;
 
           const compileExpect = expect(
@@ -175,8 +177,10 @@ if (testCases.length > 0) {
                 resolvedConfig: resolvedConfig!,
                 tempDir,
               });
+              entryPointMetaOutput = compilationResult.entryPointMetaOutput;
               directDependenciesMeta = compilationResult.directDependenciesMeta;
               entryPointOutputFile = compilationResult.entryPointOutputFile;
+              workerMetaOutput = compilationResult.workerMetaOutput;
               workerOutputFile = compilationResult.workerOutputFile;
             })(),
             wantCompilationError ? "does not compile" : "compiles"
@@ -200,6 +204,8 @@ if (testCases.length > 0) {
           const depsExpectation = expect(
             (async () => {
               dependencies = await handleDependencies({
+                entryPointMetaOutput: entryPointMetaOutput!,
+                metaOutput: workerMetaOutput!,
                 directDependenciesMeta: directDependenciesMeta!,
                 resolvedConfig: resolvedConfig!,
                 tempDir,

packages/cli-v3/src/commands/deploy.ts

@@ -9,7 +9,7 @@ import {
 } from "@trigger.dev/core/v3";
 import { recordSpanException } from "@trigger.dev/core/v3/workers";
 import { Command, Option as CommandOption } from "commander";
-import { build } from "esbuild";
+import { build, Metafile } from "esbuild";
 import { execa } from "execa";
 import { createHash } from "node:crypto";
 import { readFileSync } from "node:fs";
@@ -32,7 +32,11 @@ import {
 import { ReadConfigResult, readConfig } from "../utilities/configFiles.js";
 import { createTempDir, writeJSONFile } from "../utilities/fileSystem";
 import { printStandloneInitialBanner } from "../utilities/initialBanner.js";
-import { parsePackageName, stripWorkspaceFromVersion } from "../utilities/installPackages";
+import {
+  detectPackageNameFromImportPath,
+  parsePackageName,
+  stripWorkspaceFromVersion,
+} from "../utilities/installPackages";
 import { logger } from "../utilities/logger.js";
 import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
 import { login } from "./login";
@@ -1286,6 +1290,17 @@ async function compileProject(
 
       logger.debug(`Writing compiled files to ${tempDir}`);
 
+      // Get the metaOutput for the result build
+      const metaOutput = result.metafile!.outputs[posix.join("out", "stdin.js")];
+
+      invariant(metaOutput, "Meta output for the result build is missing");
+
+      // Get the metaOutput for the entryPoint build
+      const entryPointMetaOutput =
+        entryPointResult.metafile!.outputs[posix.join("out", "stdin.js")];
+
+      invariant(entryPointMetaOutput, "Meta output for the entryPoint build is missing");
+
       // Get the outputFile and the sourceMapFile for the result build
       const workerOutputFile = result.outputFiles.find(
         (file) => file.path === join(config.projectDir, "out", "stdin.js")
@@ -1317,7 +1332,19 @@ async function compileProject(
       // Save the entryPoint outputFile to /tmp/dir/index.js
       await writeFile(join(tempDir, "index.js"), entryPointOutputFile.text);
 
-      const dependencies = await resolveRequiredDependencies(directDependenciesMeta, config);
+      logger.debug("Getting the imports for the worker and entryPoint builds", {
+        workerImports: metaOutput.imports,
+        entryPointImports: entryPointMetaOutput.imports,
+      });
+
+      // Get all the required dependencies from the metaOutputs and save them to /tmp/dir/package.json
+      const allImports = [...metaOutput.imports, ...entryPointMetaOutput.imports];
+
+      const dependencies = await resolveRequiredDependencies(
+        directDependenciesMeta,
+        allImports,
+        config
+      );
 
       logger.debug("gatherRequiredDependencies()", { dependencies });
 
@@ -1688,41 +1715,54 @@ export async function typecheckProject(config: ResolvedConfig) {
 // Returns the dependency names and the version to use (taken from the CLI deps package.json)
 export async function resolveRequiredDependencies(
   directDependenciesMeta: Record<string, DependencyMeta>,
+  imports: Metafile["outputs"][string]["imports"],
   config: ResolvedConfig
 ) {
   return await tracer.startActiveSpan("resolveRequiredDependencies", async (span) => {
-    const externalDirectDependenciesVersion = Object.fromEntries(
-      Object.entries(directDependenciesMeta)
-        .filter(([, { external }]) => external)
-        .map(([packageName, { version }]) => [packageName, version])
-    );
-    span.setAttribute("resolvablePackageNames", Object.keys(externalDirectDependenciesVersion));
+    const dependencies: Record<string, string> = {};
+    const missingPackages: string[] = [];
 
-    const missingPackages = Object.entries(externalDirectDependenciesVersion)
-      .filter(([, version]) => !version)
-      .map(([name]) => name);
+    for (const file of imports) {
+      if ((file.kind !== "require-call" && file.kind !== "dynamic-import") || !file.external) {
+        continue;
+      }
 
-    span.setAttributes({
-      ...flattenAttributes(externalDirectDependenciesVersion, "resolvedPackageVersions"),
-    });
-    span.setAttribute("missingPackages", missingPackages);
+      const packageName = detectPackageNameFromImportPath(file.path);
 
-    const dependencies: Record<string, string> = {};
+      if (!packageName) {
+        continue;
+      }
 
-    for (const missingPackage of missingPackages) {
-      const internalDependencyVersion =
-        (packageJson.dependencies as Record<string, string>)[missingPackage] ??
-        detectDependencyVersion(missingPackage);
+      if (!directDependenciesMeta[packageName]) {
+        continue;
+      }
 
-      if (internalDependencyVersion) {
-        dependencies[missingPackage] = stripWorkspaceFromVersion(internalDependencyVersion);
+      if (!directDependenciesMeta[packageName].external) {
+        continue;
+      }
+
+      if (!directDependenciesMeta[packageName].version) {
+        missingPackages.push(packageName);
+        const internalDependencyVersion =
+          (packageJson.dependencies as Record<string, string>)[packageName] ??
+          detectDependencyVersion(packageName);
+
+        if (internalDependencyVersion) {
+          dependencies[packageName] = stripWorkspaceFromVersion(internalDependencyVersion);
+        }
+
+        continue;
       }
-    }
 
-    for (const [packageName, version] of Object.entries(externalDirectDependenciesVersion)) {
-      dependencies[packageName] = version;
+      dependencies[packageName] = directDependenciesMeta[packageName].version;
     }
 
+    span.setAttribute("resolvablePackageNames", Object.keys(dependencies));
+    span.setAttributes({
+      ...flattenAttributes(dependencies, "resolvedPackageVersions"),
+    });
+    span.setAttribute("missingPackages", missingPackages);
+
     if (config.additionalPackages) {
       span.setAttribute("additionalPackages", config.additionalPackages);
 
@@ -1737,10 +1777,10 @@ export async function resolveRequiredDependencies(
           dependencies[packageParts.name] = packageParts.version;
           continue;
         } else {
-          const externalDependencyVersion = externalDirectDependenciesVersion[packageParts.name];
+          const dependencyVersion = dependencies[packageParts.name];
 
-          if (externalDependencyVersion) {
-            dependencies[packageParts.name] = externalDependencyVersion;
+          if (dependencyVersion) {
+            dependencies[packageParts.name] = dependencyVersion;
             continue;
           } else {
             logger.log(