Use locked package versions when resolving dependencies in deployed workers

Author: ericallam

.changeset/tender-moose-tell.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Use locked package versions when resolving dependencies in deployed workers

packages/cli-v3/src/commands/deploy.ts

@@ -1283,6 +1283,8 @@ async function compileProject(
 
       const dependencies = await gatherRequiredDependencies(allImports, config, javascriptProject);
 
+      logger.debug("gatherRequiredDependencies()", { dependencies });
+
       const packageJsonContents = {
         name: "trigger-worker",
         version: "0.0.0",
@@ -1547,6 +1549,7 @@ async function gatherRequiredDependencies(
   project: JavascriptProject
 ) {
   const dependencies: Record<string, string> = {};
+  const resolvablePackageNames = new Set<string>();
 
   for (const file of imports) {
     if ((file.kind !== "require-call" && file.kind !== "dynamic-import") || !file.external) {
@@ -1555,26 +1558,32 @@ async function gatherRequiredDependencies(
 
     const packageName = detectPackageNameFromImportPath(file.path);
 
-    if (dependencies[packageName]) {
+    if (!packageName) {
       continue;
     }
 
-    const externalDependencyVersion = await project.resolve(packageName);
+    resolvablePackageNames.add(packageName);
+  }
 
-    if (externalDependencyVersion) {
-      dependencies[packageName] = stripWorkspaceFromVersion(externalDependencyVersion);
-      continue;
-    }
+  const resolvedPackageVersions = await project.resolveAll(Array.from(resolvablePackageNames));
+  const missingPackages = Array.from(resolvablePackageNames).filter(
+    (packageName) => !resolvedPackageVersions[packageName]
+  );
 
+  for (const missingPackage of missingPackages) {
     const internalDependencyVersion =
-      (packageJson.dependencies as Record<string, string>)[packageName] ??
-      detectDependencyVersion(packageName);
+      (packageJson.dependencies as Record<string, string>)[missingPackage] ??
+      detectDependencyVersion(missingPackage);
 
     if (internalDependencyVersion) {
-      dependencies[packageName] = stripWorkspaceFromVersion(internalDependencyVersion);
+      dependencies[missingPackage] = stripWorkspaceFromVersion(internalDependencyVersion);
     }
   }
 
+  for (const [packageName, version] of Object.entries(resolvedPackageVersions)) {
+    dependencies[packageName] = version;
+  }
+
   if (config.additionalPackages) {
     for (const packageName of config.additionalPackages) {
       if (dependencies[packageName]) {

packages/cli-v3/src/utilities/javascriptProject.ts

@@ -5,51 +5,10 @@ import { logger } from "./logger";
 import { PackageManager, getUserPackageManager } from "./getUserPackageManager";
 import { PackageJson } from "type-fest";
 import { assertExhaustive } from "./assertExhaustive";
+import { builtinModules } from "node:module";
 
 export type ResolveOptions = { allowDev: boolean };
 
-const BuiltInModules = new Set([
-  "assert",
-  "async_hooks",
-  "buffer",
-  "child_process",
-  "cluster",
-  "console",
-  "constants",
-  "crypto",
-  "dgram",
-  "dns",
-  "domain",
-  "events",
-  "fs",
-  "http",
-  "http2",
-  "https",
-  "inspector",
-  "module",
-  "net",
-  "os",
-  "path",
-  "perf_hooks",
-  "process",
-  "punycode",
-  "querystring",
-  "readline",
-  "repl",
-  "stream",
-  "string_decoder",
-  "timers",
-  "tls",
-  "trace_events",
-  "tty",
-  "url",
-  "util",
-  "v8",
-  "vm",
-  "worker_threads",
-  "zlib",
-]);
-
 export class JavascriptProject {
   private _packageJson?: PackageJson;
   private _packageManager?: PackageManager;
@@ -84,26 +43,72 @@ export class JavascriptProject {
     }
   }
 
-  async resolve(packageName: string, options?: ResolveOptions): Promise<string | undefined> {
-    if (BuiltInModules.has(packageName)) {
-      return undefined;
-    }
+  async resolveAll(
+    packageNames: string[],
+    options?: ResolveOptions
+  ): Promise<Record<string, string>> {
+    const externalPackages = packageNames.filter((packageName) => !isBuiltInModule(packageName));
 
     const opts = { allowDev: false, ...options };
 
-    const packageJsonVersion = this.packageJson.dependencies?.[packageName];
+    const command = await this.#getCommand();
 
-    if (typeof packageJsonVersion === "string") {
-      return packageJsonVersion;
-    }
+    try {
+      const versions = await command.resolveDependencyVersions(externalPackages, {
+        cwd: this.projectPath,
+      });
+
+      if (versions) {
+        logger.debug(`Resolved [${externalPackages.join(", ")}] version using ${command.name}`, {
+          versions,
+        });
+      }
+
+      // Merge the resolved versions with the package.json dependencies
+      const missingPackages = externalPackages.filter((packageName) => !versions[packageName]);
+      const missingPackageVersions: Record<string, string> = {};
 
-    if (opts.allowDev) {
-      const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
+      for (const packageName of missingPackages) {
+        const packageJsonVersion = this.packageJson.dependencies?.[packageName];
 
-      if (typeof devPackageJsonVersion === "string") {
-        return devPackageJsonVersion;
+        if (typeof packageJsonVersion === "string") {
+          logger.debug(`Resolved ${packageName} version using package.json`, {
+            packageJsonVersion,
+          });
+
+          missingPackageVersions[packageName] = packageJsonVersion;
+        }
+
+        if (opts.allowDev) {
+          const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
+
+          if (typeof devPackageJsonVersion === "string") {
+            logger.debug(`Resolved ${packageName} version using devDependencies`, {
+              devPackageJsonVersion,
+            });
+
+            missingPackageVersions[packageName] = devPackageJsonVersion;
+          }
+        }
       }
+
+      return { ...versions, ...missingPackageVersions };
+    } catch (error) {
+      logger.debug(`Failed to resolve dependency versions using ${command.name}`, {
+        packageNames,
+        error,
+      });
+
+      return {};
     }
+  }
+
+  async resolve(packageName: string, options?: ResolveOptions): Promise<string | undefined> {
+    if (isBuiltInModule(packageName)) {
+      return undefined;
+    }
+
+    const opts = { allowDev: false, ...options };
 
     const command = await this.#getCommand();
 
@@ -113,8 +118,30 @@ export class JavascriptProject {
       });
 
       if (version) {
+        logger.debug(`Resolved ${packageName} version using ${command.name}`, { version });
+
         return version;
       }
+
+      const packageJsonVersion = this.packageJson.dependencies?.[packageName];
+
+      if (typeof packageJsonVersion === "string") {
+        logger.debug(`Resolved ${packageName} version using package.json`, { packageJsonVersion });
+
+        return packageJsonVersion;
+      }
+
+      if (opts.allowDev) {
+        const devPackageJsonVersion = this.packageJson.devDependencies?.[packageName];
+
+        if (typeof devPackageJsonVersion === "string") {
+          logger.debug(`Resolved ${packageName} version using devDependencies`, {
+            devPackageJsonVersion,
+          });
+
+          return devPackageJsonVersion;
+        }
+      }
     } catch (error) {
       logger.debug(`Failed to resolve dependency version using ${command.name}`, {
         packageName,
@@ -176,6 +203,11 @@ interface PackageManagerCommands {
     packageName: string,
     options: PackageManagerOptions
   ): Promise<string | undefined>;
+
+  resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>>;
 }
 
 class PNPMCommands implements PackageManagerCommands {
@@ -197,7 +229,7 @@ class PNPMCommands implements PackageManagerCommands {
     const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageName} -r --json`;
     const result = JSON.parse(stdout) as PnpmList;
 
-    logger.debug(`Resolving ${packageName} version using ${this.name}`, { result });
+    logger.debug(`Resolving ${packageName} version using ${this.name}`);
 
     // Return the first dependency version that matches the package name
     for (const dep of result) {
@@ -208,6 +240,31 @@ class PNPMCommands implements PackageManagerCommands {
       }
     }
   }
+
+  async resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>> {
+    const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageNames} -r --json`;
+    const result = JSON.parse(stdout) as PnpmList;
+
+    logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
+
+    const results: Record<string, string> = {};
+
+    // Return the first dependency version that matches the package name
+    for (const dep of result) {
+      for (const packageName of packageNames) {
+        const dependency = dep.dependencies?.[packageName];
+
+        if (dependency) {
+          results[packageName] = dependency.version;
+        }
+      }
+    }
+
+    return results;
+  }
 }
 
 type NpmDependency = {
@@ -246,6 +303,28 @@ class NPMCommands implements PackageManagerCommands {
     return this.#recursivelySearchDependencies(output.dependencies, packageName);
   }
 
+  async resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>> {
+    const { stdout } = await $({ cwd: options.cwd })`${this.cmd} list ${packageNames} --json`;
+    const output = JSON.parse(stdout) as NpmListOutput;
+
+    logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`, { output });
+
+    const results: Record<string, string> = {};
+
+    for (const packageName of packageNames) {
+      const version = this.#recursivelySearchDependencies(output.dependencies, packageName);
+
+      if (version) {
+        results[packageName] = version;
+      }
+    }
+
+    return results;
+  }
+
   #recursivelySearchDependencies(
     dependencies: Record<string, NpmDependency>,
     packageName: string
@@ -286,7 +365,7 @@ class YarnCommands implements PackageManagerCommands {
 
     const lines = stdout.split("\n");
 
-    logger.debug(`Resolving ${packageName} version using ${this.name}`, { lines });
+    logger.debug(`Resolving ${packageName} version using ${this.name}`);
 
     for (const line of lines) {
       const json = JSON.parse(line);
@@ -296,4 +375,54 @@ class YarnCommands implements PackageManagerCommands {
       }
     }
   }
+
+  async resolveDependencyVersions(
+    packageNames: string[],
+    options: PackageManagerOptions
+  ): Promise<Record<string, string>> {
+    const { stdout } = await $({ cwd: options.cwd })`${this.cmd} info ${packageNames} --json`;
+
+    const lines = stdout.split("\n");
+
+    logger.debug(`Resolving ${packageNames.join(" ")} version using ${this.name}`);
+
+    const results: Record<string, string> = {};
+
+    for (const line of lines) {
+      const json = JSON.parse(line);
+
+      const packageName = this.#parseYarnValueIntoPackageName(json.value);
+
+      if (packageNames.includes(packageName)) {
+        results[packageName] = json.children.Version;
+      }
+    }
+
+    return results;
+  }
+
+  // The "value" when doing yarn info is formatted like this:
+  // "package-name@npm:version" or "package-name@workspace:version"
+  // This function will parse the value into just the package name.
+  // This correctly handles scoped packages as well e.g. @scope/package-name@npm:version
+  #parseYarnValueIntoPackageName(value: string): string {
+    const parts = value.split("@");
+
+    // If the value does not contain an "@" symbol, then it's just the package name
+    if (parts.length === 3) {
+      return parts[1] as string;
+    }
+
+    // If the value contains an "@" symbol, then the package name is the first part
+    return parts[0] as string;
+  }
+}
+
+function isBuiltInModule(module: string): boolean {
+  // if the module has node: prefix, it's a built-in module
+  if (module.startsWith("node:")) {
+    return true;
+  }
+
+  return builtinModules.includes(module);
 }