Monorepo packages automatic handling (#1208)

* Add fixture

* Update fixture name

* Update fixture to work with npm & pnpm

* Update setup/teardown and README

* Add support for monorepo to e2e suite utilities

* Use direct dependencies info in esbuild plugin

* Renaming

* Add support for yarn

* Add fixture for npm

* Remove console.log call

* Edge cases

* Remove unused options

* Fix error unknown yarn in CI

* Fix logger debug call's error field

* Add changeset

* Fix missing span end

Author: codenem

.changeset/lucky-items-film.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Automatically bundle internal packages that use file, link or workspace protocl

.github/workflows/e2e.yml

@@ -42,6 +42,9 @@ jobs:
       - name: 🔧 Build worker template files
         run: pnpm --filter trigger.dev run build:workers
 
+      - name: Enable corepack
+        run: corepack enable
+
       - name: Run E2E Tests
         run: |
           PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e

packages/cli-v3/e2e/README.md

@@ -117,9 +117,9 @@ This will test your fixture project, and generate outputs in the `packages/cli-v
 
     ```sh
     cd packages/cli-v3/e2e/fixtures/<fixture-name>
-    rm -rf node_modules
+    rm -rf **/node_modules
     npm install
-    rm -rf node_modules
+    rm -rf **/node_modules
     corepack use [email protected] # will update the yarn lockfile
     ```
 

packages/cli-v3/e2e/compile.ts

@@ -2,7 +2,7 @@ import { esbuildDecorators } from "@anatine/esbuild-decorators";
 import { build } from "esbuild";
 import { readFileSync } from "node:fs";
 import { writeFile } from "node:fs/promises";
-import { basename, join, posix, resolve } from "node:path";
+import { basename, join, posix, relative, resolve, sep } from "node:path";
 import invariant from "tiny-invariant";
 
 import {
@@ -15,9 +15,12 @@ import { writeJSONFile } from "../src/utilities/fileSystem.js";
 import { logger } from "../src/utilities/logger.js";
 import { createTaskFileImports, gatherTaskFiles } from "../src/utilities/taskFiles.js";
 import { escapeImportPath } from "../src/utilities/windows.js";
+import { E2EJavascriptProject } from "./javascriptProject.js";
+import { PackageManager } from "../src/utilities/getUserPackageManager.js";
 
 type CompileOptions = {
   outputMetafile?: string;
+  packageManager: PackageManager;
   resolvedConfig: ReadConfigResult;
   tempDir: string;
 };
@@ -28,6 +31,7 @@ export async function compile(options: CompileOptions) {
   }
 
   const {
+    packageManager,
     tempDir,
     resolvedConfig: { config },
   } = options;
@@ -61,6 +65,9 @@ export async function compile(options: CompileOptions) {
     );
   }
 
+  const e2eJsProject = new E2EJavascriptProject(config.projectDir, packageManager);
+  const directDependenciesMeta = await e2eJsProject.extractDirectDependenciesMeta();
+
   const result = await build({
     stdin: {
       contents: workerContents,
@@ -86,7 +93,12 @@ export async function compile(options: CompileOptions) {
     },
     plugins: [
       mockServerOnlyPlugin(),
-      bundleDependenciesPlugin("workerFacade", config.dependenciesToBundle, config.tsconfigPath),
+      bundleDependenciesPlugin(
+        "workerFacade",
+        directDependenciesMeta,
+        config.dependenciesToBundle,
+        config.tsconfigPath
+      ),
       workerSetupImportConfigPlugin(configPath),
       esbuildDecorators({
         tsconfig: config.tsconfigPath,
@@ -127,7 +139,12 @@ export async function compile(options: CompileOptions) {
       __PROJECT_CONFIG__: JSON.stringify(config),
     },
     plugins: [
-      bundleDependenciesPlugin("entryPoint.ts", config.dependenciesToBundle, config.tsconfigPath),
+      bundleDependenciesPlugin(
+        "entryPoint.ts",
+        directDependenciesMeta,
+        config.dependenciesToBundle,
+        config.tsconfigPath
+      ),
     ],
   });
 
@@ -145,17 +162,22 @@ export async function compile(options: CompileOptions) {
   logger.debug(`Writing compiled files to ${tempDir}`);
 
   // Get the metaOutput for the result build
+  const pathsToProjectDir = relative(
+    join(process.cwd(), "e2e", "fixtures"),
+    config.projectDir
+  ).split(sep);
+
   const metaOutput =
     result.metafile!.outputs[
-      posix.join("e2e", "fixtures", basename(config.projectDir), "out", "stdin.js")
+      posix.join("e2e", "fixtures", ...pathsToProjectDir, "out", "stdin.js")
     ];
 
   invariant(metaOutput, "Meta output for the result build is missing");
 
   // Get the metaOutput for the entryPoint build
   const entryPointMetaOutput =
     entryPointResult.metafile!.outputs[
-      posix.join("e2e", "fixtures", basename(config.projectDir), "out", "stdin.js")
+      posix.join("e2e", "fixtures", ...pathsToProjectDir, "out", "stdin.js")
     ];
 
   invariant(entryPointMetaOutput, "Meta output for the entryPoint build is missing");

packages/cli-v3/e2e/fixtures.config.ts

@@ -1,6 +1,7 @@
 export interface TestCase {
   resolveEnv?: { [key: string]: string };
   id: string;
+  workspaceRelativeDir?: string;
   skipTypecheck?: boolean;
   wantConfigNotFoundError?: boolean;
   wantConfigInvalidError?: boolean;
@@ -11,6 +12,16 @@ export interface TestCase {
 }
 
 export const fixturesConfig: TestCase[] = [
+  {
+    id: "compile-monorepo-packages",
+    skipTypecheck: true,
+    workspaceRelativeDir: "packages/trigger",
+  },
+  {
+    id: "compile-monorepo-packages-npm",
+    skipTypecheck: true,
+    workspaceRelativeDir: "packages/trigger",
+  },
   {
     id: "config-infisical-sdk",
     skipTypecheck: true,