Add basic support for yarn and pnpm workspaces (#83)

* Extract helper for monorepo package ordering

* Add handling for yarn workspaces

* Add minimal handling for pnpm workspaces

* Check for lerna.json last

* Store and reuse monorepo ordering

* Handle yarn2 workspace syntax

* Harden against malformed package.json files

* Harden against some unusual repos seen in the wild

Author: amcasey

package-lock.json

@@ -16,13 +16,15 @@
         "@typescript/server-harness": "^0.3.4",
         "@typescript/server-replay": "^0.2.11",
         "glob": "^7.2.3",
+        "js-yaml": "^4.1.0",
         "json5": "^2.2.1",
         "random-seed": "^0.3.0",
         "simple-git": "^3.10.0"
     },
     "devDependencies": {
         "@types/glob": "^7.2.0",
         "@types/jest": "^27.5.2",
+        "@types/js-yaml": "^4.0.5",
         "@types/node": "^14.18.21",
         "@types/random-seed": "^0.3.3",
         "jest": "^27.4.4",

package.json

@@ -124,10 +124,21 @@ async function cloneRepo(
     }
 }
 
+async function getMonorepoPackages(repoDir: string): Promise<readonly string[] | undefined> {
+    try {
+        return await pu.getMonorepoOrder(repoDir);
+    }
+    catch (e) {
+        reportError(e, `Error identifying monorepo packages for ${repoDir} - treating as separate packages`);
+        return undefined;
+    }
+}
+
 async function installPackagesAndGetCommands(
     repo: git.Repo,
     downloadDir: string,
     repoDir: string,
+    monorepoPackages: readonly string[],
     cleanOnFailure: boolean,
     diagnosticOutput: boolean): Promise<ip.InstallCommand[] | undefined> {
     const packageInstallStart = performance.now();
@@ -139,7 +150,7 @@ async function installPackagesAndGetCommands(
                 /*ignoreScripts*/ true,
                 /*quietOutput*/ !diagnosticOutput,
                 /*recursiveSearch*/ !isUserTestRepo,
-                /*lernaPackages*/ undefined,
+                /*monorepoPackages*/ monorepoPackages,
             repo.types);
         await installPackages(repoDir, commands, packageTimeout);
         return commands;
@@ -180,10 +191,11 @@ async function getTsServerRepoResult(
     }
 
     const repoDir = path.join(downloadDir, repo.name);
+    const monorepoPackages = await getMonorepoPackages(repoDir);
 
     // Presumably, people occasionally browse repos without installing the packages first
-    const installCommands = (prng.random() > 0.2)
-        ? (await installPackagesAndGetCommands(repo, downloadDir, repoDir, /*cleanOnFailure*/ true, diagnosticOutput))!
+    const installCommands = (prng.random() > 0.2) && monorepoPackages
+        ? (await installPackagesAndGetCommands(repo, downloadDir, repoDir, monorepoPackages, /*cleanOnFailure*/ true, diagnosticOutput))!
         : [];
 
     const isUserTestRepo = !repo.url;
@@ -374,8 +386,9 @@ export async function getTscRepoResult(
     }
 
     const repoDir = path.join(downloadDir, repo.name);
+    const monorepoPackages = await getMonorepoPackages(repoDir);
 
-    if (!await installPackagesAndGetCommands(repo, downloadDir, repoDir, /*cleanOnFailure*/ false, diagnosticOutput)) {
+    if (!monorepoPackages || !await installPackagesAndGetCommands(repo, downloadDir, repoDir, monorepoPackages, /*cleanOnFailure*/ false, diagnosticOutput)) {
         return { status: "Package install failed" };
     }
 
@@ -384,7 +397,7 @@ export async function getTscRepoResult(
     const buildStart = performance.now();
     try {
         console.log(`Building with ${oldTscPath} (old)`);
-        const oldErrors = await ge.buildAndGetErrors(repoDir, isUserTestRepo, oldTscPath, executionTimeout, /*skipLibCheck*/ true);
+        const oldErrors = await ge.buildAndGetErrors(repoDir, monorepoPackages, isUserTestRepo, oldTscPath, executionTimeout, /*skipLibCheck*/ true);
 
         if (oldErrors.hasConfigFailure) {
             console.log("Unable to build project graph");
@@ -424,7 +437,7 @@ export async function getTscRepoResult(
         }
 
         console.log(`Building with ${newTscPath} (new)`);
-        const newErrors = await ge.buildAndGetErrors(repoDir, isUserTestRepo, newTscPath, executionTimeout, /*skipLibCheck*/ true);
+        const newErrors = await ge.buildAndGetErrors(repoDir, monorepoPackages, isUserTestRepo, newTscPath, executionTimeout, /*skipLibCheck*/ true);
 
         if (newErrors.hasConfigFailure) {
             console.log("Unable to build project graph");

src/main.ts

@@ -64,7 +64,7 @@ export function errorEquals(error1: Error, error2: Error) {
  * @param tscPath The path to tsc.js.
  * @param skipLibCheck True pass --skipLibCheck when building non-composite projects.  (Defaults to true)
  */
-export async function buildAndGetErrors(repoDir: string, isUserTestRepo: boolean, tscPath: string, timeoutMs: number, skipLibCheck: boolean = true): Promise<RepoErrors> {
+export async function buildAndGetErrors(repoDir: string, monorepoPackages: readonly string[], isUserTestRepo: boolean, tscPath: string, timeoutMs: number, skipLibCheck: boolean = true): Promise<RepoErrors> {
     const projectErrors: ProjectErrors[] = [];
 
     // If it's a user test repo and there's a build.sh in the root, don't bother searching for projects
@@ -95,7 +95,7 @@ export async function buildAndGetErrors(repoDir: string, isUserTestRepo: boolean
     const simpleBuildArgs = ["--skipLibCheck", `${skipLibCheck}`, "--incremental", "false", "--pretty", "false", "-p"];
     const compositeBuildArgs = ["-b", "-f", "-v"]; // Build mode doesn't support --skipLibCheck or --pretty
 
-    const { simpleProjects, rootCompositeProjects, hasError: hasConfigFailure } = await projectGraph.getProjectsToBuild(repoDir);
+    const { simpleProjects, rootCompositeProjects, hasError: hasConfigFailure } = await projectGraph.getProjectsToBuild(repoDir, monorepoPackages);
     // TODO: Move this inside getProjectsToBuild, separating them is pointless (originally separate to enable simple-only and composite-only runs)
     const projectsToBuild = simpleProjects.concat(rootCompositeProjects);
     if (!projectsToBuild.length) {

src/utils/getTscErrors.ts

@@ -22,8 +22,8 @@ export interface InstallCommand {
  * Traverses the given directory and returns a list of commands that can be used, in order, to install
  * the packages required for building.
  */
-export async function installPackages(repoDir: string, ignoreScripts: boolean, quietOutput: boolean, recursiveSearch: boolean, lernaPackages?: readonly string[], types?: string[]): Promise<InstallCommand[]> {
-    lernaPackages = lernaPackages ?? await utils.getLernaOrder(repoDir);
+export async function installPackages(repoDir: string, ignoreScripts: boolean, quietOutput: boolean, recursiveSearch: boolean, monorepoPackages?: readonly string[], types?: string[]): Promise<InstallCommand[]> {
+    monorepoPackages = monorepoPackages ?? await utils.getMonorepoOrder(repoDir);
 
     const isRepoYarn = await utils.exists(path.join(repoDir, "yarn.lock"));
     // The existence of .yarnrc.yml indicates that this repo uses yarn 2
@@ -36,12 +36,12 @@ export async function installPackages(repoDir: string, ignoreScripts: boolean, q
     const packageFiles = utils.glob(repoDir, globPattern);
 
     for (const packageFile of packageFiles) {
-        let inLernaPackageDir = false;
-        for (const lernaPackage of lernaPackages) {
-            if (inLernaPackageDir = packageFile.startsWith(lernaPackage)) break;
+        let inMonorepoPackageDir = false;
+        for (const monorepoPackage of monorepoPackages) {
+            if (inMonorepoPackageDir = packageFile.startsWith(monorepoPackage)) break;
         }
-        if (inLernaPackageDir) {
-            // Skipping installation of lerna package
+        if (inMonorepoPackageDir) {
+            // Skipping installation of monorepo package
             continue;
         }