Introduce a seed for the PRNG (#70)

* Introduce a seed for the PRNG

This will make it possible to repeat the randomly chosen LS operations, but will not prevent changes in the TS or the repos under test from causing the output to differ.  Pipeline timeouts may also vary from run to run.

* Use n/a as the sentinel value since empty string is not supported

Author: amcasey

azure-pipelines-gitTests-template.yml

@@ -33,6 +33,10 @@ parameters:
   - name: LANGUAGE
     displayName: Language of repos on GitHub (tsserver only)
     type: string
+  - name: PRNG_SEED
+    displayName: Pseudo-random number generator seed
+    type: string
+    default: 'n/a'
 
 jobs:
 - job: ListRepos
@@ -69,7 +73,7 @@ jobs:
       npm run build
       npm install -g pnpm
       mkdir 'RepoResults$(System.JobPositionInPhase)'
-      node dist/checkGithubRepos ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_VERSION }} ${{ parameters.NEW_VERSION }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }}
+      node dist/checkGithubRepos ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_VERSION }} ${{ parameters.NEW_VERSION }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }}
     displayName: 'Run TypeScript on repos'
     continueOnError: true
     env:

azure-pipelines-gitTests.yml

@@ -46,6 +46,10 @@ parameters:
     values:
     - TypeScript
     - JavaScript
+  - name: PRNG_SEED
+    displayName: Pseudo-random number generator seed
+    type: string
+    default: 'n/a'
 
 schedules:
   - cron: "0 18 * * Sun" # time is in UTC
@@ -72,4 +76,5 @@ extends:
     NEW_VERSION: ${{ parameters.NEW_VERSION }}
     MACHINE_COUNT: ${{ parameters.MACHINE_COUNT }}
     ENTRYPOINT: ${{ parameters.ENTRYPOINT }}
-    LANGUAGE: ${{ parameters.LANGUAGE }}
+    LANGUAGE: ${{ parameters.LANGUAGE }}
+    PRNG_SEED: ${{ parameters.PRNG_SEED }}

azure-pipelines-userTests.yml

@@ -44,6 +44,10 @@ parameters:
     values:
     - tsc
     - tsserver
+  - name: PRNG_SEED
+    displayName: Pseudo-random number generator seed
+    type: string
+    default: 'n/a'
 
 pr: none
 trigger: none
@@ -89,7 +93,7 @@ jobs:
       npm run build
       npm install -g pnpm
       mkdir 'RepoResults$(System.JobPositionInPhase)'
-      node dist/checkUserTestRepos ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_TS_REPO_URL }} ${{ parameters.OLD_HEAD_REF }} ${{ parameters.SOURCE_ISSUE }} ${{ parameters.TOP_REPOS }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }}
+      node dist/checkUserTestRepos ${{ parameters.ENTRYPOINT }} ${{ parameters.OLD_TS_REPO_URL }} ${{ parameters.OLD_HEAD_REF }} ${{ parameters.SOURCE_ISSUE }} ${{ parameters.TOP_REPOS }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }}
     displayName: 'Run user tests'
     env:
       GITHUB_PAT: $(GITHUB_PAT)

package-lock.json

@@ -17,12 +17,14 @@
         "@typescript/server-replay": "^0.2.6",
         "glob": "^7.2.3",
         "json5": "^2.2.1",
+        "random-seed": "^0.3.0",
         "simple-git": "^3.10.0"
     },
     "devDependencies": {
         "@types/glob": "^7.2.0",
         "@types/jest": "^27.5.2",
         "@types/node": "^14.18.21",
+        "@types/random-seed": "^0.3.3",
         "jest": "^27.4.4",
         "ts-jest": "^27.1.5",
         "typescript": "^4.8.3"

package.json

@@ -3,12 +3,12 @@ import { mainAsync, reportError, TsEntrypoint } from "./main";
 
 const { argv } = process;
 
-if (argv.length !== 10) {
-    console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} <ts_entrypoint> <old_ts_npm_version> <new_ts_npm_version> <repo_list_path> <worker_count> <worker_number> <result_dir_name> <diagnostic_output>`);
+if (argv.length !== 11) {
+    console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} <ts_entrypoint> <old_ts_npm_version> <new_ts_npm_version> <repo_list_path> <worker_count> <worker_number> <result_dir_name> <diagnostic_output> <prng_seed>`);
     process.exit(-1);
 }
 
-const [,, entrypoint, oldTsNpmVersion, newTsNpmVersion, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput] = argv;
+const [,, entrypoint, oldTsNpmVersion, newTsNpmVersion, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed] = argv;
 
 mainAsync({
     testType: "github",
@@ -22,6 +22,7 @@ mainAsync({
     oldTsNpmVersion,
     newTsNpmVersion,
     resultDirName,
+    prngSeed: prngSeed.toLowerCase() === "n/a" ? undefined : prngSeed,
 }).catch(err => {
     reportError(err, "Unhandled exception");
     process.exit(1);

src/checkGithubRepos.ts

@@ -3,12 +3,12 @@ import { mainAsync, reportError, TsEntrypoint } from "./main";
 
 const { argv } = process;
 
-if (argv.length !== 12) {
-    console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} <ts_entrypoint> <old_ts_repo_url> <old_head_ref> <pr_number> <is_top_repos> <repo_list_path> <worker_count> <worker_number> <result_dir_name> <diagnostic_output>`);
+if (argv.length !== 13) {
+    console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} <ts_entrypoint> <old_ts_repo_url> <old_head_ref> <pr_number> <is_top_repos> <repo_list_path> <worker_count> <worker_number> <result_dir_name> <diagnostic_output> <prng_seed>`);
     process.exit(-1);
 }
 
-const [,, entrypoint, oldTsRepoUrl, oldHeadRef, prNumber, buildWithNewWhenOldFails, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput] = argv;
+const [,, entrypoint, oldTsRepoUrl, oldHeadRef, prNumber, buildWithNewWhenOldFails, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed] = argv;
 
 mainAsync({
     testType: "user",
@@ -23,6 +23,7 @@ mainAsync({
     workerNumber: +workerNumber,
     resultDirName,
     diagnosticOutput: diagnosticOutput.toLowerCase() === "true",
+    prngSeed: prngSeed.toLowerCase() === "n/a" ? undefined : prngSeed,
 }).catch(err => {
     reportError(err, "Unhandled exception");
     process.exit(1);

src/checkUserTestRepos.ts

@@ -7,6 +7,7 @@ import ip = require("./utils/installPackages");
 import ut = require("./utils/userTestUtils");
 import fs = require("fs");
 import path = require("path");
+import randomSeed = require("random-seed");
 
 interface Params {
     /**
@@ -45,6 +46,11 @@ interface Params {
      * Which TypeScript entrypoint (tsc or tsserver) to test.
      */
     entrypoint: TsEntrypoint;
+    /**
+     * Used to make runs repeatable (e.g. when confirming that a PR no longer introduces failures).
+     * Pass undefined to have a seed generated.
+     */
+    prngSeed: string | undefined;
 }
 export interface GitParams extends Params {
     testType: "github";
@@ -64,6 +70,8 @@ const processCwd = process.cwd();
 const packageTimeout = 10 * 60 * 1000;
 const executionTimeout = 10 * 60 * 1000;
 
+const prng = randomSeed.create();
+
 export type RepoStatus =
     | "Unknown failure"
     | "Git clone failed"
@@ -174,7 +182,7 @@ async function getTsServerRepoResult(
     const repoDir = path.join(downloadDir, repo.name);
 
     // Presumably, people occasionally browse repos without installing the packages first
-    const installCommands = (Math.random() > 0.2)
+    const installCommands = (prng.random() > 0.2)
         ? (await installPackagesAndGetCommands(repo, downloadDir, repoDir, /*cleanOnFailure*/ true, diagnosticOutput))!
         : [];
 
@@ -186,7 +194,7 @@ async function getTsServerRepoResult(
     const lsStart = performance.now();
     try {
         console.log(`Testing with ${newTsServerPath} (new)`);
-        const newSpawnResult = await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "exerciseServer.js"), repoDir, replayScriptPath, newTsServerPath, diagnosticOutput.toString()], executionTimeout);
+        const newSpawnResult = await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "exerciseServer.js"), repoDir, replayScriptPath, newTsServerPath, diagnosticOutput.toString(), prng.string(10)], executionTimeout);
         if (!newSpawnResult) {
             // CONSIDER: It might be interesting to treat timeouts as failures, but they'd be harder to baseline and more likely to have flaky repros
             console.log(`New server timed out after ${executionTimeout} ms`);
@@ -548,6 +556,10 @@ function getWorkerRepos(allRepos: readonly git.Repo[], workerCount: number, work
 }
 
 export async function mainAsync(params: GitParams | UserParams): Promise<void> {
+    if (params.prngSeed) {
+        prng.seed(params.prngSeed);
+    }
+
     const downloadDir = params.tmpfs ? "/mnt/ts_downloads" : path.join(processCwd, "ts_downloads");
     // TODO: check first whether the directory exists and skip downloading if possible
     // TODO: Seems like this should come after the typescript download

src/main.ts

@@ -6,6 +6,7 @@ import process = require("process");
 import path = require("path");
 import glob = require("glob");
 import { performance } from "perf_hooks";
+import randomSeed = require("random-seed");
 import { EXIT_BAD_ARGS, EXIT_UNHANDLED_EXCEPTION, EXIT_SERVER_EXIT_FAILED, EXIT_SERVER_CRASH, EXIT_SERVER_ERROR, EXIT_LANGUAGE_SERVICE_DISABLED } from "./exerciseServerConstants";
 
 const testDirPlaceholder = "@PROJECT_ROOT@";
@@ -14,15 +15,16 @@ const exitTimeoutMs = 5000;
 
 const argv = process.argv;
 
-if (argv.length !== 6) {
-    console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} <project_dir> <requests_path> <server_path> <diagnostic_output>`);
+if (argv.length !== 7) {
+    console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} <project_dir> <requests_path> <server_path> <diagnostic_output> <prng_seed>`);
     process.exit(EXIT_BAD_ARGS);
 }
 
 // CONVENTION: stderr is for output to the log; stdout is for output to the user
 
-const [, , testDir, replayScriptPath, tsserverPath, diag] = argv;
+const [, , testDir, replayScriptPath, tsserverPath, diag, seed] = argv;
 const diagnosticOutput = diag.toLocaleLowerCase() === "true";
+const prng = randomSeed.create(seed);
 
 exerciseServer(testDir, replayScriptPath, tsserverPath).catch(e => {
     console.error(e);
@@ -138,7 +140,7 @@ async function exerciseServerWorker(testDir: string, tsserverPath: string, repla
         // NB: greater than 1 behaves the same as 1
         const skipFileProb = 1000 / files.length;
         for (const openFileRelativePath of files) {
-            if (Math.random() > skipFileProb) continue;
+            if (prng.random() > skipFileProb) continue;
 
             const openFileAbsolutePath = path.join(testDirPlaceholder, openFileRelativePath).replace(/\\/g, "/");
 
@@ -271,7 +273,7 @@ async function exerciseServerWorker(testDir: string, tsserverPath: string, repla
                             "file": openFileAbsolutePath,
                             "line": line,
                             "offset": column,
-                            "includeExternalModuleExports": Math.random() < 0.01, // auto-imports are too slow to test everywhere
+                            "includeExternalModuleExports": prng.random() < 0.01, // auto-imports are too slow to test everywhere
                             "includeInsertTextCompletions": true,
                             "triggerKind": 1,
                         }
@@ -369,7 +371,7 @@ async function exerciseServerWorker(testDir: string, tsserverPath: string, repla
     }
 
     async function message(request: any, prob = 1) {
-        if (Math.random() > prob) return undefined;
+        if (prng.random() > prob) return undefined;
 
         request = {
             "seq": seq++,