Merge pull request microsoft#32613 from microsoft/singleHostFsWatchFile

Create only single StatFileWatcher through node

Author: sheetalkamat

src/compiler/sys.ts

@@ -304,6 +304,53 @@ namespace ts {
         }
     }
 
+    /* @internal */
+    export function createSingleFileWatcherPerName(
+        watchFile: HostWatchFile,
+        useCaseSensitiveFileNames: boolean
+    ): HostWatchFile {
+        interface SingleFileWatcher {
+            watcher: FileWatcher;
+            refCount: number;
+        }
+        const cache = createMap<SingleFileWatcher>();
+        const callbacksCache = createMultiMap<FileWatcherCallback>();
+        const toCanonicalFileName = createGetCanonicalFileName(useCaseSensitiveFileNames);
+
+        return (fileName, callback, pollingInterval) => {
+            const path = toCanonicalFileName(fileName);
+            const existing = cache.get(path);
+            if (existing) {
+                existing.refCount++;
+            }
+            else {
+                cache.set(path, {
+                    watcher: watchFile(
+                        fileName,
+                        (fileName, eventKind) => forEach(
+                            callbacksCache.get(path),
+                            cb => cb(fileName, eventKind)
+                        ),
+                        pollingInterval
+                    ),
+                    refCount: 1
+                });
+            }
+            callbacksCache.add(path, callback);
+
+            return {
+                close: () => {
+                    const watcher = Debug.assertDefined(cache.get(path));
+                    callbacksCache.remove(path, callback);
+                    watcher.refCount--;
+                    if (watcher.refCount) return;
+                    cache.delete(path);
+                    closeFileWatcherOf(watcher);
+                }
+            };
+        };
+    }
+
     /**
      * Returns true if file status changed
      */
@@ -695,6 +742,7 @@ namespace ts {
             const useNonPollingWatchers = process.env.TSC_NONPOLLING_WATCHER;
             const tscWatchFile = process.env.TSC_WATCHFILE;
             const tscWatchDirectory = process.env.TSC_WATCHDIRECTORY;
+            const fsWatchFile = createSingleFileWatcherPerName(fsWatchFileWorker, useCaseSensitiveFileNames);
             let dynamicPollingWatchFile: HostWatchFile | undefined;
             const nodeSystem: System = {
                 args: process.argv.slice(2),
@@ -835,7 +883,7 @@ namespace ts {
                 return useNonPollingWatchers ?
                     createNonPollingWatchFile() :
                     // Default to do not use polling interval as it is before this experiment branch
-                    (fileName, callback) => fsWatchFile(fileName, callback);
+                    (fileName, callback) => fsWatchFile(fileName, callback, /*pollingInterval*/ undefined);
             }
 
             function getWatchDirectory(): HostWatchDirectory {
@@ -916,7 +964,7 @@ namespace ts {
                 }
             }
 
-            function fsWatchFile(fileName: string, callback: FileWatcherCallback, pollingInterval?: number): FileWatcher {
+            function fsWatchFileWorker(fileName: string, callback: FileWatcherCallback, pollingInterval?: number): FileWatcher {
                 _fs.watchFile(fileName, { persistent: true, interval: pollingInterval || 250 }, fileChanged);
                 let eventKind: FileWatcherEventKind;
                 return {

src/compiler/tsbuild.ts

@@ -564,8 +564,51 @@ namespace ts {
     }
 
     function getBuildOrder(state: SolutionBuilderState) {
-        return state.buildOrder ||
-            (state.buildOrder = createBuildOrder(state, state.rootNames.map(f => resolveProjectName(state, f))));
+        return state.buildOrder || createStateBuildOrder(state);
+    }
+
+    function createStateBuildOrder(state: SolutionBuilderState) {
+        const buildOrder = createBuildOrder(state, state.rootNames.map(f => resolveProjectName(state, f)));
+        if (arrayIsEqualTo(state.buildOrder, buildOrder)) return state.buildOrder!;
+
+        // Clear all to ResolvedConfigFilePaths cache to start fresh
+        state.resolvedConfigFilePaths.clear();
+        const currentProjects = arrayToSet(
+            buildOrder,
+            resolved => toResolvedConfigFilePath(state, resolved)
+        ) as ConfigFileMap<true>;
+
+        const noopOnDelete = { onDeleteValue: noop };
+        // Config file cache
+        mutateMapSkippingNewValues(state.configFileCache, currentProjects, noopOnDelete);
+        mutateMapSkippingNewValues(state.projectStatus, currentProjects, noopOnDelete);
+        mutateMapSkippingNewValues(state.buildInfoChecked, currentProjects, noopOnDelete);
+        mutateMapSkippingNewValues(state.builderPrograms, currentProjects, noopOnDelete);
+        mutateMapSkippingNewValues(state.diagnostics, currentProjects, noopOnDelete);
+        mutateMapSkippingNewValues(state.projectPendingBuild, currentProjects, noopOnDelete);
+        mutateMapSkippingNewValues(state.projectErrorsReported, currentProjects, noopOnDelete);
+
+        // Remove watches for the program no longer in the solution
+        if (state.watch) {
+            mutateMapSkippingNewValues(
+                state.allWatchedConfigFiles,
+                currentProjects,
+                { onDeleteValue: closeFileWatcher }
+            );
+
+            mutateMapSkippingNewValues(
+                state.allWatchedWildcardDirectories,
+                currentProjects,
+                { onDeleteValue: existingMap => existingMap.forEach(closeFileWatcherOf) }
+            );
+
+            mutateMapSkippingNewValues(
+                state.allWatchedInputFiles,
+                currentProjects,
+                { onDeleteValue: existingMap => existingMap.forEach(closeFileWatcher) }
+            );
+        }
+        return state.buildOrder = buildOrder;
     }
 
     function getBuildOrderFor(state: SolutionBuilderState, project: string | undefined, onlyReferences: boolean | undefined) {

src/compiler/utilities.ts

@@ -4466,8 +4466,7 @@ namespace ts {
         map.clear();
     }
 
-    export interface MutateMapOptions<T, U> {
-        createNewValue(key: string, valueInNewMap: U): T;
+    export interface MutateMapSkippingNewValuesOptions<T, U> {
         onDeleteValue(existingValue: T, key: string): void;
 
         /**
@@ -4482,8 +4481,12 @@ namespace ts {
     /**
      * Mutates the map with newMap such that keys in map will be same as newMap.
      */
-    export function mutateMap<T, U>(map: Map<T>, newMap: ReadonlyMap<U>, options: MutateMapOptions<T, U>) {
-        const { createNewValue, onDeleteValue, onExistingValue } = options;
+    export function mutateMapSkippingNewValues<T, U>(
+        map: Map<T>,
+        newMap: ReadonlyMap<U>,
+        options: MutateMapSkippingNewValuesOptions<T, U>
+    ) {
+        const { onDeleteValue, onExistingValue } = options;
         // Needs update
         map.forEach((existingValue, key) => {
             const valueInNewMap = newMap.get(key);
@@ -4497,7 +4500,20 @@ namespace ts {
                 onExistingValue(existingValue, valueInNewMap, key);
             }
         });
+    }
+
+    export interface MutateMapOptions<T, U> extends MutateMapSkippingNewValuesOptions<T, U> {
+        createNewValue(key: string, valueInNewMap: U): T;
+    }
+
+    /**
+     * Mutates the map with newMap such that keys in map will be same as newMap.
+     */
+    export function mutateMap<T, U>(map: Map<T>, newMap: ReadonlyMap<U>, options: MutateMapOptions<T, U>) {
+        // Needs update
+        mutateMapSkippingNewValues(map, newMap, options);
 
+        const { createNewValue } = options;
         // Add new values that are not already present
         newMap.forEach((valueInNewMap, key) => {
             if (!map.has(key)) {

src/harness/virtualFileSystemWithWatch.ts

@@ -314,6 +314,11 @@ interface Array<T> {}`
         invokeFileDeleteCreateAsPartInsteadOfChange: boolean;
     }
 
+    export enum Tsc_WatchFile {
+        DynamicPolling = "DynamicPriorityPolling",
+        SingleFileWatcherPerName = "SingleFileWatcherPerName"
+    }
+
     export enum Tsc_WatchDirectory {
         WatchFile = "RecursiveDirectoryUsingFsWatchFile",
         NonRecursiveWatchDirectory = "RecursiveDirectoryUsingNonRecursiveWatchDirectory",
@@ -339,7 +344,7 @@ interface Array<T> {}`
         readonly watchedFiles = createMultiMap<TestFileWatcher>();
         private readonly executingFilePath: string;
         private readonly currentDirectory: string;
-        private readonly dynamicPriorityWatchFile: HostWatchFile | undefined;
+        private readonly customWatchFile: HostWatchFile | undefined;
         private readonly customRecursiveWatchDirectory: HostWatchDirectory | undefined;
         public require: ((initialPath: string, moduleName: string) => server.RequireResult) | undefined;
 
@@ -349,9 +354,23 @@ interface Array<T> {}`
             this.executingFilePath = this.getHostSpecificPath(executingFilePath);
             this.currentDirectory = this.getHostSpecificPath(currentDirectory);
             this.reloadFS(fileOrFolderorSymLinkList);
-            this.dynamicPriorityWatchFile = this.environmentVariables && this.environmentVariables.get("TSC_WATCHFILE") === "DynamicPriorityPolling" ?
-                createDynamicPriorityPollingWatchFile(this) :
-                undefined;
+            const tscWatchFile = this.environmentVariables && this.environmentVariables.get("TSC_WATCHFILE") as Tsc_WatchFile;
+            switch (tscWatchFile) {
+                case Tsc_WatchFile.DynamicPolling:
+                    this.customWatchFile = createDynamicPriorityPollingWatchFile(this);
+                    break;
+                case Tsc_WatchFile.SingleFileWatcherPerName:
+                    this.customWatchFile = createSingleFileWatcherPerName(
+                        this.watchFileWorker.bind(this),
+                        this.useCaseSensitiveFileNames
+                    );
+                    break;
+                case undefined:
+                    break;
+                default:
+                    Debug.assertNever(tscWatchFile);
+            }
+
             const tscWatchDirectory = this.environmentVariables && this.environmentVariables.get("TSC_WATCHDIRECTORY") as Tsc_WatchDirectory;
             if (tscWatchDirectory === Tsc_WatchDirectory.WatchFile) {
                 const watchDirectory: HostWatchDirectory = (directory, cb) => this.watchFile(directory, () => cb(directory), PollingInterval.Medium);
@@ -405,7 +424,7 @@ interface Array<T> {}`
             return s;
         }
 
-        private now() {
+        now() {
             this.time += timeIncrements;
             return new Date(this.time);
         }
@@ -854,10 +873,14 @@ interface Array<T> {}`
         }
 
         watchFile(fileName: string, cb: FileWatcherCallback, pollingInterval: number) {
-            if (this.dynamicPriorityWatchFile) {
-                return this.dynamicPriorityWatchFile(fileName, cb, pollingInterval);
+            if (this.customWatchFile) {
+                return this.customWatchFile(fileName, cb, pollingInterval);
             }
 
+            return this.watchFileWorker(fileName, cb);
+        }
+
+        private watchFileWorker(fileName: string, cb: FileWatcherCallback) {
             const path = this.toFullPath(fileName);
             const callback: TestFileWatcher = { fileName, cb };
             this.watchedFiles.add(path, callback);

src/testRunner/tsconfig.json

@@ -61,7 +61,6 @@
         "unittests/semver.ts",
         "unittests/shimMap.ts",
         "unittests/transform.ts",
-        "unittests/tsbuildWatchMode.ts",
         "unittests/config/commandLineParsing.ts",
         "unittests/config/configurationExtension.ts",
         "unittests/config/convertCompilerOptionsFromJson.ts",
@@ -103,6 +102,8 @@
         "unittests/tsbuild/resolveJsonModule.ts",
         "unittests/tsbuild/sample.ts",
         "unittests/tsbuild/transitiveReferences.ts",
+        "unittests/tsbuild/watchEnvironment.ts",
+        "unittests/tsbuild/watchMode.ts",
         "unittests/tscWatch/consoleClearing.ts",
         "unittests/tscWatch/emit.ts",
         "unittests/tscWatch/emitAndErrorUpdates.ts",

src/testRunner/unittests/tsbuild/watchEnvironment.ts

@@ -0,0 +1,124 @@
+namespace ts.tscWatch {
+    describe("unittests:: tsbuild:: watchEnvironment:: tsbuild:: watchMode:: with different watch environments", () => {
+        describe("when watchFile can create multiple watchers per file", () => {
+            verifyWatchFileOnMultipleProjects(/*singleWatchPerFile*/ false);
+        });
+
+        describe("when watchFile is single watcher per file", () => {
+            verifyWatchFileOnMultipleProjects(
+                /*singleWatchPerFile*/ true,
+                arrayToMap(["TSC_WATCHFILE"], identity, () => TestFSWithWatch.Tsc_WatchFile.SingleFileWatcherPerName)
+            );
+        });
+
+        function verifyWatchFileOnMultipleProjects(singleWatchPerFile: boolean, environmentVariables?: Map<string>) {
+            it("watchFile on same file multiple times because file is part of multiple projects", () => {
+                const project = `${TestFSWithWatch.tsbuildProjectsLocation}/myproject`;
+                let maxPkgs = 4;
+                const configPath = `${project}/tsconfig.json`;
+                const typing: File = {
+                    path: `${project}/typings/xterm.d.ts`,
+                    content: "export const typing = 10;"
+                };
+
+                const allPkgFiles = pkgs(pkgFiles);
+                const system = createWatchedSystem([libFile, typing, ...flatArray(allPkgFiles)], { currentDirectory: project, environmentVariables });
+                writePkgReferences();
+                const host = createSolutionBuilderWithWatchHost(system);
+                const solutionBuilder = createSolutionBuilderWithWatch(host, ["tsconfig.json"], { watch: true, verbose: true });
+                solutionBuilder.build();
+                checkOutputErrorsInitial(system, emptyArray, /*disableConsoleClears*/ undefined, [
+                    `Projects in this build: \r\n${
+                    concatenate(
+                        pkgs(index => `    * pkg${index}/tsconfig.json`),
+                        ["    * tsconfig.json"]
+                    ).join("\r\n")}\n\n`,
+                    ...flatArray(pkgs(index => [
+                        `Project 'pkg${index}/tsconfig.json' is out of date because output file 'pkg${index}/index.js' does not exist\n\n`,
+                        `Building project '${project}/pkg${index}/tsconfig.json'...\n\n`
+                    ]))
+                ]);
+
+                const watchFilesDetailed = arrayToMap(flatArray(allPkgFiles), f => f.path, () => 1);
+                watchFilesDetailed.set(configPath, 1);
+                watchFilesDetailed.set(typing.path, singleWatchPerFile ? 1 : maxPkgs);
+                checkWatchedFilesDetailed(system, watchFilesDetailed);
+                system.writeFile(typing.path, `${typing.content}export const typing1 = 10;`);
+                verifyInvoke();
+
+                // Make change
+                maxPkgs--;
+                writePkgReferences();
+                system.checkTimeoutQueueLengthAndRun(1);
+                checkOutputErrorsIncremental(system, emptyArray);
+                const lastFiles = last(allPkgFiles);
+                lastFiles.forEach(f => watchFilesDetailed.delete(f.path));
+                watchFilesDetailed.set(typing.path, singleWatchPerFile ? 1 : maxPkgs);
+                checkWatchedFilesDetailed(system, watchFilesDetailed);
+                system.writeFile(typing.path, typing.content);
+                verifyInvoke();
+
+                // Make change to remove all the watches
+                maxPkgs = 0;
+                writePkgReferences();
+                system.checkTimeoutQueueLengthAndRun(1);
+                checkOutputErrorsIncremental(system, [
+                    `tsconfig.json(1,10): error TS18002: The 'files' list in config file '${configPath}' is empty.\n`
+                ]);
+                checkWatchedFilesDetailed(system, [configPath], 1);
+
+                system.writeFile(typing.path, `${typing.content}export const typing1 = 10;`);
+                system.checkTimeoutQueueLength(0);
+
+                function flatArray<T>(arr: T[][]): readonly T[] {
+                    return flatMap(arr, identity);
+                }
+                function pkgs<T>(cb: (index: number) => T): T[] {
+                    const result: T[] = [];
+                    for (let index = 0; index < maxPkgs; index++) {
+                        result.push(cb(index));
+                    }
+                    return result;
+                }
+                function createPkgReference(index: number) {
+                    return { path: `./pkg${index}` };
+                }
+                function pkgFiles(index: number): File[] {
+                    return [
+                        {
+                            path: `${project}/pkg${index}/index.ts`,
+                            content: `export const pkg${index} = ${index};`
+                        },
+                        {
+                            path: `${project}/pkg${index}/tsconfig.json`,
+                            content: JSON.stringify({
+                                complerOptions: { composite: true },
+                                include: [
+                                    "**/*.ts",
+                                    "../typings/xterm.d.ts"
+                                ]
+                            })
+                        }
+                    ];
+                }
+                function writePkgReferences() {
+                    system.writeFile(configPath, JSON.stringify({
+                        files: [],
+                        include: [],
+                        references: pkgs(createPkgReference)
+                    }));
+                }
+                function verifyInvoke() {
+                    pkgs(() => system.checkTimeoutQueueLengthAndRun(1));
+                    checkOutputErrorsIncremental(system, emptyArray, /*disableConsoleClears*/ undefined, /*logsBeforeWatchDiagnostics*/ undefined, [
+                        ...flatArray(pkgs(index => [
+                            `Project 'pkg${index}/tsconfig.json' is out of date because oldest output 'pkg${index}/index.js' is older than newest input 'typings/xterm.d.ts'\n\n`,
+                            `Building project '${project}/pkg${index}/tsconfig.json'...\n\n`,
+                            `Updating unchanged output timestamps of project '${project}/pkg${index}/tsconfig.json'...\n\n`
+                        ]))
+                    ]);
+                }
+            });
+        }
+    });
+}