Avoid running restores for dotnet-watch run (#23421)

* Tweaks to make dotnet-watch run faster

* Previously dotnet-watch calculated the watch file list on every run by invoking MSBuild. This
  changes the tool to only calculate it if an MSBuild file (.targets, .props, .csproj etc) file changed
* For dotnet watch run and dotnet watch test command, use --no-restore if changed file is not an MSBuild file.

* Add opt-out switch

* Update src/Tools/dotnet-watch/README.md

* Fixup typo

* Update src/Tools/dotnet-watch/README.md

Author: pranavkm

src/Tools/dotnet-watch/README.md

@@ -29,6 +29,7 @@ Some configuration options can be passed to `dotnet watch` through environment v
 | Variable                                       | Effect                                                   |
 | ---------------------------------------------- | -------------------------------------------------------- |
 | DOTNET_USE_POLLING_FILE_WATCHER                | If set to "1" or "true", `dotnet watch` will use a polling file watcher instead of CoreFx's `FileSystemWatcher`. Used when watching files on network shares or Docker mounted volumes.                       |
+| DOTNET_WATCH_SUPPRESS_MSBUILD_INCREMENTALISM   | By default, `dotnet watch` optimizes the build by avoiding certain operations such as running restore or re-evaluating the set of watched files on every file change. If set to "1" or "true",  these optimizations are disabled. |
 
 ### MSBuild
 

src/Tools/dotnet-watch/src/DotNetWatchContext.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Extensions.Tools.Internal;
+
+namespace Microsoft.DotNet.Watcher.Tools
+{
+    public class DotNetWatchContext
+    {
+        public IReporter Reporter { get; set; } = NullReporter.Singleton;
+
+        public ProcessSpec ProcessSpec { get; set; }
+
+        public IFileSet FileSet { get; set; }
+
+        public int Iteration { get; set; }
+
+        public string ChangedFile { get; set; }
+
+        public bool RequiresMSBuildRevaluation { get; set; }
+
+        public bool SuppressMSBuildIncrementalism { get; set; }
+    }
+}

src/Tools/dotnet-watch/src/DotNetWatcher.cs

@@ -3,9 +3,12 @@
 
 using System;
 using System.Globalization;
+using System.IO;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.DotNet.Watcher.Internal;
+using Microsoft.DotNet.Watcher.Tools;
 using Microsoft.Extensions.CommandLineUtils;
 using Microsoft.Extensions.Tools.Internal;
 
@@ -15,33 +18,63 @@ public class DotNetWatcher
     {
         private readonly IReporter _reporter;
         private readonly ProcessRunner _processRunner;
+        private readonly IWatchFilter[] _filters;
 
-        public DotNetWatcher(IReporter reporter)
+        public DotNetWatcher(IReporter reporter, IFileSetFactory fileSetFactory)
         {
             Ensure.NotNull(reporter, nameof(reporter));
 
             _reporter = reporter;
             _processRunner = new ProcessRunner(reporter);
+
+            _filters = new IWatchFilter[]
+            {
+                new MSBuildEvaluationFilter(fileSetFactory),
+                new NoRestoreFilter(),
+            };
         }
 
-        public async Task WatchAsync(ProcessSpec processSpec, IFileSetFactory fileSetFactory,
-            CancellationToken cancellationToken)
+        public async Task WatchAsync(ProcessSpec processSpec, CancellationToken cancellationToken)
         {
             Ensure.NotNull(processSpec, nameof(processSpec));
 
-            var cancelledTaskSource = new TaskCompletionSource<object>();
-            cancellationToken.Register(state => ((TaskCompletionSource<object>) state).TrySetResult(null),
+            var cancelledTaskSource = new TaskCompletionSource();
+            cancellationToken.Register(state => ((TaskCompletionSource)state).TrySetResult(),
                 cancelledTaskSource);
 
-            var iteration = 1;
+            var initialArguments = processSpec.Arguments.ToArray();
+            var suppressMSBuildIncrementalism = Environment.GetEnvironmentVariable("DOTNET_WATCH_SUPPRESS_MSBUILD_INCREMENTALISM");
+            var context = new DotNetWatchContext
+            {
+                Iteration = -1,
+                ProcessSpec = processSpec,
+                Reporter = _reporter,
+                SuppressMSBuildIncrementalism = suppressMSBuildIncrementalism == "1" || suppressMSBuildIncrementalism == "true",
+            };
+
+            if (context.SuppressMSBuildIncrementalism)
+            {
+                _reporter.Verbose("MSBuild incremental optimizations suppressed.");
+            }
 
             while (true)
             {
-                processSpec.EnvironmentVariables["DOTNET_WATCH_ITERATION"] = iteration.ToString(CultureInfo.InvariantCulture);
-                iteration++;
+                context.Iteration++;
+
+                // Reset arguments
+                processSpec.Arguments = initialArguments;
+
+                for (var i = 0; i < _filters.Length; i++)
+                {
+                    await _filters[i].ProcessAsync(context, cancellationToken);
+                }
+
+                // Reset for next run
+                context.RequiresMSBuildRevaluation = false;
 
-                var fileSet = await fileSetFactory.CreateAsync(cancellationToken);
+                processSpec.EnvironmentVariables["DOTNET_WATCH_ITERATION"] = (context.Iteration + 1).ToString(CultureInfo.InvariantCulture);
 
+                var fileSet = context.FileSet;
                 if (fileSet == null)
                 {
                     _reporter.Error("Failed to find a list of files to watch");
@@ -91,10 +124,13 @@ public async Task WatchAsync(ProcessSpec processSpec, IFileSetFactory fileSetFac
                         return;
                     }
 
+                    context.ChangedFile = fileSetTask.Result;
                     if (finishedTask == processTask)
                     {
+                        // Process exited. Redo evaludation
+                        context.RequiresMSBuildRevaluation = true;
                         // Now wait for a file to change before restarting process
-                        await fileSetWatcher.GetChangedFileAsync(cancellationToken, () => _reporter.Warn("Waiting for a file to change before restarting dotnet..."));
+                        context.ChangedFile = await fileSetWatcher.GetChangedFileAsync(cancellationToken, () => _reporter.Warn("Waiting for a file to change before restarting dotnet..."));
                     }
 
                     if (!string.IsNullOrEmpty(fileSetTask.Result))

src/Tools/dotnet-watch/src/IWatchFilter.cs

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.DotNet.Watcher.Tools
+{
+    public interface IWatchFilter
+    {
+        ValueTask ProcessAsync(DotNetWatchContext context, CancellationToken cancellationToken);
+    }
+}

src/Tools/dotnet-watch/src/Internal/FileSet.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -20,6 +20,8 @@ public FileSet(IEnumerable<string> files)
 
         public int Count => _files.Count;
 
+        public static IFileSet Empty = new FileSet(Array.Empty<string>());
+
         public IEnumerator<string> GetEnumerator() => _files.GetEnumerator();
         IEnumerator IEnumerable.GetEnumerator() => _files.GetEnumerator();
     }

src/Tools/dotnet-watch/src/MSBuildEvaluationFilter.cs

@@ -0,0 +1,124 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.DotNet.Watcher.Tools
+{
+    public class MSBuildEvaluationFilter : IWatchFilter
+    {
+        // File types that require an MSBuild re-evaluation
+        private static readonly string[] _msBuildFileExtensions = new[]
+        {
+            ".csproj", ".props", ".targets", ".fsproj", ".vbproj", ".vcxproj",
+        };
+        private static readonly int[] _msBuildFileExtensionHashes = _msBuildFileExtensions
+            .Select(e => e.GetHashCode(StringComparison.OrdinalIgnoreCase))
+            .ToArray();
+
+        private readonly IFileSetFactory _factory;
+
+        private List<(string fileName, DateTime lastWriteTimeUtc)> _msbuildFileTimestamps;
+
+        public MSBuildEvaluationFilter(IFileSetFactory factory)
+        {
+            _factory = factory;
+        }
+
+        public async ValueTask ProcessAsync(DotNetWatchContext context, CancellationToken cancellationToken)
+        {
+            if (context.SuppressMSBuildIncrementalism)
+            {
+                context.RequiresMSBuildRevaluation = true;
+                context.FileSet = await _factory.CreateAsync(cancellationToken);
+                return;
+            }
+
+            if (context.Iteration == 0 || RequiresMSBuildRevaluation(context))
+            {
+                context.RequiresMSBuildRevaluation = true;
+            }
+
+            if (context.RequiresMSBuildRevaluation)
+            {
+                context.Reporter.Verbose("Evaluating dotnet-watch file set.");
+
+                context.FileSet = await _factory.CreateAsync(cancellationToken);
+                _msbuildFileTimestamps = GetMSBuildFileTimeStamps(context);
+            }
+        }
+
+        private bool RequiresMSBuildRevaluation(DotNetWatchContext context)
+        {
+            var changedFile = context.ChangedFile;
+            if (!string.IsNullOrEmpty(changedFile) && IsMsBuildFileExtension(changedFile))
+            {
+                return true;
+            }
+
+            // The filewatcher may miss changes to files. For msbuild files, we can verify that they haven't been modified
+            // since the previous iteration.
+            // We do not have a way to identify renames or new additions that the file watcher did not pick up,
+            // without performing an evaluation. We will start off by keeping it simple and comparing the timestamps
+            // of known MSBuild files from previous run. This should cover the vast majority of cases.
+
+            foreach (var (file, lastWriteTimeUtc) in _msbuildFileTimestamps)
+            {
+                if (GetLastWriteTimeUtcSafely(file) != lastWriteTimeUtc)
+                {
+                    context.Reporter.Verbose($"Re-evaluation needed due to changes in {file}.");
+
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        private List<(string fileName, DateTime lastModifiedUtc)> GetMSBuildFileTimeStamps(DotNetWatchContext context)
+        {
+            var msbuildFiles = new List<(string fileName, DateTime lastModifiedUtc)>();
+            foreach (var file in context.FileSet)
+            {
+                if (!string.IsNullOrEmpty(file) && IsMsBuildFileExtension(file))
+                {
+                    msbuildFiles.Add((file, GetLastWriteTimeUtcSafely(file)));
+                }
+            }
+
+            return msbuildFiles;
+        }
+
+        protected virtual DateTime GetLastWriteTimeUtcSafely(string file)
+        {
+            try
+            {
+                return File.GetLastWriteTimeUtc(file);
+            }
+            catch
+            {
+                return DateTime.UtcNow;
+            }
+        }
+
+        static bool IsMsBuildFileExtension(string fileName)
+        {
+            var extension = Path.GetExtension(fileName.AsSpan());
+            var hashCode = string.GetHashCode(extension, StringComparison.OrdinalIgnoreCase);
+            for (var i = 0; i < _msBuildFileExtensionHashes.Length; i++)
+            {
+                if (_msBuildFileExtensionHashes[i] == hashCode && extension.Equals(_msBuildFileExtensions[i], StringComparison.OrdinalIgnoreCase))
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+    }
+}

src/Tools/dotnet-watch/src/NoRestoreFilter.cs

@@ -0,0 +1,75 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Tools.Internal;
+
+namespace Microsoft.DotNet.Watcher.Tools
+{
+    public sealed class NoRestoreFilter : IWatchFilter
+    {
+        private bool _canUseNoRestore;
+        private string[] _noRestoreArguments;
+
+        public ValueTask ProcessAsync(DotNetWatchContext context, CancellationToken cancellationToken)
+        {
+            if (context.SuppressMSBuildIncrementalism)
+            {
+                return default;
+            }
+
+            if (context.Iteration == 0)
+            {
+                var arguments = context.ProcessSpec.Arguments;
+                _canUseNoRestore = CanUseNoRestore(arguments, context.Reporter);
+                if (_canUseNoRestore)
+                {
+                    // Create run --no-restore <other args>
+                    _noRestoreArguments = arguments.Take(1).Append("--no-restore").Concat(arguments.Skip(1)).ToArray();
+                    context.Reporter.Verbose($"No restore arguments: {string.Join(" ", _noRestoreArguments)}");
+                }
+            }
+            else if (_canUseNoRestore)
+            {
+                if (context.RequiresMSBuildRevaluation)
+                {
+                    context.Reporter.Verbose("Cannot use --no-restore since msbuild project files have changed.");
+                }
+                else
+                {
+                    context.Reporter.Verbose("Modifying command to use --no-restore");
+                    context.ProcessSpec.Arguments = _noRestoreArguments;
+                }
+            }
+
+            return default;
+        }
+
+        private static bool CanUseNoRestore(IEnumerable<string> arguments, IReporter reporter)
+        {
+            // For some well-known dotnet commands, we can pass in the --no-restore switch to avoid unnecessary restores between iterations.
+            // For now we'll support the "run" and "test" commands.
+            if (arguments.Any(a => string.Equals(a, "--no-restore", StringComparison.Ordinal)))
+            {
+                // Did the user already configure a --no-restore?
+                return false;
+            }
+
+            var dotnetCommand = arguments.FirstOrDefault();
+            if (string.Equals(dotnetCommand, "run", StringComparison.Ordinal) || string.Equals(dotnetCommand, "test", StringComparison.Ordinal))
+            {
+                reporter.Verbose("Watch command can be configured to use --no-restore.");
+                return true;
+            }
+            else
+            {
+                reporter.Verbose($"Watch command will not use --no-restore. Unsupport dotnet-command '{dotnetCommand}'.");
+                return false;
+            }
+        }
+    }
+}

src/Tools/dotnet-watch/src/Program.cs

@@ -162,8 +162,8 @@ private async Task<int> MainInternalAsync(
                 _reporter.Output("Polling file watcher is enabled");
             }
 
-            await new DotNetWatcher(reporter)
-                .WatchAsync(processInfo, fileSetFactory, cancellationToken);
+            await new DotNetWatcher(reporter, fileSetFactory)
+                .WatchAsync(processInfo, cancellationToken);
 
             return 0;
         }