Merge pull request #41081 from rintaro/sourcekit-compile-dependency

[SourceKit] Add automatic dependency check to 'compile' request

Author: rintaro

include/swift/IDE/CompileInstance.h

@@ -36,15 +36,20 @@ class CompileInstance {
 
   struct Options {
     unsigned MaxASTReuseCount = 100;
+    unsigned DependencyCheckIntervalSecond = 5;
   } Opts;
 
   std::mutex mtx;
 
   std::unique_ptr<CompilerInstance> CI;
   llvm::hash_code CachedArgHash;
   std::atomic<bool> CachedCIInvalidated;
+  llvm::sys::TimePoint<> DependencyCheckedTimestamp;
+  llvm::StringMap<llvm::hash_code> InMemoryDependencyHash;
   unsigned CachedReuseCount;
 
+  bool shouldCheckDependencies() const;
+
   /// Perform cached sema. Returns \c true if the CI is not reusable.
   bool performCachedSemaIfPossible(DiagnosticConsumer *DiagC);
 
@@ -54,8 +59,8 @@ class CompileInstance {
                  DiagnosticConsumer *DiagC);
 
   /// Perform Parse and Sema, potentially CI from previous compilation is
-  /// reused.
-  void performSema(llvm::ArrayRef<const char *> Args,
+  /// reused. Returns \c true if there was any error.
+  bool performSema(llvm::ArrayRef<const char *> Args,
                    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
                    DiagnosticConsumer *DiagC,
                    std::shared_ptr<std::atomic<bool>> CancellationFlag);

lib/IDE/CMakeLists.txt

@@ -8,6 +8,7 @@ add_swift_host_library(swiftIDE STATIC
   CompileInstance.cpp
   CompletionInstance.cpp
   ConformingMethodList.cpp
+  DependencyChecking.cpp
   ExprContextAnalysis.cpp
   Formatting.cpp
   FuzzyStringMatcher.cpp

lib/IDE/CompileInstance.cpp

@@ -12,6 +12,7 @@
 
 #include "swift/IDE/CompileInstance.h"
 
+#include "DependencyChecking.h"
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/Module.h"
@@ -204,6 +205,16 @@ bool CompileInstance::performCachedSemaIfPossible(DiagnosticConsumer *DiagC) {
 
   SourceManager &SM = CI->getSourceMgr();
   auto FS = SM.getFileSystem();
+
+  if (shouldCheckDependencies()) {
+    if (areAnyDependentFilesInvalidated(*CI, *FS, /*excludeBufferID=*/None,
+                                        DependencyCheckedTimestamp,
+                                        InMemoryDependencyHash)) {
+      return true;
+    }
+    DependencyCheckedTimestamp = std::chrono::system_clock::now();
+  }
+
   SourceManager tmpSM(FS);
 
   // Collect modified function body.
@@ -230,9 +241,6 @@ bool CompileInstance::setupCI(
     llvm::ArrayRef<const char *> origArgs,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fileSystem,
     DiagnosticConsumer *diagC) {
-  CI->addDiagnosticConsumer(diagC);
-  SWIFT_DEFER { CI->removeDiagnosticConsumer(diagC); };
-
   auto &Diags = CI->getDiags();
 
   SmallVector<const char *, 16> args;
@@ -271,6 +279,11 @@ bool CompileInstance::setupCI(
   /// Declare the frontend to be used for multiple compilations.
   invocation.getFrontendOptions().ReuseFrontendForMutipleCompilations = true;
 
+  // Enable dependency trakcing (excluding system modules) to invalidate the
+  // compiler instance if any dependent files are modified.
+  invocation.getFrontendOptions().IntermoduleDependencyTracking =
+      IntermoduleDepTrackingMode::ExcludeSystem;
+
   std::string InstanceSetupError;
   if (CI->setup(invocation, InstanceSetupError)) {
     assert(Diags.hadAnyError());
@@ -280,7 +293,7 @@ bool CompileInstance::setupCI(
   return true;
 }
 
-void CompileInstance::performSema(
+bool CompileInstance::performSema(
     llvm::ArrayRef<const char *> Args,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fileSystem,
     DiagnosticConsumer *DiagC,
@@ -297,30 +310,33 @@ void CompileInstance::performSema(
     if (!performCachedSemaIfPossible(DiagC)) {
       // If we compileted cacehd Sema operation. We're done.
       ++CachedReuseCount;
-      return;
+      return CI->getDiags().hadAnyError();
     }
   }
 
   // Performing a new operation. Reset the compiler instance.
-  CachedArgHash = hash_code();
-  CachedReuseCount = 0;
   CI = std::make_unique<CompilerInstance>();
+  CI->addDiagnosticConsumer(DiagC);
 
   if (!setupCI(Args, fileSystem, DiagC)) {
     // Failed to setup the CI.
     CI.reset();
-    return;
+    return true;
   }
 
-  CI->addDiagnosticConsumer(DiagC);
-  SWIFT_DEFER { CI->removeDiagnosticConsumer(DiagC); };
-
-  // CI is potentially reusable.
+  // Remember cache related information.
+  DependencyCheckedTimestamp = std::chrono::system_clock::now();
   CachedArgHash = ArgsHash;
+  CachedReuseCount = 0;
+  InMemoryDependencyHash.clear();
+  cacheDependencyHashIfNeeded(*CI, /*excludeBufferID=*/None,
+                              InMemoryDependencyHash);
 
   // Perform!
   CI->getASTContext().CancellationFlag = CancellationFlag;
   CI->performSema();
+  CI->removeDiagnosticConsumer(DiagC);
+  return CI->getDiags().hadAnyError();
 }
 
 bool CompileInstance::performCompile(
@@ -334,14 +350,24 @@ bool CompileInstance::performCompile(
   if (CancellationFlag && CancellationFlag->load(std::memory_order_relaxed))
     return true;
 
-  performSema(Args, fileSystem, DiagC, CancellationFlag);
-  if (CI->getDiags().hadAnyError())
+  if (performSema(Args, fileSystem, DiagC, CancellationFlag))
     return true;
 
   // Cancellation check after Sema.
   if (CI->isCancellationRequested())
     return true;
 
+  CI->addDiagnosticConsumer(DiagC);
+  SWIFT_DEFER { CI->removeDiagnosticConsumer(DiagC); };
   int ReturnValue = 0;
   return performCompileStepsPostSema(*CI, ReturnValue, /*observer=*/nullptr);
 }
+
+bool CompileInstance::shouldCheckDependencies() const {
+  assert(CI);
+  using namespace std::chrono;
+  auto now = system_clock::now();
+  auto threshold =
+      DependencyCheckedTimestamp + seconds(Opts.DependencyCheckIntervalSecond);
+  return threshold <= now;
+}

lib/IDE/CompletionInstance.cpp

@@ -12,6 +12,7 @@
 
 #include "swift/IDE/CompletionInstance.h"
 
+#include "DependencyChecking.h"
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsFrontend.h"
@@ -170,115 +171,6 @@ static DeclContext *getEquivalentDeclContextFromSourceFile(DeclContext *DC,
   return newDC;
 }
 
-/// For each dependency file in \p CI, run \p callback until the callback
-/// returns \c true. Returns \c true if any callback call returns \c true, \c
-/// false otherwise.
-static bool
-forEachDependencyUntilTrue(CompilerInstance &CI, unsigned excludeBufferID,
-                           llvm::function_ref<bool(StringRef)> callback) {
-  // Check files in the current module.
-  for (FileUnit *file : CI.getMainModule()->getFiles()) {
-    StringRef filename;
-    if (auto SF = dyn_cast<SourceFile>(file)) {
-      if (SF->getBufferID() == excludeBufferID)
-        continue;
-      filename = SF->getFilename();
-    } else if (auto LF = dyn_cast<LoadedFile>(file))
-      filename = LF->getFilename();
-    else
-      continue;
-
-    // Ignore synthesized files.
-    if (filename.empty() || filename.front() == '<')
-      continue;
-
-    if (callback(filename))
-      return true;
-  }
-
-  // Check other non-system depenencies (e.g. modules, headers).
-  for (auto &dep : CI.getDependencyTracker()->getDependencies()) {
-    if (callback(dep))
-      return true;
-  }
-  for (auto dep : CI.getDependencyTracker()->getIncrementalDependencyPaths()) {
-    if (callback(dep))
-      return true;
-  }
-
-  return false;
-}
-
-/// Collect hash codes of the dependencies into \c Map.
-static void cacheDependencyHashIfNeeded(CompilerInstance &CI,
-                                        unsigned excludeBufferID,
-                                        llvm::StringMap<llvm::hash_code> &Map) {
-  auto &FS = CI.getFileSystem();
-  forEachDependencyUntilTrue(
-      CI, excludeBufferID, [&](StringRef filename) {
-        if (Map.count(filename))
-          return false;
-
-        auto stat = FS.status(filename);
-        if (!stat)
-          return false;
-
-        // We will check the hash only if the modification time of the dependecy
-        // is zero. See 'areAnyDependentFilesInvalidated() below'.
-        if (stat->getLastModificationTime() != llvm::sys::TimePoint<>())
-          return false;
-
-        auto buf = FS.getBufferForFile(filename);
-        Map[filename] = llvm::hash_value(buf.get()->getBuffer());
-        return false;
-      });
-}
-
-/// Check if any dependent files are modified since \p timestamp.
-static bool areAnyDependentFilesInvalidated(
-    CompilerInstance &CI, llvm::vfs::FileSystem &FS,
-    unsigned excludeBufferID, llvm::sys::TimePoint<> timestamp,
-    llvm::StringMap<llvm::hash_code> &Map) {
-
-  return forEachDependencyUntilTrue(
-      CI, excludeBufferID, [&](StringRef filePath) {
-        auto stat = FS.status(filePath);
-        if (!stat)
-          // Missing.
-          return true;
-
-        auto lastModTime = stat->getLastModificationTime();
-        if (lastModTime > timestamp)
-          // Modified.
-          return true;
-
-        // If the last modification time is zero, this file is probably from a
-        // virtual file system. We need to check the content.
-        if (lastModTime == llvm::sys::TimePoint<>()) {
-          // Get the hash code of the last content.
-          auto oldHashEntry = Map.find(filePath);
-          if (oldHashEntry == Map.end())
-            // Unreachable? Not virtual in old filesystem, but virtual in new
-            // one.
-            return true;
-          auto oldHash = oldHashEntry->second;
-
-          // Calculate the hash code of the current content.
-          auto newContent = FS.getBufferForFile(filePath);
-          if (!newContent)
-            // Unreachable? stat succeeded, but coundn't get the content.
-            return true;
-
-          auto newHash = llvm::hash_value(newContent.get()->getBuffer());
-
-          if (oldHash != newHash)
-            return true;
-        }
-
-        return false;
-      });
-}
-
 } // namespace
 
 bool CompletionInstance::performCachedOperationIfPossible(