Merge pull request swiftlang#31547 from rintaro/5.3-ide-completion-fastcheckdep-rdar62336432

[5.3][CodeCompletion] Give up fast-completion if dependent files are modified

Author: rintaro

include/swift/IDE/CompletionInstance.h

@@ -18,6 +18,7 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Chrono.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
 
@@ -38,20 +39,30 @@ makeCodeCompletionMemoryBuffer(const llvm::MemoryBuffer *origBuf,
 /// Manages \c CompilerInstance for completion like operations.
 class CompletionInstance {
   unsigned MaxASTReuseCount = 100;
+  unsigned DependencyCheckIntervalSecond = 5;
 
   std::mutex mtx;
 
   std::unique_ptr<CompilerInstance> CachedCI;
+  ModuleDecl *CurrentModule = nullptr;
   llvm::hash_code CachedArgHash;
+  llvm::sys::TimePoint<> DependencyCheckedTimestamp;
+  llvm::StringMap<llvm::hash_code> InMemoryDependencyHash;
   unsigned CachedReuseCount = 0;
 
+  void cacheCompilerInstance(std::unique_ptr<CompilerInstance> CI,
+                             llvm::hash_code ArgsHash);
+
+  bool shouldCheckDependencies() const;
+
   /// Calls \p Callback with cached \c CompilerInstance if it's usable for the
   /// specified completion request.
   /// Returns \c if the callback was called. Returns \c false if the compiler
   /// argument has changed, primary file is not the same, the \c Offset is not
   /// in function bodies, or the interface hash of the file has changed.
   bool performCachedOperationIfPossible(
       const swift::CompilerInvocation &Invocation, llvm::hash_code ArgsHash,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
       llvm::MemoryBuffer *completionBuffer, unsigned int Offset,
       DiagnosticConsumer *DiagC,
       llvm::function_ref<void(CompilerInstance &, bool)> Callback);
@@ -69,6 +80,8 @@ class CompletionInstance {
       llvm::function_ref<void(CompilerInstance &, bool)> Callback);
 
 public:
+  void setDependencyCheckIntervalSecond(unsigned Value);
+
   /// Calls \p Callback with a \c CompilerInstance which is prepared for the
   /// second pass. \p Callback is resposible to perform the second pass on it.
   /// The \c CompilerInstance may be reused from the previous completions,

lib/IDE/CompletionInstance.cpp

@@ -21,11 +21,14 @@
 #include "swift/Basic/LangOptions.h"
 #include "swift/Basic/PrettyStackTrace.h"
 #include "swift/Basic/SourceManager.h"
+#include "swift/ClangImporter/ClangModule.h"
 #include "swift/Driver/FrontendUtil.h"
 #include "swift/Frontend/Frontend.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/PersistentParserState.h"
+#include "swift/Serialization/SerializedModuleLoader.h"
 #include "swift/Subsystems.h"
+#include "clang/AST/ASTContext.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/Support/MemoryBuffer.h"
 
@@ -162,10 +165,118 @@ static DeclContext *getEquivalentDeclContextFromSourceFile(DeclContext *DC,
   return newDC;
 }
 
+/// For each dependency file in \p CI, run \p callback until the callback
+/// returns \c true. Returns \c true if any callback call returns \c true, \c
+/// false otherwise.
+static bool
+forEachDependencyUntilTrue(CompilerInstance &CI, ModuleDecl *CurrentModule,
+                           unsigned excludeBufferID,
+                           llvm::function_ref<bool(StringRef)> callback) {
+  // Check files in the current module.
+  for (FileUnit *file : CurrentModule->getFiles()) {
+    StringRef filename;
+    if (auto SF = dyn_cast<SourceFile>(file)) {
+      if (SF->getBufferID() == excludeBufferID)
+        continue;
+      filename = SF->getFilename();
+    } else if (auto LF = dyn_cast<LoadedFile>(file))
+      filename = LF->getFilename();
+    else
+      continue;
+
+    // Ignore synthesized files.
+    if (filename.empty() || filename.front() == '<')
+      continue;
+
+    if (callback(filename))
+      return true;
+  }
+
+  // Check other non-system depenencies (e.g. modules, headers).
+  for (auto &dep : CI.getDependencyTracker()->getDependencies()) {
+    if (callback(dep))
+      return true;
+  }
+
+  return false;
+}
+
+/// Collect hash codes of the dependencies into \c Map.
+static void cacheDependencyHashIfNeeded(CompilerInstance &CI,
+                                        ModuleDecl *CurrentModule,
+                                        unsigned excludeBufferID,
+                                        llvm::StringMap<llvm::hash_code> &Map) {
+  auto &FS = CI.getFileSystem();
+  forEachDependencyUntilTrue(
+      CI, CurrentModule, excludeBufferID, [&](StringRef filename) {
+        if (Map.count(filename))
+          return false;
+
+        auto stat = FS.status(filename);
+        if (!stat)
+          return false;
+
+        // We will check the hash only if the modification time of the dependecy
+        // is zero. See 'areAnyDependentFilesInvalidated() below'.
+        if (stat->getLastModificationTime() != llvm::sys::TimePoint<>())
+          return false;
+
+        auto buf = FS.getBufferForFile(filename);
+        Map[filename] = llvm::hash_value(buf.get()->getBuffer());
+        return false;
+      });
+}
+
+/// Check if any dependent files are modified since \p timestamp.
+static bool areAnyDependentFilesInvalidated(
+    CompilerInstance &CI, ModuleDecl *CurrentModule, llvm::vfs::FileSystem &FS,
+    unsigned excludeBufferID, llvm::sys::TimePoint<> timestamp,
+    llvm::StringMap<llvm::hash_code> &Map) {
+
+  return forEachDependencyUntilTrue(
+      CI, CurrentModule, excludeBufferID, [&](StringRef filePath) {
+        auto stat = FS.status(filePath);
+        if (!stat)
+          // Missing.
+          return true;
+
+        auto lastModTime = stat->getLastModificationTime();
+        if (lastModTime > timestamp)
+          // Modified.
+          return true;
+
+        // If the last modification time is zero, this file is probably from a
+        // virtual file system. We need to check the content.
+        if (lastModTime == llvm::sys::TimePoint<>()) {
+          // Get the hash code of the last content.
+          auto oldHashEntry = Map.find(filePath);
+          if (oldHashEntry == Map.end())
+            // Unreachable? Not virtual in old filesystem, but virtual in new
+            // one.
+            return true;
+          auto oldHash = oldHashEntry->second;
+
+          // Calculate the hash code of the current content.
+          auto newContent = FS.getBufferForFile(filePath);
+          if (!newContent)
+            // Unreachable? stat succeeded, but coundn't get the content.
+            return true;
+
+          auto newHash = llvm::hash_value(newContent.get()->getBuffer());
+
+          if (oldHash != newHash)
+            return true;
+        }
+
+        return false;
+      });
+}
+
 } // namespace
 
 bool CompletionInstance::performCachedOperationIfPossible(
     const swift::CompilerInvocation &Invocation, llvm::hash_code ArgsHash,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
     llvm::MemoryBuffer *completionBuffer, unsigned int Offset,
     DiagnosticConsumer *DiagC,
     llvm::function_ref<void(CompilerInstance &, bool)> Callback) {
@@ -187,10 +298,18 @@ bool CompletionInstance::performCachedOperationIfPossible(
   auto &oldInfo = oldState->getCodeCompletionDelayedDeclState();
 
   auto &SM = CI.getSourceMgr();
-  if (SM.getIdentifierForBuffer(SM.getCodeCompletionBufferID()) !=
-      completionBuffer->getBufferIdentifier())
+  auto bufferName = completionBuffer->getBufferIdentifier();
+  if (SM.getIdentifierForBuffer(SM.getCodeCompletionBufferID()) != bufferName)
     return false;
 
+  if (shouldCheckDependencies()) {
+    if (areAnyDependentFilesInvalidated(
+            CI, CurrentModule, *FileSystem, SM.getCodeCompletionBufferID(),
+            DependencyCheckedTimestamp, InMemoryDependencyHash))
+      return false;
+    DependencyCheckedTimestamp = std::chrono::system_clock::now();
+  }
+
   // Parse the new buffer into temporary SourceFile.
   SourceManager tmpSM;
   auto tmpBufferID = tmpSM.addMemBufferCopy(completionBuffer);
@@ -265,8 +384,7 @@ bool CompletionInstance::performCachedOperationIfPossible(
         completionBuffer->getBuffer().slice(startOffset, endOffset);
     auto newOffset = Offset - startOffset;
 
-    newBufferID = SM.addMemBufferCopy(sourceText,
-                                      completionBuffer->getBufferIdentifier());
+    newBufferID = SM.addMemBufferCopy(sourceText, bufferName);
     SM.openVirtualFile(SM.getLocForBufferStart(newBufferID),
                        tmpSM.getDisplayNameForLoc(startLoc),
                        tmpSM.getLineAndColumn(startLoc).first - 1);
@@ -312,8 +430,7 @@ bool CompletionInstance::performCachedOperationIfPossible(
       endOffset = tmpSM.getLocOffsetInBuffer(endLoc, tmpBufferID);
       sourceText = sourceText.slice(0, endOffset);
     }
-    newBufferID = SM.addMemBufferCopy(sourceText,
-                                      completionBuffer->getBufferIdentifier());
+    newBufferID = SM.addMemBufferCopy(sourceText, bufferName);
     SM.setCodeCompletionPoint(newBufferID, Offset);
 
     // Create a new module and a source file using the current AST context.
@@ -334,6 +451,7 @@ bool CompletionInstance::performCachedOperationIfPossible(
     performImportResolution(*newSF);
     bindExtensions(*newSF);
 
+    CurrentModule = newM;
     traceDC = newM;
 #ifndef NDEBUG
     const auto *reparsedState = newSF->getDelayedParserState();
@@ -360,6 +478,8 @@ bool CompletionInstance::performCachedOperationIfPossible(
   }
 
   CachedReuseCount += 1;
+  cacheDependencyHashIfNeeded(CI, CurrentModule, SM.getCodeCompletionBufferID(),
+                              InMemoryDependencyHash);
 
   return true;
 }
@@ -372,7 +492,15 @@ bool CompletionInstance::performNewOperation(
     llvm::function_ref<void(CompilerInstance &, bool)> Callback) {
   llvm::PrettyStackTraceString trace("While performing new completion");
 
+  auto isCachedCompletionRequested = ArgsHash.hasValue();
+
   auto TheInstance = std::make_unique<CompilerInstance>();
+
+  // Track dependencies in fast-completion mode to invalidate the compiler
+  // instance if any dependent files are modified.
+  if (isCachedCompletionRequested)
+    TheInstance->createDependencyTracker(false);
+
   {
     auto &CI = *TheInstance;
     if (DiagC)
@@ -410,15 +538,41 @@ bool CompletionInstance::performNewOperation(
     Callback(CI, /*reusingASTContext=*/false);
   }
 
-  if (ArgsHash.hasValue()) {
-    CachedCI = std::move(TheInstance);
-    CachedArgHash = *ArgsHash;
-    CachedReuseCount = 0;
-  }
+  // Cache the compiler instance if fast completion is enabled.
+  if (isCachedCompletionRequested)
+    cacheCompilerInstance(std::move(TheInstance), *ArgsHash);
 
   return true;
 }
 
+void CompletionInstance::cacheCompilerInstance(
+    std::unique_ptr<CompilerInstance> CI, llvm::hash_code ArgsHash) {
+  CachedCI = std::move(CI);
+  CurrentModule = CachedCI->getMainModule();
+  CachedArgHash = ArgsHash;
+  auto now = std::chrono::system_clock::now();
+  DependencyCheckedTimestamp = now;
+  CachedReuseCount = 0;
+  InMemoryDependencyHash.clear();
+  cacheDependencyHashIfNeeded(
+      *CachedCI, CurrentModule,
+      CachedCI->getASTContext().SourceMgr.getCodeCompletionBufferID(),
+      InMemoryDependencyHash);
+}
+
+bool CompletionInstance::shouldCheckDependencies() const {
+  assert(CachedCI);
+  using namespace std::chrono;
+  auto now = system_clock::now();
+  return DependencyCheckedTimestamp + seconds(DependencyCheckIntervalSecond) <
+         now;
+}
+
+void CompletionInstance::setDependencyCheckIntervalSecond(unsigned Value) {
+  std::lock_guard<std::mutex> lock(mtx);
+  DependencyCheckIntervalSecond = Value;
+}
+
 bool swift::ide::CompletionInstance::performOperation(
     swift::CompilerInvocation &Invocation, llvm::ArrayRef<const char *> Args,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
@@ -454,8 +608,9 @@ bool swift::ide::CompletionInstance::performOperation(
     // the cached completion instance.
     std::lock_guard<std::mutex> lock(mtx);
 
-    if (performCachedOperationIfPossible(Invocation, ArgsHash, completionBuffer,
-                                         Offset, DiagC, Callback))
+    if (performCachedOperationIfPossible(Invocation, ArgsHash, FileSystem,
+                                         completionBuffer, Offset, DiagC,
+                                         Callback))
       return true;
 
     if (performNewOperation(ArgsHash, Invocation, FileSystem, completionBuffer,

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework/Headers/ClangFW.h

@@ -0,0 +1,2 @@
+
+#import <ClangFW/Funcs.h>

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework/Headers/Funcs.h

@@ -0,0 +1,4 @@
+#ifndef CLANGFW_FUNCS_H
+#define CLANGFW_FUNCS_H
+int clangFWFunc();
+#endif

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework/Modules/module.modulemap

@@ -0,0 +1,4 @@
+framework module ClangFW {
+  umbrella header "ClangFW.h"
+  export *
+}

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework_mod/Headers/Funcs.h

@@ -0,0 +1,5 @@
+#ifndef CLANGFW_FUNCS_H
+#define CLANGFW_FUNCS_H
+int clangFWFunc_mod();
+#endif
+

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject/Bridging.h

@@ -0,0 +1 @@
+#import "LocalCFunc.h"

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject/Library.swift

@@ -0,0 +1,5 @@
+func localSwiftFunc() -> Int {}
+
+struct MyStruct {
+  func myStructMethod() {}
+}

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject/LibraryExt.swift

@@ -0,0 +1,3 @@
+extension MyStruct {
+  func extensionMethod() {}
+}

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject/LocalCFunc.h

@@ -0,0 +1,6 @@
+#ifndef MYPROJECT_LOCALCFUNC_H
+#define MYPROJECT_LOCALCFUNC_H
+
+int localClangFunc();
+
+#endif

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject_mod/Library.swift

@@ -0,0 +1,5 @@
+func localSwiftFunc_mod() -> Int {}
+
+struct MyStruct {
+  func myStructMethod_mod() {}
+}

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject_mod/LocalCFunc.h

@@ -0,0 +1,7 @@
+#ifndef MYPROJECT_LOCALCFUNC_H
+#define MYPROJECT_LOCALCFUNC_H
+
+int localClangFunc_mod();
+
+#endif
+

test/SourceKit/CodeComplete/Inputs/checkdeps/SwiftFW_src/Funcs.swift

@@ -0,0 +1,2 @@
+public func swiftFWFunc() -> Int { return 1 }
+

test/SourceKit/CodeComplete/Inputs/checkdeps/SwiftFW_src_mod/Funcs.swift

@@ -0,0 +1 @@
+public func swiftFWFunc_mod() -> Int { return 1 }

test/SourceKit/CodeComplete/Inputs/checkdeps/test.swift

@@ -0,0 +1,5 @@
+import Foundation
+import ClangFW // < 500 headers framework
+func foo() {
+  /* HERE */
+}