[CodeCompletion] Give up fast-completion if dependent files are modified

Check if dependencies are modified since the last checking.
Dependencies:

 - Other source files in the current module
 - Dependent files collected by the dependency tracker

When:

 - If the last dependency check was over N (defaults to 5) seconds ago

Invalidate if:

 - The dependency file is missing
 - The modification time of the dependecy is greater than the last check
 - If the modification time is zero, compare the content using the file
   system from the previous completion and the current completion

rdar://problem/62336432

Author: rintaro

include/swift/IDE/CompletionInstance.h

@@ -18,6 +18,7 @@
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Chrono.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
 
@@ -38,20 +39,28 @@ makeCodeCompletionMemoryBuffer(const llvm::MemoryBuffer *origBuf,
 /// Manages \c CompilerInstance for completion like operations.
 class CompletionInstance {
   unsigned MaxASTReuseCount = 100;
+  unsigned DependencyCheckIntervalSecond = 5;
 
   std::mutex mtx;
 
   std::unique_ptr<CompilerInstance> CachedCI;
   llvm::hash_code CachedArgHash;
+  llvm::sys::TimePoint<> DependencyCheckedTimestamp;
   unsigned CachedReuseCount = 0;
 
+  void cacheCompilerInstance(std::unique_ptr<CompilerInstance> CI,
+                             llvm::hash_code ArgsHash);
+
+  bool shouldCheckDependencies() const;
+
   /// Calls \p Callback with cached \c CompilerInstance if it's usable for the
   /// specified completion request.
   /// Returns \c if the callback was called. Returns \c false if the compiler
   /// argument has changed, primary file is not the same, the \c Offset is not
   /// in function bodies, or the interface hash of the file has changed.
   bool performCachedOperationIfPossible(
       const swift::CompilerInvocation &Invocation, llvm::hash_code ArgsHash,
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
       llvm::MemoryBuffer *completionBuffer, unsigned int Offset,
       DiagnosticConsumer *DiagC,
       llvm::function_ref<void(CompilerInstance &, bool)> Callback);
@@ -69,6 +78,8 @@ class CompletionInstance {
       llvm::function_ref<void(CompilerInstance &, bool)> Callback);
 
 public:
+  void setDependencyCheckIntervalSecond(unsigned Value);
+
   /// Calls \p Callback with a \c CompilerInstance which is prepared for the
   /// second pass. \p Callback is resposible to perform the second pass on it.
   /// The \c CompilerInstance may be reused from the previous completions,

lib/IDE/CompletionInstance.cpp

@@ -18,6 +18,8 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/PrettyStackTrace.h"
 #include "swift/AST/SourceFile.h"
+#include "swift/Serialization/SerializedModuleLoader.h"
+#include "swift/ClangImporter/ClangModule.h"
 #include "swift/Basic/LangOptions.h"
 #include "swift/Basic/PrettyStackTrace.h"
 #include "swift/Basic/SourceManager.h"
@@ -28,6 +30,7 @@
 #include "swift/Subsystems.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "clang/AST/ASTContext.h"
 
 using namespace swift;
 using namespace ide;
@@ -162,10 +165,77 @@ static DeclContext *getEquivalentDeclContextFromSourceFile(DeclContext *DC,
   return newDC;
 }
 
+/// Check if any dependent files are modified since \p timestamp.
+bool areAnyDependentFilesInvalidated(CompilerInstance &CI,
+                                     llvm::vfs::FileSystem &FS,
+                                     StringRef currentFileName,
+                                     llvm::sys::TimePoint<> timestamp) {
+
+  auto isInvalidated = [&](StringRef filePath) -> bool {
+    auto stat = FS.status(filePath);
+    if (!stat)
+      // Missing.
+      return true;
+
+    auto lastModTime = stat->getLastModificationTime();
+    if (lastModTime > timestamp)
+      // Modified.
+      return true;
+
+    // If the last modification time is zero, this file is probably from a
+    // virtual file system. We need to check the content.
+    if (lastModTime == llvm::sys::TimePoint<>()) {
+      if (&CI.getFileSystem() == &FS)
+        return false;
+
+      auto oldContent = CI.getFileSystem().getBufferForFile(filePath);
+      auto newContent = FS.getBufferForFile(filePath);
+      if (!oldContent || !newContent)
+        // (unreachable?)
+        return true;
+
+      if (oldContent.get()->getBuffer() != newContent.get()->getBuffer())
+        // Different content.
+        return true;
+    }
+
+    return false;
+  };
+
+  // Check files in the current module.
+  for (FileUnit *file : CI.getMainModule()->getFiles()) {
+    StringRef filename;
+    if (auto SF = dyn_cast<SourceFile>(file))
+      filename = SF->getFilename();
+    else if (auto LF = dyn_cast<LoadedFile>(file))
+      filename = LF->getFilename();
+    else
+      continue;
+
+    // Ignore the current file and synthesized files.
+    if (filename.empty() || filename.front() == '<' ||
+        filename.equals(currentFileName))
+      continue;
+
+    if (isInvalidated(filename))
+      return true;
+  }
+
+  // Check other non-system depenencies (e.g. modules, headers).
+  for (auto &dep : CI.getDependencyTracker()->getDependencies()) {
+    if (isInvalidated(dep))
+      return true;
+  }
+
+  // All loaded module files are not modified since the timestamp.
+  return false;
+}
+
 } // namespace
 
 bool CompletionInstance::performCachedOperationIfPossible(
     const swift::CompilerInvocation &Invocation, llvm::hash_code ArgsHash,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
     llvm::MemoryBuffer *completionBuffer, unsigned int Offset,
     DiagnosticConsumer *DiagC,
     llvm::function_ref<void(CompilerInstance &, bool)> Callback) {
@@ -187,10 +257,17 @@ bool CompletionInstance::performCachedOperationIfPossible(
   auto &oldInfo = oldState->getCodeCompletionDelayedDeclState();
 
   auto &SM = CI.getSourceMgr();
-  if (SM.getIdentifierForBuffer(SM.getCodeCompletionBufferID()) !=
-      completionBuffer->getBufferIdentifier())
+  auto bufferName = completionBuffer->getBufferIdentifier();
+  if (SM.getIdentifierForBuffer(SM.getCodeCompletionBufferID()) != bufferName)
     return false;
 
+  if (shouldCheckDependencies()) {
+    if (areAnyDependentFilesInvalidated(CI, *FileSystem, bufferName,
+                                        DependencyCheckedTimestamp))
+      return false;
+    DependencyCheckedTimestamp = std::chrono::system_clock::now();
+  }
+
   // Parse the new buffer into temporary SourceFile.
   SourceManager tmpSM;
   auto tmpBufferID = tmpSM.addMemBufferCopy(completionBuffer);
@@ -263,8 +340,7 @@ bool CompletionInstance::performCachedOperationIfPossible(
         completionBuffer->getBuffer().slice(startOffset, endOffset);
     auto newOffset = Offset - startOffset;
 
-    newBufferID = SM.addMemBufferCopy(sourceText,
-                                      completionBuffer->getBufferIdentifier());
+    newBufferID = SM.addMemBufferCopy(sourceText, bufferName);
     SM.openVirtualFile(SM.getLocForBufferStart(newBufferID),
                        tmpSM.getDisplayNameForLoc(startLoc),
                        tmpSM.getLineAndColumn(startLoc).first - 1);
@@ -310,8 +386,7 @@ bool CompletionInstance::performCachedOperationIfPossible(
       endOffset = tmpSM.getLocOffsetInBuffer(endLoc, tmpBufferID);
       sourceText = sourceText.slice(0, endOffset);
     }
-    newBufferID = SM.addMemBufferCopy(sourceText,
-                                      completionBuffer->getBufferIdentifier());
+    newBufferID = SM.addMemBufferCopy(sourceText, bufferName);
     SM.setCodeCompletionPoint(newBufferID, Offset);
 
     // Create a new module and a source file using the current AST context.
@@ -369,7 +444,15 @@ bool CompletionInstance::performNewOperation(
     llvm::function_ref<void(CompilerInstance &, bool)> Callback) {
   llvm::PrettyStackTraceString trace("While performing new completion");
 
+  auto isCachedCompletionRequested = ArgsHash.hasValue();
+
   auto TheInstance = std::make_unique<CompilerInstance>();
+
+  // Track dependencies in fast-completion mode to invalidate the compiler
+  // instance if any dependent files are modified.
+  if (isCachedCompletionRequested)
+    TheInstance->createDependencyTracker(false);
+
   {
     auto &CI = *TheInstance;
     if (DiagC)
@@ -407,15 +490,34 @@ bool CompletionInstance::performNewOperation(
     Callback(CI, /*reusingASTContext=*/false);
   }
 
-  if (ArgsHash.hasValue()) {
-    CachedCI = std::move(TheInstance);
-    CachedArgHash = *ArgsHash;
-    CachedReuseCount = 0;
-  }
+  // Cache the compiler instance if fast completion is enabled.
+  if (isCachedCompletionRequested)
+    cacheCompilerInstance(std::move(TheInstance), *ArgsHash);
 
   return true;
 }
 
+void CompletionInstance::cacheCompilerInstance(
+    std::unique_ptr<CompilerInstance> CI, llvm::hash_code ArgsHash) {
+  CachedCI = std::move(CI);
+  CachedArgHash = ArgsHash;
+  auto now = std::chrono::system_clock::now();
+  DependencyCheckedTimestamp = now;
+  CachedReuseCount = 0;
+}
+
+bool CompletionInstance::shouldCheckDependencies() const {
+  assert(CachedCI);
+  using namespace std::chrono;
+  auto now = system_clock::now();
+  return DependencyCheckedTimestamp + seconds(DependencyCheckIntervalSecond) < now;
+}
+
+void CompletionInstance::setDependencyCheckIntervalSecond(unsigned Value) {
+  std::lock_guard<std::mutex> lock(mtx);
+  DependencyCheckIntervalSecond = Value;
+}
+
 bool swift::ide::CompletionInstance::performOperation(
     swift::CompilerInvocation &Invocation, llvm::ArrayRef<const char *> Args,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
@@ -451,8 +553,9 @@ bool swift::ide::CompletionInstance::performOperation(
     // the cached completion instance.
     std::lock_guard<std::mutex> lock(mtx);
 
-    if (performCachedOperationIfPossible(Invocation, ArgsHash, completionBuffer,
-                                         Offset, DiagC, Callback))
+    if (performCachedOperationIfPossible(Invocation, ArgsHash, FileSystem,
+                                         completionBuffer, Offset, DiagC,
+                                         Callback))
       return true;
 
     if (performNewOperation(ArgsHash, Invocation, FileSystem, completionBuffer,

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework/Headers/ClangFW.h

@@ -0,0 +1,2 @@
+
+#import <ClangFW/Funcs.h>

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework/Headers/Funcs.h

@@ -0,0 +1,4 @@
+#ifndef CLANGFW_FUNCS_H
+#define CLANGFW_FUNCS_H
+int clangFWFunc();
+#endif

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework/Modules/module.modulemap

@@ -0,0 +1,4 @@
+framework module ClangFW {
+  umbrella header "ClangFW.h"
+  export *
+}

test/SourceKit/CodeComplete/Inputs/checkdeps/ClangFW.framework_mod/Headers/Funcs.h

@@ -0,0 +1,5 @@
+#ifndef CLANGFW_FUNCS_H
+#define CLANGFW_FUNCS_H
+int clangFWFunc_mod();
+#endif
+

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject/Bridging.h

@@ -0,0 +1 @@
+#import "LocalCFunc.h"

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject/Library.swift

@@ -0,0 +1 @@
+func localSwiftFunc() -> Int {}

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject/LocalCFunc.h

@@ -0,0 +1,6 @@
+#ifndef MYPROJECT_LOCALCFUNC_H
+#define MYPROJECT_LOCALCFUNC_H
+
+int localClangFunc();
+
+#endif

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject_mod/Library.swift

@@ -0,0 +1 @@
+func localSwiftFunc_mod() -> Int {}

test/SourceKit/CodeComplete/Inputs/checkdeps/MyProject_mod/LocalCFunc.h

@@ -0,0 +1,7 @@
+#ifndef MYPROJECT_LOCALCFUNC_H
+#define MYPROJECT_LOCALCFUNC_H
+
+int localClangFunc_mod();
+
+#endif
+

test/SourceKit/CodeComplete/Inputs/checkdeps/SwiftFW_src/Funcs.swift

@@ -0,0 +1,2 @@
+public func swiftFWFunc() -> Int { return 1 }
+

test/SourceKit/CodeComplete/Inputs/checkdeps/SwiftFW_src_mod/Funcs.swift

@@ -0,0 +1 @@
+public func swiftFWFunc_mod() -> Int { return 1 }

test/SourceKit/CodeComplete/Inputs/checkdeps/test.swift

@@ -0,0 +1,5 @@
+import Foundation
+import ClangFW // < 500 headers framework
+func foo() {
+  /* HERE */
+}

test/SourceKit/CodeComplete/complete_checkdeps_bridged.swift

@@ -0,0 +1,72 @@
+import ClangFW
+import SwiftFW
+
+func foo() {
+  /*HERE*/
+}
+
+// RUN: %empty-directory(%t/Frameworks)
+// RUN: %empty-directory(%t/MyProject)
+
+// RUN: COMPILER_ARGS=( \
+// RUN:   -target %target-triple \
+// RUN:   -module-name MyProject \
+// RUN:   -F %t/Frameworks \
+// RUN:   -I %t/MyProject \
+// RUN:   -import-objc-header %t/MyProject/Bridging.h \
+// RUN:   %t/MyProject/Library.swift \
+// RUN:   %s \
+// RUN: )
+// RUN: INPUT_DIR=%S/Inputs/checkdeps
+// RUN: DEPCHECK_INTERVAL=1
+// RUN: SLEEP_TIME=2
+
+// RUN: cp -R $INPUT_DIR/MyProject %t/
+// RUN: cp -R $INPUT_DIR/ClangFW.framework %t/Frameworks/
+// RUN: %empty-directory(%t/Frameworks/SwiftFW.framework/Modules/SwiftFW.swiftmodule)
+// RUN: %target-swift-frontend -emit-module -module-name SwiftFW -o %t/Frameworks/SwiftFW.framework/Modules/SwiftFW.swiftmodule/%target-swiftmodule-name $INPUT_DIR/SwiftFW_src/Funcs.swift
+
+// RUN: %sourcekitd-test \
+// RUN:   -req=global-config -completion-check-dependency-interval ${DEPCHECK_INTERVAL} == \
+
+// RUN:   -shell -- echo "### Initial" == \
+// RUN:   -req=complete -pos=5:3 %s -- ${COMPILER_ARGS[@]} == \
+
+// RUN:   -shell -- echo '### Modify bridging header library file' == \
+// RUN:   -shell -- cp -R $INPUT_DIR/MyProject_mod/LocalCFunc.h %t/MyProject/ == \
+// RUN:   -shell -- sleep $SLEEP_TIME == \
+// RUN:   -req=complete -pos=5:3 %s -- ${COMPILER_ARGS[@]} == \
+
+// RUN:   -shell -- echo '### Fast completion' == \
+// RUN:   -shell -- sleep $SLEEP_TIME == \
+// RUN:   -req=complete -pos=5:3 %s -- ${COMPILER_ARGS[@]} \
+
+// RUN:   | %FileCheck %s
+
+
+// CHECK-LABEL: ### Initial
+// CHECK: key.results: [
+// CHECK-DAG: key.description: "clangFWFunc()"
+// CHECK-DAG: key.description: "swiftFWFunc()"
+// CHECK-DAG: key.description: "localClangFunc()"
+// CHECK-DAG: key.description: "localSwiftFunc()"
+// CHECK: ]
+// CHECK-NOT: key.reusingastcontext: 1
+
+// CHECK-LABEL: ### Modify bridging header library file
+// CHECK: key.results: [
+// CHECK-DAG: key.description: "clangFWFunc()"
+// CHECK-DAG: key.description: "swiftFWFunc()"
+// CHECK-DAG: key.description: "localClangFunc_mod()"
+// CHECK-DAG: key.description: "localSwiftFunc()"
+// CHECK: ]
+// CHECK-NOT: key.reusingastcontext: 1
+
+// CHECK-LABEL: ### Fast completion
+// CHECK: key.results: [
+// CHECK-DAG: key.description: "clangFWFunc()"
+// CHECK-DAG: key.description: "swiftFWFunc()"
+// CHECK-DAG: key.description: "localClangFunc_mod()"
+// CHECK-DAG: key.description: "localSwiftFunc()"
+// CHECK: ]
+// CHECK: key.reusingastcontext: 1