Merge pull request swiftlang#37364 from rintaro/sourcekit-syntaxinfo-rdar77665805

[SourceKit] Update SyntaxInfo but with lazy parsing in "edit" request

Author: rintaro

test/SourceKit/CodeExpand/code-expand-rdar77665805.swift

@@ -0,0 +1,18 @@
+enum E { case foo, bar }
+func foo(x: (E) -> Void) {}
+func test() {
+  foo(x: <#T##(E) -> Void#>)
+}
+
+// RUN: %sourcekitd-test \
+// RUN:   -req=open %s -- %s == \
+// RUN:   -req=edit -offset=0 -length=53 -replace="" -req-opts=enablesyntaxmap=0,enablesubstructure=0,enablediagnostics=0 %s -- %s == \
+// RUN:   -req=expand-placeholder -offset=23 -length=18 %s \
+// RUN: | %FileCheck %s
+
+// CHECK: {
+// CHECK:   key.offset: 19,
+// CHECK:   key.length: 23,
+// CHECK:   key.sourcetext: " { <#E#> in\n<#code#>\n}"
+// CHECK: }
+

tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

@@ -677,7 +677,8 @@ class SwiftDocumentSyntaxInfo {
   std::string PrimaryFile;
   /// Whether or not the AST stored in the source file is up-to-date or just an
   /// artifact of incremental syntax parsing
-  bool HasUpToDateAST;
+  bool IncrementalParsingEnabled;
+  bool IsParsed;
 
 public:
   SwiftDocumentSyntaxInfo(const CompilerInvocation &CompInv,
@@ -713,13 +714,16 @@ class SwiftDocumentSyntaxInfo {
         Parser->getParser().Context.evaluator);
     Parser->getDiagnosticEngine().addConsumer(DiagConsumer);
 
-    // If there is a syntax parsing cache, incremental syntax parsing is
-    // performed and thus the generated AST may not be up-to-date.
-    HasUpToDateAST = CompInv.getMainFileSyntaxParsingCache() == nullptr;
+    IncrementalParsingEnabled =
+        CompInv.getMainFileSyntaxParsingCache() != nullptr;
+    IsParsed = false;
   }
 
-  void parse() {
-    Parser->parse();
+  void parseIfNeeded() {
+    if (!IsParsed) {
+      Parser->parse();
+      IsParsed = true;
+    }
   }
 
   SourceFile &getSourceFile() {
@@ -738,7 +742,7 @@ class SwiftDocumentSyntaxInfo {
     return SM;
   }
 
-  bool hasUpToDateAST() { return HasUpToDateAST; }
+  bool isIncrementalParsingEnabled() { return IncrementalParsingEnabled; }
 
   ArrayRef<DiagnosticEntryInfo> getDiagnostics() {
     return DiagConsumer.getDiagnosticsForBuffer(BufferID);
@@ -1074,6 +1078,7 @@ struct SwiftEditorDocument::Implementation {
 
   std::shared_ptr<SwiftDocumentSyntaxInfo> getSyntaxInfo() {
     llvm::sys::ScopedLock L(AccessMtx);
+    SyntaxInfo->parseIfNeeded();
     return SyntaxInfo;
   }
 
@@ -1949,9 +1954,10 @@ void SwiftEditorDocument::updateSemaInfo() {
   ::updateSemaInfo(SemanticInfo, EditableBuffer);
 }
 
-void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
-                                SwiftLangSupport &Lang, bool BuildSyntaxTree,
-                                SyntaxParsingCache *SyntaxCache) {
+void SwiftEditorDocument::resetSyntaxInfo(ImmutableTextSnapshotRef Snapshot,
+                                          SwiftLangSupport &Lang,
+                                          bool BuildSyntaxTree,
+                                          SyntaxParsingCache *SyntaxCache) {
   llvm::sys::ScopedLock L(Impl.AccessMtx);
 
   assert(Impl.SemanticInfo && "Impl.SemanticInfo must be set");
@@ -1982,15 +1988,14 @@ void SwiftEditorDocument::parse(ImmutableTextSnapshotRef Snapshot,
   // Access to Impl.SyntaxInfo is guarded by Impl.AccessMtx
   Impl.SyntaxInfo.reset(
     new SwiftDocumentSyntaxInfo(CompInv, Snapshot, Args, Impl.FilePath));
-
-  Impl.SyntaxInfo->parse();
 }
 
 static UIdent SemaDiagStage("source.diagnostic.stage.swift.sema");
 static UIdent ParseDiagStage("source.diagnostic.stage.swift.parse");
 
 void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer, bool ReportDiags) {
   llvm::sys::ScopedLock L(Impl.AccessMtx);
+  Impl.SyntaxInfo->parseIfNeeded();
 
   Impl.ParserDiagnostics = Impl.SyntaxInfo->getDiagnostics();
   if (ReportDiags) {
@@ -2105,8 +2110,8 @@ SwiftEditorDocument::getSyntaxTree() const {
 
 std::string SwiftEditorDocument::getFilePath() const { return Impl.FilePath; }
 
-bool SwiftEditorDocument::hasUpToDateAST() const {
-  return Impl.SyntaxInfo->hasUpToDateAST();
+bool SwiftEditorDocument::isIncrementalParsingEnabled() const {
+  return Impl.SyntaxInfo->isIncrementalParsingEnabled();
 }
 
 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
@@ -2335,7 +2340,7 @@ void SwiftLangSupport::editorOpen(
     EditorDoc = new SwiftEditorDocument(Name, *this, fileSystem);
     Snapshot = EditorDoc->initializeText(
         Buf, Args, Consumer.needsSemanticInfo(), fileSystem);
-    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
+    EditorDoc->resetSyntaxInfo(Snapshot, *this, Consumer.syntaxTreeEnabled());
     if (EditorDocuments->getOrUpdate(Name, *this, EditorDoc)) {
       // Document already exists, re-initialize it. This should only happen
       // if we get OPEN request while the previous document is not closed.
@@ -2349,7 +2354,7 @@ void SwiftLangSupport::editorOpen(
   if (!Snapshot) {
     Snapshot = EditorDoc->initializeText(
         Buf, Args, Consumer.needsSemanticInfo(), fileSystem);
-    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled());
+    EditorDoc->resetSyntaxInfo(Snapshot, *this, Consumer.syntaxTreeEnabled());
   }
 
   if (Consumer.needsSemanticInfo()) {
@@ -2402,7 +2407,7 @@ void verifyIncrementalParse(SwiftEditorDocumentRef EditorDoc,
   SwiftDocumentSyntaxInfo ScratchSyntaxInfo(Invocation,
                                             EditorDoc->getLatestSnapshot(),
                                             Args, EditorDoc->getFilePath());
-  ScratchSyntaxInfo.parse();
+  ScratchSyntaxInfo.parseIfNeeded();
 
   // Dump the from-scratch syntax tree
   std::string FromScratchTreeString;
@@ -2507,19 +2512,22 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
     }
 
     // If client doesn't need any information, we doen't need to parse it.
+
+
+    SyntaxParsingCache *SyntaxCachePtr = nullptr;
+    if (SyntaxCache.hasValue()) {
+      SyntaxCachePtr = SyntaxCache.getPointer();
+    }
+    EditorDoc->resetSyntaxInfo(Snapshot, *this, Consumer.syntaxTreeEnabled(),
+                               SyntaxCachePtr);
+
     if (!Consumer.documentStructureEnabled() &&
         !Consumer.syntaxMapEnabled() &&
         !Consumer.diagnosticsEnabled() &&
         !Consumer.syntaxTreeEnabled()) {
       return;
     }
 
-    SyntaxParsingCache *SyntaxCachePtr = nullptr;
-    if (SyntaxCache.hasValue()) {
-      SyntaxCachePtr = SyntaxCache.getPointer();
-    }
-    EditorDoc->parse(Snapshot, *this, Consumer.syntaxTreeEnabled(),
-                     SyntaxCachePtr);
     // Do not report syntactic diagnostics; will be handled in readSemanticInfo.
     EditorDoc->readSyntaxInfo(Consumer, /*ReportDiags=*/false);
 
@@ -2578,11 +2586,12 @@ void SwiftLangSupport::editorFormatText(StringRef Name, unsigned Line,
     return;
   }
 
-  if (!EditorDoc->hasUpToDateAST()) {
-    // An up-to-date AST is needed for formatting. If it does not exist, fall
-    // back to a full reparse of the file
-    EditorDoc->parse(EditorDoc->getLatestSnapshot(), *this,
-                     /*BuildSyntaxTree=*/true);
+  if (EditorDoc->isIncrementalParsingEnabled()) {
+    // If incremental parsing is enabled, AST is not updated properly. Fall
+    // back to a full reparse of the file.
+    EditorDoc->resetSyntaxInfo(EditorDoc->getLatestSnapshot(), *this,
+                               /*BuildSyntaxTree=*/true,
+                               /*SyntaxCache=*/nullptr);
   }
 
   EditorDoc->formatText(Line, Length, Consumer);
@@ -2616,5 +2625,13 @@ void SwiftLangSupport::editorExpandPlaceholder(StringRef Name, unsigned Offset,
     return;
   }
 
+  if (EditorDoc->isIncrementalParsingEnabled()) {
+    // If incremental parsing is enabled, AST is not updated properly. Fall
+    // back to a full reparse of the file.
+    EditorDoc->resetSyntaxInfo(EditorDoc->getLatestSnapshot(), *this,
+                               /*BuildSyntaxTree=*/true,
+                               /*SyntaxCache=*/nullptr);
+  }
+
   EditorDoc->expandPlaceholder(Offset, Length, Consumer);
 }

tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h

@@ -107,9 +107,9 @@ class SwiftEditorDocument :
   ImmutableTextSnapshotRef getLatestSnapshot() const;
   std::pair<unsigned, unsigned> getLineAndColumnInBuffer(unsigned Offset);
 
-  void parse(ImmutableTextSnapshotRef Snapshot, SwiftLangSupport &Lang,
-             bool BuildSyntaxTree,
-             swift::SyntaxParsingCache *SyntaxCache = nullptr);
+  void resetSyntaxInfo(ImmutableTextSnapshotRef Snapshot,
+                       SwiftLangSupport &Lang, bool BuildSyntaxTree,
+                       swift::SyntaxParsingCache *SyntaxCache = nullptr);
   void readSyntaxInfo(EditorConsumer &consumer, bool ReportDiags);
   void readSemanticInfo(ImmutableTextSnapshotRef Snapshot,
                         EditorConsumer& Consumer);
@@ -129,7 +129,7 @@ class SwiftEditorDocument :
 
   /// Whether or not the AST stored for this document is up-to-date or just an
   /// artifact of incremental syntax parsing
-  bool hasUpToDateAST() const;
+  bool isIncrementalParsingEnabled() const;
 
   /// Returns the virtual filesystem associated with this document.
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getFileSystem() const;

tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp

@@ -834,11 +834,24 @@ static int handleTestInvocation(TestOptions Opts, TestOptions &InitOpts) {
     break;
 
   case SourceKitRequest::ExpandPlaceholder:
-    sourcekitd_request_dictionary_set_uid(Req, KeyRequest, RequestEditorOpen);
-    sourcekitd_request_dictionary_set_string(Req, KeyName, SemaName.c_str());
-    sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxMap, false);
-    sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, false);
-    sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
+    if (Opts.Length) {
+      // Single placeholder by location.
+      sourcekitd_request_dictionary_set_uid(Req, KeyRequest, RequestEditorExpandPlaceholder);
+      sourcekitd_request_dictionary_set_string(Req, KeyName, SemaName.c_str());
+      sourcekitd_request_dictionary_set_int64(Req, KeyOffset, ByteOffset);
+      sourcekitd_request_dictionary_set_int64(Req, KeyLength, Opts.Length);
+    } else {
+      if (ByteOffset) {
+        llvm::errs() << "Missing '-length <number>'\n";
+        return 1;
+      }
+      // Expand all placeholders.
+      sourcekitd_request_dictionary_set_uid(Req, KeyRequest, RequestEditorOpen);
+      sourcekitd_request_dictionary_set_string(Req, KeyName, SemaName.c_str());
+      sourcekitd_request_dictionary_set_int64(Req, KeyEnableSyntaxMap, false);
+      sourcekitd_request_dictionary_set_int64(Req, KeyEnableStructure, false);
+      sourcekitd_request_dictionary_set_int64(Req, KeySyntacticOnly, !Opts.UsedSema);
+    }
     break;
 
   case SourceKitRequest::SyntaxTree:
@@ -1341,7 +1354,13 @@ static bool handleResponse(sourcekitd_response_t Resp, const TestOptions &Opts,
       break;
 
       case SourceKitRequest::ExpandPlaceholder:
-        expandPlaceholders(SourceBuf.get(), llvm::outs());
+        if (Opts.Length) {
+          // Single placeholder by location.
+          sourcekitd_response_description_dump_filedesc(Resp, STDOUT_FILENO);
+        } else {
+          // Expand all placeholders.
+          expandPlaceholders(SourceBuf.get(), llvm::outs());
+        }
         break;
       case SourceKitRequest::ModuleGroups:
         printModuleGroupNames(Info, llvm::outs());