Merge pull request #18016 from ahoppen/003-incremental-syntax-coloring

[libSyntax] Incremental syntax colouring

Author: ahoppen

include/swift/Basic/JSONSerialization.h

@@ -27,6 +27,7 @@
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include <map>
 #include <vector>
 
 namespace swift {
@@ -341,6 +342,10 @@ struct unvalidatedObjectTraits : public std::integral_constant<bool,
 && !has_ObjectValidateTraits<T>::value> {};
 
 class Output {
+public:
+  using UserInfoMap = std::map<void *, void *>;
+
+private:
   enum State {
     ArrayFirstValue,
     ArrayOtherValue,
@@ -353,13 +358,19 @@ class Output {
   bool PrettyPrint;
   bool NeedBitValueComma;
   bool EnumerationMatchFound;
+  UserInfoMap UserInfo;
 
 public:
-  Output(llvm::raw_ostream &os, bool PrettyPrint = true) : Stream(os),
-      PrettyPrint(PrettyPrint), NeedBitValueComma(false),
-      EnumerationMatchFound(false) {}
+  Output(llvm::raw_ostream &os, UserInfoMap UserInfo = {},
+         bool PrettyPrint = true)
+      : Stream(os), PrettyPrint(PrettyPrint), NeedBitValueComma(false),
+        EnumerationMatchFound(false), UserInfo(UserInfo) {}
   virtual ~Output() = default;
 
+  UserInfoMap &getUserInfo() {
+    return UserInfo;
+  }
+
   unsigned beginArray();
   bool preflightElement(unsigned, void *&);
   void postflightElement(void*);

include/swift/Basic/OwnedString.h

@@ -71,7 +71,7 @@ class OwnedString {
   OwnedString(): OwnedString(nullptr, 0, StringOwnership::Unowned) {}
 
   OwnedString(const char *Data, size_t Length):
-    OwnedString(Data, Length, StringOwnership::Unowned) {}
+    OwnedString(Data, Length, StringOwnership::Copied) {}
 
   OwnedString(StringRef Str) : OwnedString(Str.data(), Str.size()) {}
 
@@ -106,7 +106,7 @@ class OwnedString {
     return *this;
   }
 
-  OwnedString copy() {
+  OwnedString copy() const {
     return OwnedString(Data, Length, StringOwnership::Copied);
   }
 

include/swift/Parse/SyntaxParsingCache.h

@@ -47,6 +47,13 @@ namespace swift {
 
 using namespace swift::syntax;
 
+struct SyntaxReuseRegion {
+  /// The byte offset at which the range begins
+  uintptr_t Start;
+  /// The byte offset at which the end ends
+  uintptr_t End;
+};
+
 class SyntaxParsingCache {
   /// The syntax tree prior to the edit
   SourceFileSyntax OldSyntaxTree;
@@ -61,7 +68,7 @@ class SyntaxParsingCache {
 
   /// If \c RecordReuseInformation buffer offsets of ranges that have been
   /// successfully looked up in this cache are stored.
-  std::vector<std::pair<unsigned, unsigned>> ReusedRanges;
+  std::vector<SyntaxReuseRegion> ReusedRanges;
 
 public:
   SyntaxParsingCache(SourceFileSyntax OldSyntaxTree)
@@ -85,7 +92,7 @@ class SyntaxParsingCache {
   /// Return the ranges of the new source file that have been successfully
   /// looked up in this cache as a (start, end) pair of byte offsets in the
   /// post-edit file.
-  std::vector<std::pair<unsigned, unsigned>> getReusedRanges() const {
+  std::vector<SyntaxReuseRegion> getReusedRanges() const {
     return ReusedRanges;
   }
 

include/swift/Subsystems.h

@@ -61,6 +61,7 @@ namespace swift {
   class SILParserTUState;
   class SourceFile;
   class SourceManager;
+  class SyntaxParsingCache;
   class Token;
   class TopLevelContext;
   struct TypeLoc;
@@ -327,7 +328,8 @@ namespace swift {
   class ParserUnit {
   public:
     ParserUnit(SourceManager &SM, unsigned BufferID,
-               const LangOptions &LangOpts, StringRef ModuleName);
+               const LangOptions &LangOpts, StringRef ModuleName,
+               SyntaxParsingCache *SyntaxCache = nullptr);
     ParserUnit(SourceManager &SM, unsigned BufferID);
     ParserUnit(SourceManager &SM, unsigned BufferID,
                unsigned Offset, unsigned EndOffset);

include/swift/Syntax/Serialization/SyntaxSerialization.h

@@ -27,6 +27,10 @@
 namespace swift {
 namespace json {
 
+/// The associated value will be interpreted as \c bool. If \c true the node IDs
+/// will not be included in the serialized JSON.
+static void *DontSerializeNodeIdsUserInfoKey = &DontSerializeNodeIdsUserInfoKey;
+
 /// Serialization traits for SourcePresence.
 template <>
 struct ScalarEnumerationTraits<syntax::SourcePresence> {
@@ -141,8 +145,12 @@ struct ObjectTraits<syntax::RawSyntax> {
     }
     auto presence = value.getPresence();
     out.mapRequired("presence", presence);
-    auto nodeId = value.getId();
-    out.mapRequired("id", nodeId);
+
+    bool omitNodeId = (bool)out.getUserInfo()[DontSerializeNodeIdsUserInfoKey];
+    if (!omitNodeId) {
+      auto nodeId = value.getId();
+      out.mapRequired("id", nodeId);
+    }
   }
 };
 

lib/FrontendTool/FrontendTool.cpp

@@ -242,7 +242,8 @@ static bool emitLoadedModuleTraceIfNeeded(ASTContext &ctxt,
   std::string stringBuffer;
   {
     llvm::raw_string_ostream memoryBuffer(stringBuffer);
-    json::Output jsonOutput(memoryBuffer, /*PrettyPrint=*/false);
+    json::Output jsonOutput(memoryBuffer, /*UserInfo=*/{},
+                            /*PrettyPrint=*/false);
     json::jsonize(jsonOutput, trace, /*Required=*/true);
   }
   stringBuffer += "\n";
@@ -289,7 +290,7 @@ static bool emitSyntax(SourceFile *SF, LangOptions &LangOpts,
   auto os = getFileOutputStream(OutputFilename, SF->getASTContext());
   if (!os) return true;
 
-  json::Output jsonOut(*os, /*PrettyPrint=*/false);
+  json::Output jsonOut(*os, /*UserInfo=*/{}, /*PrettyPrint=*/false);
   auto Root = SF->getSyntaxRoot().getRaw();
   jsonOut << *Root;
   *os << "\n";

lib/Parse/Parser.cpp

@@ -1018,9 +1018,11 @@ ParserUnit::ParserUnit(SourceManager &SM, unsigned BufferID)
 }
 
 ParserUnit::ParserUnit(SourceManager &SM, unsigned BufferID,
-                       const LangOptions &LangOpts, StringRef ModuleName)
-  : Impl(*new Implementation(SM, BufferID, LangOpts, ModuleName)) {
+                       const LangOptions &LangOpts, StringRef ModuleName,
+                       SyntaxParsingCache *SyntaxCache)
+    : Impl(*new Implementation(SM, BufferID, LangOpts, ModuleName)) {
 
+  Impl.SF->SyntaxParsingCache = SyntaxCache;
   Impl.TheParser.reset(new Parser(BufferID, *Impl.SF, nullptr));
 }
 

test/incrParse/invalid.swift

@@ -1,8 +1,17 @@
 // RUN: %empty-directory(%t)
 // RUN: %incparse-test %s --test-case NO_CHANGES
+// RUN: %incparse-test %s --test-case NESTED_INITIALIZERS
 
 func start() {}
 
 class Bar
 
 let y = 1
+
+class NestedInitializers {
+  <<NESTED_INITIALIZERS<|||init() {>>>
+    init() {
+
+    }
+  <<NESTED_INITIALIZERS<|||}>>>
+}

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -188,6 +188,13 @@ struct DiagnosticEntryInfo : DiagnosticEntryInfoBase {
   SmallVector<DiagnosticEntryInfoBase, 1> Notes;
 };
 
+struct SourceFileRange {
+  /// The byte offset at which the range begins
+  uintptr_t Start;
+  /// The byte offset at which the end ends
+  uintptr_t End;
+};
+
 class EditorConsumer {
   virtual void anchor();
 public:
@@ -240,6 +247,10 @@ class EditorConsumer {
   virtual bool handleSerializedSyntaxTree(StringRef Text) = 0;
   virtual bool syntaxTreeEnabled() = 0;
 
+  virtual bool syntaxReuseInfoEnabled() = 0;
+  virtual bool handleSyntaxReuseRegions(
+      std::vector<SourceFileRange> ReuseRegions) = 0;
+
   virtual void finished() {}
 
   // FIXME: This is just for bootstrapping incremental syntax tree parsing.