Merge pull request #20391 from rintaro/5.0-incrparse-rdar45259469

[5.0][incrParse] A couple of fixes for incrParse

Author: nkcsgexi

include/swift/Parse/SyntaxParsingCache.h

@@ -34,8 +34,11 @@ struct SourceEdit {
   /// The length of the string that replaced the range described above.
   size_t ReplacementLength;
 
+  SourceEdit(size_t Start, size_t End, size_t ReplacementLength)
+      : Start(Start), End(End), ReplacementLength(ReplacementLength){};
+
   /// The length of the range that has been replaced
-  size_t originalLength() { return End - Start; }
+  size_t originalLength() const { return End - Start; }
 
   /// Check if the characters replaced by this edit fall into the given range
   /// or are directly adjacent to it
@@ -65,12 +68,16 @@ class SyntaxParsingCache {
       : OldSyntaxTree(OldSyntaxTree) {}
 
   /// Add an edit that transformed the source file which created this cache into
-  /// the source file that is now being parsed incrementally. The order in which
-  /// the edits are added using this method needs to be the same order in which
-  /// the edits were applied to the source file.
-  void addEdit(size_t Start, size_t End, size_t ReplacementLength) {
-    Edits.push_back({Start, End, ReplacementLength});
-  }
+  /// the source file that is now being parsed incrementally. \c Start must be a
+  /// position from the *original* source file, and it must not overlap any
+  /// other edits previously added. For instance, given:
+  ///   (aaa, bbb)
+  ///   0123456789
+  /// When you want to turn this into:
+  ///   (c, dddd)
+  ///   0123456789
+  /// edits should be: { 1, 4, 1 } and { 6, 9, 4 }.
+  void addEdit(size_t Start, size_t End, size_t ReplacementLength);
 
   /// Check if a syntax node of the given kind at the given position can be
   /// reused for a new syntax tree.
@@ -86,11 +93,13 @@ class SyntaxParsingCache {
   getReusedRegions(const SourceFileSyntax &SyntaxTree) const;
 
   /// Translates a post-edit position to a pre-edit position by undoing the
-  /// specified edits.
+  /// specified edits. Returns \c None if no pre-edit position exists because
+  /// the post-edit position has been inserted by an edit.
+  /// 
   /// Should not be invoked externally. Only public for testing purposes.
-  static size_t
+  static Optional<size_t>
   translateToPreEditPosition(size_t PostEditPosition,
-                             llvm::SmallVector<SourceEdit, 4> Edits);
+                             ArrayRef<SourceEdit> Edits);
 
 private:
   llvm::Optional<Syntax> lookUpFrom(const Syntax &Node, size_t NodeStart,

include/swift/Syntax/SyntaxData.h

@@ -133,15 +133,16 @@ class SyntaxData final
   }
 
 public:
-  /// Get the node immediately before this current node. Return 0 if we cannot
-  /// find such node.
+  /// Get the node immediately before this current node that does contain a
+  /// non-missing token. Return nullptr if we cannot find such node.
   RC<SyntaxData> getPreviousNode() const;
 
-  /// Get the node immediately after this current node. Return 0 if we cannot
-  /// find such node.
+  /// Get the node immediately after this current node that does contain a
+  /// non-missing token. Return nullptr if we cannot find such node.
   RC<SyntaxData> getNextNode() const;
 
-  /// Get the first token node in this tree
+  /// Get the first non-missing token node in this tree. Return nullptr if this
+  /// node does not contain non-missing tokens.
   RC<SyntaxData> getFirstToken() const;
 
   ~SyntaxData() {

lib/Parse/SyntaxParsingCache.cpp

@@ -16,6 +16,13 @@
 using namespace swift;
 using namespace swift::syntax;
 
+void SyntaxParsingCache::addEdit(size_t Start, size_t End,
+                                 size_t ReplacementLength) {
+  assert((Edits.empty() || Edits.back().End <= Start) &&
+         "'Start' must be greater than or equal to 'End' of the previous edit");
+  Edits.emplace_back(Start, End, ReplacementLength);
+}
+
 bool SyntaxParsingCache::nodeCanBeReused(const Syntax &Node, size_t NodeStart,
                                          size_t Position,
                                          SyntaxKind Kind) const {
@@ -37,6 +44,7 @@ bool SyntaxParsingCache::nodeCanBeReused(const Syntax &Node, size_t NodeStart,
   if (auto NextNode = Node.getData().getNextNode()) {
     auto NextLeafNode = NextNode->getFirstToken();
     auto NextRawNode = NextLeafNode->getRaw();
+    assert(NextRawNode->isPresent());
     NextLeafNodeLength += NextRawNode->getTokenText().size();
     for (auto TriviaPiece : NextRawNode->getLeadingTrivia()) {
       NextLeafNodeLength += TriviaPiece.getTextLength();
@@ -66,7 +74,7 @@ llvm::Optional<Syntax> SyntaxParsingCache::lookUpFrom(const Syntax &Node,
   size_t ChildStart = NodeStart;
   for (size_t I = 0, E = Node.getNumChildren(); I < E; ++I) {
     llvm::Optional<Syntax> Child = Node.getChild(I);
-    if (!Child.hasValue()) {
+    if (!Child.hasValue() || Child->isMissing()) {
       continue;
     }
     auto ChildEnd = ChildStart + Child->getTextLength();
@@ -79,23 +87,31 @@ llvm::Optional<Syntax> SyntaxParsingCache::lookUpFrom(const Syntax &Node,
   return llvm::None;
 }
 
-size_t SyntaxParsingCache::translateToPreEditPosition(
-    size_t PostEditPosition, llvm::SmallVector<SourceEdit, 4> Edits) {
+Optional<size_t>
+SyntaxParsingCache::translateToPreEditPosition(size_t PostEditPosition,
+                                               ArrayRef<SourceEdit> Edits) {
   size_t Position = PostEditPosition;
-  for (auto I = Edits.begin(), E = Edits.end(); I != E; ++I) {
-    auto Edit = *I;
-    if (Edit.End + Edit.ReplacementLength - Edit.originalLength() <= Position) {
-      Position = Position - Edit.ReplacementLength + Edit.originalLength();
-    }
+  for (auto &Edit : Edits) {
+    if (Edit.Start > Position)
+      // Remaining edits doesn't affect the position. (Edits are sorted)
+      break;
+    if (Edit.Start + Edit.ReplacementLength > Position)
+      // This is a position inserted by the edit, and thus doesn't exist in the
+      // pre-edit version of the file.
+      return None;
+
+    Position = Position - Edit.ReplacementLength + Edit.originalLength();
   }
   return Position;
 }
 
 llvm::Optional<Syntax> SyntaxParsingCache::lookUp(size_t NewPosition,
                                                   SyntaxKind Kind) {
-  size_t OldPosition = translateToPreEditPosition(NewPosition, Edits);
+  Optional<size_t> OldPosition = translateToPreEditPosition(NewPosition, Edits);
+  if (!OldPosition.hasValue())
+    return None;
 
-  auto Node = lookUpFrom(OldSyntaxTree, /*NodeStart=*/0, OldPosition, Kind);
+  auto Node = lookUpFrom(OldSyntaxTree, /*NodeStart=*/0, *OldPosition, Kind);
   if (Node.hasValue()) {
     ReusedNodeIds.insert(Node->getId());
   }

lib/Syntax/SyntaxData.cpp

@@ -59,7 +59,8 @@ RC<SyntaxData> SyntaxData::getPreviousNode() const {
     if (hasParent()) {
       for (size_t I = N - 1; ; I--) {
         if (auto C = getParent()->getChild(I)) {
-          return C;
+          if (C->getRaw()->isPresent() && C->getFirstToken())
+            return C;
         }
         if (I == 0)
           break;
@@ -73,27 +74,35 @@ RC<SyntaxData> SyntaxData::getNextNode() const {
   if (hasParent()) {
     for (size_t I = getIndexInParent() + 1, N = Parent->getNumChildren();
          I != N; I++) {
-      if (auto C = getParent()->getChild(I))
-        return C;
+      if (auto C = getParent()->getChild(I)) {
+        if (C->getRaw()->isPresent() && C->getFirstToken())
+          return C;
+      }
     }
     return Parent->getNextNode();
   }
   return nullptr;
 }
 
 RC<SyntaxData> SyntaxData::getFirstToken() const {
+  if (getRaw()->isToken()) {
+    // Get a reference counted version of this
+    assert(hasParent() && "The syntax tree should not conisist only of the root");
+    return getParent()->getChild(getIndexInParent());
+  }
+
   for (size_t I = 0, E = getNumChildren(); I < E; ++I) {
     if (auto Child = getChild(I)) {
-      if (!Child->getRaw()->isMissing()) {
-        return Child->getFirstToken();
+      if (Child->getRaw()->isMissing())
+        continue;
+      if (Child->getRaw()->isToken()) {
+        return Child;
+      } else if (auto Token = Child->getFirstToken()) {
+        return Token;
       }
     }
   }
-
-  // Get a reference counted version of this
-  assert(getRaw()->isToken() && "Leaf node that is no token?");
-  assert(hasParent() && "The syntax tree should not conisist only of the root");
-  return getParent()->getChild(getIndexInParent());
+  return nullptr;
 }
 
 AbsolutePosition SyntaxData::getAbsolutePositionBeforeLeadingTrivia() const {

test/incrParse/add-else-to-ifconfig.swift

@@ -0,0 +1,7 @@
+// RUN: %empty-directory(%t)
+// RUN: %validate-incrparse %s --test-case ADD_ELSE
+
+func container() {
+#if false
+  <<ADD_ELSE<|||#else>>>
+

test/incrParse/add-space-at-missing-brace.swift

@@ -0,0 +1,10 @@
+// RUN: %empty-directory(%t)
+// RUN: %validate-incrparse %s --test-case INSERT_SPACE
+
+class AnimationType {
+  func foo(x: Blah) {
+    switch x {
+    case (.
+
+extension AnimationType {
+  public<<INSERT_SPACE<||| >>> 

test/incrParse/inserted_text_starts_identifier.swift

@@ -0,0 +1,6 @@
+// RUN: %empty-directory(%t)
+// RUN: %validate-incrparse %s --test-case STRING
+
+// SR-8995 rdar://problem/45259469
+
+self = <<STRING<|||_                            _>>>foo(1)[object1, object2] + o bar(1)