[libSyntax] Create deferred nodes in SyntaxParseActions

include/swift/Parse/ParsedRawSyntaxNode.h

@@ -16,6 +16,7 @@
 #include "swift/Basic/Debug.h"
 #include "swift/Basic/SourceLoc.h"
 #include "swift/Parse/ParsedTrivia.h"
+#include "swift/Parse/SyntaxParseActions.h"
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/SyntaxKind.h"
 #include "llvm/Support/Debug.h"
@@ -41,76 +42,40 @@ class SyntaxParsingContext;
 /// in the current parsing context.
 class ParsedRawSyntaxNode {
   friend class ParsedRawSyntaxRecorder;
-  enum class DataKind: uint8_t {
-    Null,
-    Recorded,
-    DeferredLayout,
-    DeferredToken,
-  };
-
-  struct RecordedSyntaxNode {
-    OpaqueSyntaxNode OpaqueNode;
-  };
-  struct DeferredLayoutNode {
-    MutableArrayRef<ParsedRawSyntaxNode> Children;
-  };
-  struct DeferredTokenNode {
-    SourceLoc TokLoc;
-    unsigned TokLength;
-    StringRef LeadingTrivia;
-    StringRef TrailingTrivia;
-  };
-
-  union {
-    RecordedSyntaxNode RecordedData;
-    DeferredLayoutNode DeferredLayout;
-    DeferredTokenNode DeferredToken;
-  };
+  using DataKind = RecordedOrDeferredNode::Kind;
+
+  /// The opaque data of this node. Needs to be interpreted by the \c
+  /// SyntaxParseActions, which created it.
+  RecordedOrDeferredNode Data;
+
   /// The range of this node, including trivia.
   CharSourceRange Range;
   uint16_t SynKind;
   uint16_t TokKind;
-  DataKind DK;
   /// Primary used for capturing a deferred missing token.
   bool IsMissing = false;
 
-  ParsedRawSyntaxNode(syntax::SyntaxKind k, CharSourceRange r,
-                      MutableArrayRef<ParsedRawSyntaxNode> deferredNodes)
-      : DeferredLayout({deferredNodes}), Range(r), SynKind(uint16_t(k)),
-        TokKind(uint16_t(tok::unknown)), DK(DataKind::DeferredLayout) {
-    assert(getKind() == k && "Syntax kind with too large value!");
-  }
-
-  ParsedRawSyntaxNode(tok tokKind, SourceLoc tokLoc, unsigned tokLength,
-                      StringRef leadingTrivia, StringRef trailingTrivia)
-      : DeferredToken{tokLoc, tokLength, leadingTrivia, trailingTrivia},
-        Range{tokLoc.getAdvancedLoc(-leadingTrivia.size()),
-              (unsigned)leadingTrivia.size() + tokLength +
-                  (unsigned)trailingTrivia.size()},
-        SynKind(uint16_t(syntax::SyntaxKind::Token)),
-        TokKind(uint16_t(tokKind)), DK(DataKind::DeferredToken) {
-    assert(getTokenKind() == tokKind && "Token kind is too large value!");
-  }
   ParsedRawSyntaxNode(const ParsedRawSyntaxNode &other) = delete;
   ParsedRawSyntaxNode &operator=(const ParsedRawSyntaxNode &other) = delete;
 
 public:
   ParsedRawSyntaxNode()
-      : RecordedData{}, Range(), SynKind(uint16_t(syntax::SyntaxKind::Unknown)),
-        TokKind(uint16_t(tok::unknown)), DK(DataKind::Null) {}
-
-  ParsedRawSyntaxNode(syntax::SyntaxKind k, tok tokKind, CharSourceRange r,
-                      OpaqueSyntaxNode n, bool IsMissing = false)
-      : RecordedData{n}, Range(r), SynKind(uint16_t(k)),
-        TokKind(uint16_t(tokKind)), DK(DataKind::Recorded),
-        IsMissing(IsMissing) {
-    assert(getKind() == k && "Syntax kind with too large value!");
-    assert(getTokenKind() == tokKind && "Token kind with too large value!");
+      : Data(nullptr, DataKind::Null), Range(),
+        SynKind(uint16_t(syntax::SyntaxKind::Unknown)),
+        TokKind(uint16_t(tok::unknown)) {}
+
+  ParsedRawSyntaxNode(OpaqueSyntaxNode Opaque, CharSourceRange Range,
+                      syntax::SyntaxKind SynKind, tok TokKind, DataKind DK,
+                      bool IsMissing)
+      : Data(Opaque, DK), Range(Range), SynKind(uint16_t(SynKind)),
+        TokKind(uint16_t(TokKind)), IsMissing(IsMissing) {
+    assert(getKind() == SynKind && "Syntax kind with too large value!");
+    assert(getTokenKind() == TokKind && "Token kind with too large value!");
   }
 
 #ifndef NDEBUG
   bool ensureDataIsNotRecorded() {
-    if (DK != DataKind::Recorded)
+    if (getDataKind() != DataKind::Recorded)
       return true;
     llvm::dbgs() << "Leaking node: ";
     dump(llvm::dbgs());
@@ -122,23 +87,10 @@ class ParsedRawSyntaxNode {
   ParsedRawSyntaxNode &operator=(ParsedRawSyntaxNode &&other) {
     assert(ensureDataIsNotRecorded() &&
            "recorded data is being destroyed by assignment");
-    switch (other.DK) {
-    case DataKind::Null:
-      break;
-    case DataKind::Recorded:
-      RecordedData = std::move(other.RecordedData);
-      break;
-    case DataKind::DeferredLayout:
-      DeferredLayout = std::move(other.DeferredLayout);
-      break;
-    case DataKind::DeferredToken:
-      DeferredToken = std::move(other.DeferredToken);
-      break;
-    }
+    Data = std::move(other.Data);
     Range = std::move(other.Range);
     SynKind = std::move(other.SynKind);
     TokKind = std::move(other.TokKind);
-    DK = std::move(other.DK);
     IsMissing = std::move(other.IsMissing);
     other.reset();
     return *this;
@@ -150,6 +102,33 @@ class ParsedRawSyntaxNode {
     assert(ensureDataIsNotRecorded() && "recorded data is being destructed");
   }
 
+  /// Returns the type of this node (recorded, deferred layout, deferred token,
+  /// null).
+  DataKind getDataKind() const { return Data.getKind(); }
+
+  /// Returns the opaque data of this node, assuming that it is deferred. This
+  /// must be interpreted by the \c SyntaxParseAction, which likely also needs
+  /// the node type (layout or token) to interpret the data.
+  /// The data opaque data returned by this function *must not* be used to
+  /// record the node, only to insepect it.
+  OpaqueSyntaxNode getUnsafeDeferredOpaqueData() const {
+    assert(isDeferredLayout() || isDeferredToken());
+    return Data.getOpaque();
+  }
+
+  RecordedOrDeferredNode takeRecordedOrDeferredNode() {
+    RecordedOrDeferredNode Data = this->Data;
+    reset();
+    return Data;
+  }
+
+  /// Return the opaque data of this node and reset it.
+  OpaqueSyntaxNode takeData() {
+    OpaqueSyntaxNode Data = this->Data.getOpaque();
+    reset();
+    return Data;
+  }
+
   syntax::SyntaxKind getKind() const { return syntax::SyntaxKind(SynKind); }
   tok getTokenKind() const { return tok(TokKind); }
 
@@ -160,118 +139,73 @@ class ParsedRawSyntaxNode {
     return getTokenKind() == tokKind;
   }
 
-  bool isNull() const {
-    return DK == DataKind::Null;
-  }
+  bool isNull() const { return getDataKind() == DataKind::Null; }
 
-  bool isRecorded() const { return DK == DataKind::Recorded; }
-  bool isDeferredLayout() const { return DK == DataKind::DeferredLayout; }
-  bool isDeferredToken() const { return DK == DataKind::DeferredToken; }
+  bool isRecorded() const { return getDataKind() == DataKind::Recorded; }
+  bool isDeferredLayout() const {
+    return getDataKind() == DataKind::DeferredLayout;
+  }
+  bool isDeferredToken() const {
+    return getDataKind() == DataKind::DeferredToken;
+  }
 
   /// Primary used for a deferred missing token.
   bool isMissing() const { return IsMissing; }
 
   void reset() {
-    RecordedData = {};
+    Data = RecordedOrDeferredNode(nullptr, DataKind::Null);
     SynKind = uint16_t(syntax::SyntaxKind::Unknown);
     TokKind = uint16_t(tok::unknown);
-    DK = DataKind::Null;
     IsMissing = false;
   }
 
   ParsedRawSyntaxNode unsafeCopy() const {
     ParsedRawSyntaxNode copy;
-    switch (DK) {
-    case DataKind::DeferredLayout:
-      copy.DeferredLayout = DeferredLayout;
-      break;
-    case DataKind::DeferredToken:
-      copy.DeferredToken = DeferredToken;
-      break;
-    case DataKind::Recorded:
-      copy.RecordedData = RecordedData;
-      break;
-    case DataKind::Null:
-      break;
-    }
+    copy.Data = Data;
     copy.Range = Range;
     copy.SynKind = SynKind;
     copy.TokKind = TokKind;
-    copy.DK = DK;
     copy.IsMissing = IsMissing;
     return copy;
   }
 
   /// Returns the range of this node including leading and trailing trivia.
   CharSourceRange getRange() const { return Range; }
 
-  // Recorded Data ===========================================================//
-
-  const OpaqueSyntaxNode &getOpaqueNode() const {
-    assert(isRecorded());
-    return RecordedData.OpaqueNode;
-  }
-  OpaqueSyntaxNode takeOpaqueNode() {
-    assert(isRecorded());
-    auto opaque = RecordedData.OpaqueNode;
-    reset();
-    return opaque;
-  }
-
   // Deferred Layout Data ====================================================//
 
-  ArrayRef<ParsedRawSyntaxNode> getDeferredChildren() const {
-    assert(DK == DataKind::DeferredLayout);
-    return DeferredLayout.Children;
-  }
+  /// If this node is a deferred layout node, return the child at index \p
+  /// ChildIndex.
+  /// Note that this may be an expensive operation since the \c
+  /// SyntaxParseAction, which created the node (implicitly passed via the
+  /// \p SyntaxContext) needs to be consulted to retrieve the child.
+  ParsedRawSyntaxNode
+  getDeferredChild(size_t ChildIndex,
+                   const SyntaxParsingContext *SyntaxContext) const;
 
-  MutableArrayRef<ParsedRawSyntaxNode> getDeferredChildren() {
-    assert(DK == DataKind::DeferredLayout);
-    return DeferredLayout.Children;
-  }
+  size_t
+  getDeferredNumChildren(const SyntaxParsingContext *SyntaxContext) const;
 
   ParsedRawSyntaxNode copyDeferred() const {
+    assert(isDeferredLayout() || isDeferredToken() && "node not deferred");
     ParsedRawSyntaxNode copy;
-    switch (DK) {
-    case DataKind::DeferredLayout:
-      copy.DeferredLayout = DeferredLayout;
-      break;
-    case DataKind::DeferredToken:
-      copy.DeferredToken = DeferredToken;
-      break;
-    default:
-      llvm_unreachable("node not deferred");
-    }
+    copy.Data = Data;
     copy.Range = Range;
     copy.SynKind = SynKind;
     copy.TokKind = TokKind;
-    copy.DK = DK;
     copy.IsMissing = IsMissing;
     return copy;
   }
 
-  // Deferred Token Data =====================================================//
-
-  CharSourceRange getDeferredTokenRange() const {
-    assert(DK == DataKind::DeferredToken);
-    return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};
-  }
-  StringRef getDeferredLeadingTrivia() const {
-    assert(DK == DataKind::DeferredToken);
-    return DeferredToken.LeadingTrivia;
-  }
-  StringRef getDeferredTrailingTrivia() const {
-    assert(DK == DataKind::DeferredToken);
-    return DeferredToken.TrailingTrivia;
-  }
-
   //==========================================================================//
 
   /// Dump this piece of syntax recursively for debugging or testing.
   SWIFT_DEBUG_DUMP;
 
-  /// Dump this piece of syntax recursively.
-  void dump(raw_ostream &OS, unsigned Indent = 0) const;
+  /// Dump this piece of syntax recursively. If \p Context is passed, this
+  /// method is also able to traverse its children and dump them.
+  void dump(raw_ostream &OS, const SyntaxParsingContext *Context = nullptr,
+            unsigned Indent = 0) const;
 
   static ParsedRawSyntaxNode null() {
     return ParsedRawSyntaxNode{};

include/swift/Parse/ParsedRawSyntaxRecorder.h

@@ -41,6 +41,12 @@ namespace syntax {
 class ParsedRawSyntaxRecorder final {
   std::shared_ptr<SyntaxParseActions> SPActions;
 
+  /// Assuming that \p node is a deferred layout or token node, record it and
+  /// return the recorded node.
+  /// This consumes the data from \c node, which is unusable after it has been
+  /// recorded. The returned node should be used afterwards instead.
+  ParsedRawSyntaxNode recordDeferredNode(ParsedRawSyntaxNode &node);
+
 public:
   explicit ParsedRawSyntaxRecorder(std::shared_ptr<SyntaxParseActions> spActions)
     : SPActions(std::move(spActions)) {}
@@ -69,6 +75,10 @@ class ParsedRawSyntaxRecorder final {
                                                      SourceLoc loc);
 
   /// Form a deferred syntax layout node.
+  /// All nodes in \p deferred nodes must be deferred. Otherwise, we'd have a
+  /// deferred layout node with recorded child nodes. Should we decide to
+  /// discard the deferred layout node, we would also need to discard its
+  /// recorded children, which cannot be done.
   ParsedRawSyntaxNode
   makeDeferred(syntax::SyntaxKind k,
                MutableArrayRef<ParsedRawSyntaxNode> deferredNodes,
@@ -85,9 +95,17 @@ class ParsedRawSyntaxRecorder final {
   ParsedRawSyntaxNode lookupNode(size_t lexerOffset, SourceLoc loc,
                                  syntax::SyntaxKind kind);
 
-  #ifndef NDEBUG
+  /// For a deferred layout node \p parent, retrieve the deferred child node
+  /// at \p ChildIndex.
+  ParsedRawSyntaxNode getDeferredChild(const ParsedRawSyntaxNode &parent,
+                                       size_t ChildIndex) const;
+
+  /// For a deferred layout node \p node, retrieve the number of children.
+  size_t getDeferredNumChildren(const ParsedRawSyntaxNode &node) const;
+
+#ifndef NDEBUG
   static void verifyElementRanges(ArrayRef<ParsedRawSyntaxNode> elements);
-  #endif
+#endif
 };
 
 } // end namespace swift

include/swift/Parse/ParsedSyntaxNodes.h.gyb

@@ -62,9 +62,13 @@ public:
   /// ${line}
 %       end
 %       if child.is_optional:
-  llvm::Optional<Parsed${child.type_name}> getDeferred${child.name}();
+  llvm::Optional<Parsed${child.type_name}> getDeferred${child.name}(
+      const SyntaxParsingContext *SyntaxContext
+  );
 %       else:
-  Parsed${child.type_name} getDeferred${child.name}();
+  Parsed${child.type_name} getDeferred${child.name}(
+      const SyntaxParsingContext *SyntaxContext
+  );
 %       end
 %     end