Merge pull request #12517 from gparker42/revert-ee7a062

Revert "libSyntax: create a basic infrastructure for generating libSy…

Author: Greg Parker

include/swift/AST/Module.h

@@ -28,8 +28,6 @@
 #include "swift/Basic/SourceLoc.h"
 #include "swift/Basic/STLExtras.h"
 #include "swift/Parse/Token.h"
-#include "swift/Syntax/SyntaxNodes.h"
-#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
@@ -1084,23 +1082,9 @@ class SourceFile final : public FileUnit {
     return (bool)AllCorrectedTokens;
   }
 
-  syntax::SourceFileSyntax getSyntaxRoot() const {
-    assert(SyntaxRoot && "no syntax root is set.");
-    return *SyntaxRoot;
-  }
-
 private:
-  friend class syntax::SyntaxParsingContext;
-  friend class syntax::SyntaxParsingContextRoot;
-
   /// If not None, the underlying vector should contain tokens of this source file.
   Optional<std::vector<Token>> AllCorrectedTokens;
-
-  /// All of the raw token syntax nodes in the underlying source.
-  std::vector<syntax::RawTokenInfo> AllRawTokenSyntax;
-
-  /// The root of the syntax tree representing the source file.
-  Optional<syntax::SourceFileSyntax> SyntaxRoot;
 };
 
 

include/swift/Basic/OwnedString.h

@@ -49,6 +49,10 @@ class OwnedString {
     assert(Length >= 0 && "expected length to be non-negative");
 
     if (Ownership == StringOwnership::Copied && Data) {
+      assert(
+        Length <= strlen(Data) &&
+        "expected length to be a valid index, within the length of the string");
+
       char *substring = static_cast<char *>(malloc(Length + 1));
       assert(substring && "expected successful malloc of copy");
 

include/swift/Parse/Lexer.h

@@ -23,7 +23,6 @@
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/References.h"
 #include "swift/Syntax/Trivia.h"
-#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -246,7 +245,7 @@ class Lexer {
   }
 
   /// Lex a full token including leading and trailing trivia.
-  syntax::RawTokenInfo fullLex();
+  RC<syntax::RawTokenSyntax> fullLex();
 
   bool isKeepingComments() const {
     return RetainComments == CommentRetentionMode::ReturnAsTokens;

include/swift/Parse/Parser.h

@@ -25,8 +25,6 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
-#include "swift/Syntax/RawTokenSyntax.h"
-#include "swift/Syntax/SyntaxParsingContext.h"
 #include "swift/Basic/OptionSet.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/LocalContext.h"
@@ -327,9 +325,6 @@ class Parser {
   /// This vector is managed by \c StructureMarkerRAII objects.
   llvm::SmallVector<StructureMarker, 16> StructureMarkers;
 
-  /// Current syntax parsing context where call backs should be directed to.
-  syntax::SyntaxParsingContext *SyntaxContext;
-
 public:
   Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
          PersistentParserState *PersistentState = nullptr);
@@ -1422,11 +1417,6 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
                    unsigned Offset = 0,
                    unsigned EndOffset = 0);
 
-
-void populateTokenSyntaxMap(const LangOptions &LangOpts,
-                            const SourceManager &SM,
-                            unsigned BufferID,
-                            std::vector<syntax::RawTokenInfo> &Result);
 } // end namespace swift
 
 #endif

include/swift/Syntax/Syntax.h

@@ -127,9 +127,6 @@ class Syntax {
   /// if it has one, otherwise 0.
   CursorIndex getIndexInParent() const;
 
-  /// Returns true if this syntax node represents a token.
-  bool isToken() const;
-
   /// Returns true if this syntax node represents a statement.
   bool isStmt() const;
 
@@ -155,6 +152,9 @@ class Syntax {
   /// Returns true if the node is "present" in the source.
   bool isPresent() const;
 
+
+  bool isToken() const;
+
   /// Print the syntax node with full fidelity to the given output stream.
   void print(llvm::raw_ostream &OS) const;
 

include/swift/Syntax/SyntaxFactory.h.gyb

@@ -54,8 +54,6 @@ struct SyntaxFactory {
   static Optional<Syntax>
   createSyntax(SyntaxKind Kind, llvm::ArrayRef<Syntax> Elements);
 
-  static SyntaxKind getUnknownKind(SyntaxKind Kind);
-
 % for node in SYNTAX_NODES:
 %   if node.children:
 %     child_params = []
@@ -107,7 +105,7 @@ struct SyntaxFactory {
     llvm::Optional<TokenSyntax> TrailingComma = llvm::None);
 
   /// Creates a TypeIdentifierSyntax with the provided name and leading/trailing
-  /// trivia.
+  /// trivia. 
   static TypeIdentifierSyntax makeTypeIdentifier(OwnedString TypeName,
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});
 
@@ -119,7 +117,7 @@ struct SyntaxFactory {
   /// Creates a TypeIdentifierSyntax for the `Any` type.
   static TypeIdentifierSyntax makeAnyTypeIdentifier(
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});
-
+  
   /// Creates a TypeIdentifierSyntax for the `Self` type.
   static TypeIdentifierSyntax makeSelfTypeIdentifier(
     const Trivia &LeadingTrivia = {}, const Trivia &TrailingTrivia = {});

include/swift/Syntax/SyntaxParsingContext.h

@@ -20,7 +20,7 @@
 #include "swift/AST/Identifier.h"
 #include "swift/Basic/LangOptions.h"
 #include "swift/Basic/SourceManager.h"
-#include "swift/Syntax/SyntaxParsingContext.h"
+#include "swift/Syntax/TokenSyntax.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -737,12 +737,11 @@ static bool rangeContainsPlaceholderEnd(const char *CurPtr,
   return false;
 }
 
-syntax::RawTokenInfo Lexer::fullLex() {
+RC<syntax::RawTokenSyntax> Lexer::fullLex() {
   if (NextToken.isEscapedIdentifier()) {
     LeadingTrivia.push_back(syntax::TriviaPiece::backtick());
     TrailingTrivia.push_front(syntax::TriviaPiece::backtick());
   }
-  auto Loc = NextToken.getLoc();
   auto Result = syntax::RawTokenSyntax::make(NextToken.getKind(),
                                         OwnedString(NextToken.getText()).copy(),
                                         syntax::SourcePresence::Present,
@@ -752,7 +751,7 @@ syntax::RawTokenInfo Lexer::fullLex() {
   if (NextToken.isNot(tok::eof)) {
     lexImpl();
   }
-  return {Loc, Result};
+  return Result;
 }
 
 /// lexOperatorIdentifier - Match identifiers formed out of punctuation.

lib/Parse/Lexer.cpp

@@ -18,9 +18,6 @@
 #include "swift/AST/DiagnosticsParse.h"
 #include "swift/Basic/EditorPlaceholder.h"
 #include "swift/Parse/CodeCompletionCallbacks.h"
-#include "swift/Syntax/SyntaxFactory.h"
-#include "swift/Syntax/TokenSyntax.h"
-#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
@@ -30,7 +27,6 @@
 #include "llvm/Support/raw_ostream.h"
 
 using namespace swift;
-using namespace swift::syntax;
 
 /// parseExpr
 ///
@@ -39,9 +35,6 @@ using namespace swift::syntax;
 ///
 /// \param isExprBasic Whether we're only parsing an expr-basic.
 ParserResult<Expr> Parser::parseExprImpl(Diag<> Message, bool isExprBasic) {
-  // Start a context for creating expression syntax.
-  SyntaxParsingContextChild ExprParsingContext(SyntaxContext, SyntaxKind::Expr);
-
   // If we are parsing a refutable pattern, check to see if this is the start
   // of a let/var/is pattern.  If so, parse it to an UnresolvedPatternExpr and
   // name binding will perform final validation.
@@ -472,7 +465,6 @@ ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
     Tok.setKind(tok::oper_prefix);
     LLVM_FALLTHROUGH;
   case tok::oper_prefix:
-    SyntaxContext->addTokenSyntax(Tok.getLoc());
     Operator = parseExprOperator();
     break;
   case tok::oper_binary_spaced:
@@ -765,6 +757,7 @@ UnresolvedDeclRefExpr *Parser::parseExprOperator() {
   SourceLoc loc = Tok.getLoc();
   Identifier name = Context.getIdentifier(Tok.getText());
   consumeToken();
+
   // Bypass local lookup.
   return new (Context) UnresolvedDeclRefExpr(name, refKind, DeclNameLoc(loc));
 }
@@ -1388,8 +1381,6 @@ ParserResult<Expr> Parser::parseExprPostfix(Diag<> ID, bool isExprBasic) {
   case tok::integer_literal: {
     StringRef Text = copyAndStripUnderscores(Context, Tok.getText());
     SourceLoc Loc = consumeToken(tok::integer_literal);
-    SyntaxContext->addTokenSyntax(Loc);
-    SyntaxContext->makeNode(SyntaxKind::IntegerLiteralExpr);
     Result = makeParserResult(new (Context) IntegerLiteralExpr(Text, Loc,
                                                            /*Implicit=*/false));
     break;
@@ -1795,20 +1786,11 @@ createStringLiteralExprFromSegment(ASTContext &Ctx,
 ///   expr-literal:
 ///     string_literal
 ParserResult<Expr> Parser::parseExprStringLiteral() {
-
   SmallVector<Lexer::StringSegment, 1> Segments;
   L->getStringLiteralSegments(Tok, Segments);
 
   Token EntireTok = Tok;
 
-  // Create a syntax node for string literal.
-  SyntaxContext->addTokenSyntax(Tok.getLoc());
-  SyntaxContext->makeNode(SyntaxKind::StringLiteralExpr);
-  SyntaxParsingContextChild LocalContext(SyntaxContext, SyntaxKind::Expr);
-
-  // FIXME: Avoid creating syntax nodes for string interpolation.
-  LocalContext.disable();
-
   // The start location of the entire string literal.
   SourceLoc Loc = Tok.getLoc();