Formalize some SourceFile parsing outputs

include/swift/AST/ASTContext.h

@@ -407,6 +407,12 @@ class ASTContext final {
                               array.size());
   }
 
+  template <typename T>
+  MutableArrayRef<T>
+  AllocateCopy(const std::vector<T> &vec,
+               AllocationArena arena = AllocationArena::Permanent) const {
+    return AllocateCopy(ArrayRef<T>(vec), arena);
+  }
 
   template<typename T>
   ArrayRef<T> AllocateCopy(const SmallVectorImpl<T> &vec,

include/swift/AST/ParseRequests.h

@@ -19,6 +19,7 @@
 #include "swift/AST/ASTTypeIDs.h"
 #include "swift/AST/EvaluatorDependencies.h"
 #include "swift/AST/SimpleRequest.h"
+#include "swift/Syntax/SyntaxNodes.h"
 
 namespace swift {
 
@@ -81,10 +82,17 @@ class ParseAbstractFunctionBodyRequest :
   void cacheResult(BraceStmt *value) const;
 };
 
+struct SourceFileParsingResult {
+  ArrayRef<Decl *> TopLevelDecls;
+  Optional<ArrayRef<Token>> CollectedTokens;
+  Optional<llvm::MD5> InterfaceHash;
+  Optional<syntax::SourceFileSyntax> SyntaxRoot;
+};
+
 /// Parse the top-level decls of a SourceFile.
 class ParseSourceFileRequest
     : public SimpleRequest<
-          ParseSourceFileRequest, ArrayRef<Decl *>(SourceFile *),
+          ParseSourceFileRequest, SourceFileParsingResult(SourceFile *),
           RequestFlags::SeparatelyCached | RequestFlags::DependencySource> {
 public:
   using SimpleRequest::SimpleRequest;
@@ -93,13 +101,13 @@ class ParseSourceFileRequest
   friend SimpleRequest;
 
   // Evaluation.
-  ArrayRef<Decl *> evaluate(Evaluator &evaluator, SourceFile *SF) const;
+  SourceFileParsingResult evaluate(Evaluator &evaluator, SourceFile *SF) const;
 
 public:
   // Caching.
   bool isCached() const { return true; }
-  Optional<ArrayRef<Decl *>> getCachedResult() const;
-  void cacheResult(ArrayRef<Decl *> decls) const;
+  Optional<SourceFileParsingResult> getCachedResult() const;
+  void cacheResult(SourceFileParsingResult result) const;
 
 public:
   evaluator::DependencySource

include/swift/AST/ParseTypeIDZone.def

@@ -23,5 +23,5 @@ SWIFT_REQUEST(Parse, ParseAbstractFunctionBodyRequest,
               BraceStmt *(AbstractFunctionDecl *), SeparatelyCached,
               NoLocationInfo)
 SWIFT_REQUEST(Parse, ParseSourceFileRequest,
-              ArrayRef<Decl *>(SourceFile *), SeparatelyCached,
+              SourceFileParsingResult(SourceFile *), SeparatelyCached,
               NoLocationInfo)

include/swift/AST/SourceFile.h

@@ -32,8 +32,6 @@ class SourceFile final : public FileUnit {
   friend class ParseSourceFileRequest;
 
 public:
-  struct SourceFileSyntaxInfo;
-
   /// Possible attributes for imports in source files.
   enum class ImportFlags {
     /// The imported module is exposed to anyone who imports the parent module.
@@ -110,12 +108,24 @@ class SourceFile final : public FileUnit {
     /// and adjust the client call 'performParseOnly'.
     DisablePoundIfEvaluation = 1 << 1,
 
+    /// Whether to build a syntax tree.
+    BuildSyntaxTree = 1 << 2,
+
+    /// Whether to save the file's parsed tokens.
+    CollectParsedTokens = 1 << 3,
+
+    /// Whether to compute the interface hash of the file.
+    EnableInterfaceHash = 1 << 4,
+
     /// Whether to suppress warnings when parsing. This is set for secondary
     /// files, as they get parsed multiple times.
-    SuppressWarnings = 1 << 2
+    SuppressWarnings = 1 << 5,
   };
   using ParsingOptions = OptionSet<ParsingFlags>;
 
+  /// Retrieve the parsing options specified in the LangOptions.
+  static ParsingOptions getDefaultParsingOptions(const LangOptions &langOpts);
+
 private:
   std::unique_ptr<SourceLookupCache> Cache;
   SourceLookupCache &getCache() const;
@@ -313,7 +323,6 @@ class SourceFile final : public FileUnit {
   llvm::StringMap<SourceFilePathInfo> getInfoForUsedFilePaths() const;
 
   SourceFile(ModuleDecl &M, SourceFileKind K, Optional<unsigned> bufferID,
-             bool KeepParsedTokens = false, bool KeepSyntaxTree = false,
              ParsingOptions parsingOpts = {}, bool isPrimary = false);
 
   ~SourceFile();
@@ -544,50 +553,35 @@ class SourceFile final : public FileUnit {
   /// Set the root refinement context for the file.
   void setTypeRefinementContext(TypeRefinementContext *TRC);
 
-  void enableInterfaceHash() {
-    assert(!hasInterfaceHash());
-    InterfaceHash.emplace();
-  }
-
+  /// Whether this file has an interface hash available.
   bool hasInterfaceHash() const {
-    return InterfaceHash.hasValue();
+    return ParsingOpts.contains(ParsingFlags::EnableInterfaceHash);
   }
 
-  NullablePtr<llvm::MD5> getInterfaceHashPtr() {
-    return InterfaceHash ? InterfaceHash.getPointer() : nullptr;
-  }
-
-  void getInterfaceHash(llvm::SmallString<32> &str) const {
-    // Copy to preserve idempotence.
-    llvm::MD5 md5 = *InterfaceHash;
-    llvm::MD5::MD5Result result;
-    md5.final(result);
-    llvm::MD5::stringifyResult(result, str);
-  }
+  /// Output this file's interface hash into the provided string buffer.
+  void getInterfaceHash(llvm::SmallString<32> &str) const;
 
   void dumpInterfaceHash(llvm::raw_ostream &out) {
     llvm::SmallString<32> str;
     getInterfaceHash(str);
     out << str << '\n';
   }
 
-  std::vector<Token> &getTokenVector();
-
+  /// If this source file has been told to collect its parsed tokens, retrieve
+  /// those tokens.
   ArrayRef<Token> getAllTokens() const;
 
-  bool shouldCollectToken() const;
+  /// Whether the parsed tokens of this source file should be saved, allowing
+  /// them to be accessed from \c getAllTokens.
+  bool shouldCollectTokens() const;
 
   bool shouldBuildSyntaxTree() const;
 
-  bool canBeParsedInFull() const;
-
   /// Whether the bodies of types and functions within this file can be lazily
   /// parsed.
   bool hasDelayedBodyParsing() const;
 
   syntax::SourceFileSyntax getSyntaxRoot() const;
-  void setSyntaxRoot(syntax::SourceFileSyntax &&Root);
-  bool hasSyntaxRoot() const;
 
   OpaqueTypeDecl *lookupOpaqueResultType(StringRef MangledName) override;
 
@@ -602,10 +596,12 @@ class SourceFile final : public FileUnit {
 
 private:
 
-  /// If not None, the underlying vector should contain tokens of this source file.
-  Optional<std::vector<Token>> AllCorrectedTokens;
+  /// If not \c None, the underlying vector contains the parsed tokens of this
+  /// source file.
+  Optional<ArrayRef<Token>> AllCollectedTokens;
 
-  std::unique_ptr<SourceFileSyntaxInfo> SyntaxInfo;
+  /// The root of the syntax tree representing the source file.
+  std::unique_ptr<syntax::SourceFileSyntax> SyntaxRoot;
 };
 
 inline SourceFile::ParsingOptions operator|(SourceFile::ParsingFlags lhs,

include/swift/Parse/Parser.h

@@ -22,6 +22,7 @@
 #include "swift/AST/Expr.h"
 #include "swift/AST/DiagnosticsParse.h"
 #include "swift/AST/LayoutConstraint.h"
+#include "swift/AST/ParseRequests.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
 #include "swift/Basic/OptionSet.h"
@@ -92,8 +93,11 @@ class ConsumeTokenReceiver {
   /// This is called to update the kind of a token whose start location is Loc.
   virtual void registerTokenKindChange(SourceLoc Loc, tok NewKind) {};
 
-  /// This is called when a source file is fully parsed.
-  virtual void finalize() {};
+  /// This is called when a source file is fully parsed. It returns the
+  /// finalized vector of tokens, or \c None if the receiver isn't configured to
+  /// record them.
+  virtual Optional<std::vector<Token>> finalize() { return None; }
+
   virtual ~ConsumeTokenReceiver() = default;
 };
 
@@ -124,7 +128,10 @@ class Parser {
   /// Tracks parsed decls that LLDB requires to be inserted at the top-level.
   std::vector<Decl *> ContextSwitchedTopLevelDecls;
 
-  NullablePtr<llvm::MD5> CurrentTokenHash;
+  /// The current token hash, or \c None if the parser isn't computing a hash
+  /// for the token stream.
+  Optional<llvm::MD5> CurrentTokenHash;
+
   void recordTokenHash(const Token Tok) {
     if (!Tok.getText().empty())
       recordTokenHash(Tok.getText());
@@ -416,6 +423,14 @@ class Parser {
     return SyntaxContext->finalizeRoot();
   }
 
+  /// Retrieve the token receiver from the parser once it has finished parsing.
+  std::unique_ptr<ConsumeTokenReceiver> takeTokenReceiver() {
+    assert(Tok.is(tok::eof) && "not done parsing yet");
+    auto *receiver = TokReceiver;
+    TokReceiver = nullptr;
+    return std::unique_ptr<ConsumeTokenReceiver>(receiver);
+  }
+
   //===--------------------------------------------------------------------===//
   // Routines to save and restore parser state.
 
@@ -472,6 +487,9 @@ class Parser {
       void receive(Token tok) override {
         delayedTokens.push_back(tok);
       }
+      Optional<std::vector<Token>> finalize() override {
+        llvm_unreachable("Cannot finalize a DelayedTokenReciever");
+      }
       ~DelayedTokenReceiver() {
         if (!shouldTransfer)
           return;

include/swift/Parse/SyntaxParseActions.h

@@ -24,11 +24,13 @@ namespace swift {
 
 class CharSourceRange;
 class ParsedTriviaPiece;
+class SourceFile;
 class SourceLoc;
 enum class tok;
 
 namespace syntax {
-  enum class SyntaxKind;
+class SourceFileSyntax;
+enum class SyntaxKind;
 }
 
 typedef void *OpaqueSyntaxNode;
@@ -55,6 +57,12 @@ class SyntaxParseActions {
                                            ArrayRef<OpaqueSyntaxNode> elements,
                                            CharSourceRange range) = 0;
 
+  /// Attempt to realize an opaque raw syntax node for a source file into a
+  /// SourceFileSyntax node. This will return \c None if the parsing action
+  /// doesn't support the realization of syntax nodes.
+  virtual Optional<syntax::SourceFileSyntax>
+  realizeSyntaxRoot(OpaqueSyntaxNode root, const SourceFile &SF) = 0;
+
   /// Discard raw syntax node.
   /// 
   /// FIXME: This breaks invariant that any recorded node will be a part of the

include/swift/SyntaxParse/SyntaxTreeCreator.h

@@ -23,7 +23,8 @@ namespace swift {
   class SourceFile;
 
 namespace syntax {
-  class SyntaxArena;
+class SyntaxArena;
+class SourceFileSyntax;
 }
 
 /// Receives the parsed syntax info from the parser and constructs a persistent
@@ -51,7 +52,8 @@ class SyntaxTreeCreator: public SyntaxParseActions {
                     RC<syntax::SyntaxArena> arena);
   ~SyntaxTreeCreator();
 
-  void acceptSyntaxRoot(OpaqueSyntaxNode root, SourceFile &SF);
+  Optional<syntax::SourceFileSyntax>
+  realizeSyntaxRoot(OpaqueSyntaxNode root, const SourceFile &SF) override;
 
 private:
   OpaqueSyntaxNode recordToken(tok tokenKind,

lib/AST/Decl.cpp

@@ -44,7 +44,7 @@
 #include "swift/AST/TypeCheckRequests.h"
 #include "swift/AST/TypeLoc.h"
 #include "swift/AST/SwiftNameTranslation.h"
-#include "swift/Parse/Lexer.h"
+#include "swift/Parse/Lexer.h" // FIXME: Bad dependency
 #include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"