Formalize some SourceFile parsing outputs

Currently when parsing a SourceFile, the parser
gets handed pointers so that it can write the
interface hash and collected tokens directly into
the file. It can also call `setSyntaxRoot` at
the end of parsing to set the syntax tree.

In preparation for the removal of
`performParseOnly`, this commit formalizes these
values as outputs of `ParseSourceFileRequest`,
ensuring that the file gets parsed when the
interface hash, collected tokens, or syntax tree
is queried.

Author: hamishknight

include/swift/AST/ASTContext.h

@@ -407,6 +407,12 @@ class ASTContext final {
                               array.size());
   }
 
+  template <typename T>
+  MutableArrayRef<T>
+  AllocateCopy(const std::vector<T> &vec,
+               AllocationArena arena = AllocationArena::Permanent) const {
+    return AllocateCopy(ArrayRef<T>(vec), arena);
+  }
 
   template<typename T>
   ArrayRef<T> AllocateCopy(const SmallVectorImpl<T> &vec,

include/swift/AST/ParseRequests.h

@@ -19,6 +19,7 @@
 #include "swift/AST/ASTTypeIDs.h"
 #include "swift/AST/EvaluatorDependencies.h"
 #include "swift/AST/SimpleRequest.h"
+#include "swift/Syntax/SyntaxNodes.h"
 
 namespace swift {
 
@@ -81,10 +82,17 @@ class ParseAbstractFunctionBodyRequest :
   void cacheResult(BraceStmt *value) const;
 };
 
+struct SourceFileParsingResult {
+  ArrayRef<Decl *> TopLevelDecls;
+  Optional<ArrayRef<Token>> CollectedTokens;
+  Optional<llvm::MD5> InterfaceHash;
+  Optional<syntax::SourceFileSyntax> SyntaxRoot;
+};
+
 /// Parse the top-level decls of a SourceFile.
 class ParseSourceFileRequest
     : public SimpleRequest<
-          ParseSourceFileRequest, ArrayRef<Decl *>(SourceFile *),
+          ParseSourceFileRequest, SourceFileParsingResult(SourceFile *),
           RequestFlags::SeparatelyCached | RequestFlags::DependencySource> {
 public:
   using SimpleRequest::SimpleRequest;
@@ -93,13 +101,13 @@ class ParseSourceFileRequest
   friend SimpleRequest;
 
   // Evaluation.
-  ArrayRef<Decl *> evaluate(Evaluator &evaluator, SourceFile *SF) const;
+  SourceFileParsingResult evaluate(Evaluator &evaluator, SourceFile *SF) const;
 
 public:
   // Caching.
   bool isCached() const { return true; }
-  Optional<ArrayRef<Decl *>> getCachedResult() const;
-  void cacheResult(ArrayRef<Decl *> decls) const;
+  Optional<SourceFileParsingResult> getCachedResult() const;
+  void cacheResult(SourceFileParsingResult result) const;
 
 public:
   evaluator::DependencySource

include/swift/AST/ParseTypeIDZone.def

@@ -23,5 +23,5 @@ SWIFT_REQUEST(Parse, ParseAbstractFunctionBodyRequest,
               BraceStmt *(AbstractFunctionDecl *), SeparatelyCached,
               NoLocationInfo)
 SWIFT_REQUEST(Parse, ParseSourceFileRequest,
-              ArrayRef<Decl *>(SourceFile *), SeparatelyCached,
+              SourceFileParsingResult(SourceFile *), SeparatelyCached,
               NoLocationInfo)

include/swift/AST/SourceFile.h

@@ -558,26 +558,15 @@ class SourceFile final : public FileUnit {
     return ParsingOpts.contains(ParsingFlags::EnableInterfaceHash);
   }
 
-  NullablePtr<llvm::MD5> getInterfaceHashPtr() {
-    return InterfaceHash ? InterfaceHash.getPointer() : nullptr;
-  }
-
-  void getInterfaceHash(llvm::SmallString<32> &str) const {
-    // Copy to preserve idempotence.
-    llvm::MD5 md5 = *InterfaceHash;
-    llvm::MD5::MD5Result result;
-    md5.final(result);
-    llvm::MD5::stringifyResult(result, str);
-  }
+  /// Output this file's interface hash into the provided string buffer.
+  void getInterfaceHash(llvm::SmallString<32> &str) const;
 
   void dumpInterfaceHash(llvm::raw_ostream &out) {
     llvm::SmallString<32> str;
     getInterfaceHash(str);
     out << str << '\n';
   }
 
-  std::vector<Token> &getTokenVector();
-
   /// If this source file has been told to collect its parsed tokens, retrieve
   /// those tokens.
   ArrayRef<Token> getAllTokens() const;
@@ -593,7 +582,6 @@ class SourceFile final : public FileUnit {
   bool hasDelayedBodyParsing() const;
 
   syntax::SourceFileSyntax getSyntaxRoot() const;
-  void setSyntaxRoot(syntax::SourceFileSyntax &&Root);
   bool hasSyntaxRoot() const;
 
   OpaqueTypeDecl *lookupOpaqueResultType(StringRef MangledName) override;
@@ -611,7 +599,7 @@ class SourceFile final : public FileUnit {
 
   /// If not \c None, the underlying vector contains the parsed tokens of this
   /// source file.
-  Optional<std::vector<Token>> AllCollectedTokens;
+  Optional<ArrayRef<Token>> AllCollectedTokens;
 
   /// The root of the syntax tree representing the source file.
   std::unique_ptr<syntax::SourceFileSyntax> SyntaxRoot;

include/swift/Parse/Parser.h

@@ -22,6 +22,7 @@
 #include "swift/AST/Expr.h"
 #include "swift/AST/DiagnosticsParse.h"
 #include "swift/AST/LayoutConstraint.h"
+#include "swift/AST/ParseRequests.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
 #include "swift/Basic/OptionSet.h"
@@ -92,8 +93,11 @@ class ConsumeTokenReceiver {
   /// This is called to update the kind of a token whose start location is Loc.
   virtual void registerTokenKindChange(SourceLoc Loc, tok NewKind) {};
 
-  /// This is called when a source file is fully parsed.
-  virtual void finalize() {};
+  /// This is called when a source file is fully parsed. It returns the
+  /// finalized vector of tokens, or \c None if the receiver isn't configured to
+  /// record them.
+  virtual Optional<std::vector<Token>> finalize() { return None; }
+
   virtual ~ConsumeTokenReceiver() = default;
 };
 
@@ -124,7 +128,10 @@ class Parser {
   /// Tracks parsed decls that LLDB requires to be inserted at the top-level.
   std::vector<Decl *> ContextSwitchedTopLevelDecls;
 
-  NullablePtr<llvm::MD5> CurrentTokenHash;
+  /// The current token hash, or \c None if the parser isn't computing a hash
+  /// for the token stream.
+  Optional<llvm::MD5> CurrentTokenHash;
+
   void recordTokenHash(const Token Tok) {
     if (!Tok.getText().empty())
       recordTokenHash(Tok.getText());
@@ -416,6 +423,14 @@ class Parser {
     return SyntaxContext->finalizeRoot();
   }
 
+  /// Retrieve the token receiver from the parser once it has finished parsing.
+  std::unique_ptr<ConsumeTokenReceiver> takeTokenReceiver() {
+    assert(Tok.is(tok::eof) && "not done parsing yet");
+    auto *receiver = TokReceiver;
+    TokReceiver = nullptr;
+    return std::unique_ptr<ConsumeTokenReceiver>(receiver);
+  }
+
   //===--------------------------------------------------------------------===//
   // Routines to save and restore parser state.
 

include/swift/Parse/SyntaxParseActions.h

@@ -24,11 +24,13 @@ namespace swift {
 
 class CharSourceRange;
 class ParsedTriviaPiece;
+class SourceFile;
 class SourceLoc;
 enum class tok;
 
 namespace syntax {
-  enum class SyntaxKind;
+class SourceFileSyntax;
+enum class SyntaxKind;
 }
 
 typedef void *OpaqueSyntaxNode;
@@ -55,6 +57,12 @@ class SyntaxParseActions {
                                            ArrayRef<OpaqueSyntaxNode> elements,
                                            CharSourceRange range) = 0;
 
+  /// Attempt to realize an opaque raw syntax node for a source file into a
+  /// SourceFileSyntax node. This will return \c None if the parsing action
+  /// doesn't support the realization of syntax nodes.
+  virtual Optional<syntax::SourceFileSyntax>
+  realizeSyntaxRoot(OpaqueSyntaxNode root, const SourceFile &SF) = 0;
+
   /// Discard raw syntax node.
   /// 
   /// FIXME: This breaks invariant that any recorded node will be a part of the

include/swift/SyntaxParse/SyntaxTreeCreator.h

@@ -23,7 +23,8 @@ namespace swift {
   class SourceFile;
 
 namespace syntax {
-  class SyntaxArena;
+class SyntaxArena;
+class SourceFileSyntax;
 }
 
 /// Receives the parsed syntax info from the parser and constructs a persistent
@@ -51,7 +52,8 @@ class SyntaxTreeCreator: public SyntaxParseActions {
                     RC<syntax::SyntaxArena> arena);
   ~SyntaxTreeCreator();
 
-  void acceptSyntaxRoot(OpaqueSyntaxNode root, SourceFile &SF);
+  Optional<syntax::SourceFileSyntax>
+  realizeSyntaxRoot(OpaqueSyntaxNode root, const SourceFile &SF) override;
 
 private:
   OpaqueSyntaxNode recordToken(tok tokenKind,

lib/AST/Decl.cpp

@@ -44,7 +44,7 @@
 #include "swift/AST/TypeCheckRequests.h"
 #include "swift/AST/TypeLoc.h"
 #include "swift/AST/SwiftNameTranslation.h"
-#include "swift/Parse/Lexer.h"
+#include "swift/Parse/Lexer.h" // FIXME: Bad dependency
 #include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"

lib/AST/Module.cpp

@@ -1077,17 +1077,27 @@ LookupConformanceInModuleRequest::evaluate(
   return ProtocolConformanceRef(conformance);
 }
 
+void SourceFile::getInterfaceHash(llvm::SmallString<32> &str) const {
+  assert(hasInterfaceHash() && "Interface hash not enabled");
+  auto &eval = getASTContext().evaluator;
+  auto *mutableThis = const_cast<SourceFile *>(this);
+  auto md5 = *evaluateOrDefault(eval, ParseSourceFileRequest{mutableThis}, {})
+                  .InterfaceHash;
+  llvm::MD5::MD5Result result;
+  md5.final(result);
+  llvm::MD5::stringifyResult(result, str);
+}
+
 bool SourceFile::hasSyntaxRoot() const {
   return ParsingOpts.contains(ParsingFlags::BuildSyntaxTree);
 }
 
 syntax::SourceFileSyntax SourceFile::getSyntaxRoot() const {
-  assert(hasSyntaxRoot() && "no syntax root is set.");
-  return *SyntaxRoot;
-}
-
-void SourceFile::setSyntaxRoot(syntax::SourceFileSyntax &&Root) {
-  SyntaxRoot = std::make_unique<syntax::SourceFileSyntax>(std::move(Root));
+  assert(hasSyntaxRoot() && "has no syntax root");
+  auto &eval = getASTContext().evaluator;
+  auto *mutableThis = const_cast<SourceFile *>(this);
+  return *evaluateOrDefault(eval, ParseSourceFileRequest{mutableThis}, {})
+              .SyntaxRoot;
 }
 
 void DirectOperatorLookupRequest::writeDependencySink(
@@ -2222,18 +2232,6 @@ SourceFile::SourceFile(ModuleDecl &M, SourceFileKind K,
     assert(!problem && "multiple main files?");
     (void)problem;
   }
-
-  if (hasInterfaceHash()) {
-    InterfaceHash.emplace();
-  }
-  if (shouldCollectTokens()) {
-    AllCollectedTokens = std::vector<Token>();
-  }
-}
-
-std::vector<Token> &SourceFile::getTokenVector() {
-  assert(shouldCollectTokens() && "Disabled");
-  return *AllCollectedTokens;
 }
 
 SourceFile::ParsingOptions
@@ -2248,7 +2246,10 @@ SourceFile::getDefaultParsingOptions(const LangOptions &langOpts) {
 
 ArrayRef<Token> SourceFile::getAllTokens() const {
   assert(shouldCollectTokens() && "Disabled");
-  return *AllCollectedTokens;
+  auto &eval = getASTContext().evaluator;
+  auto *mutableThis = const_cast<SourceFile *>(this);
+  return *evaluateOrDefault(eval, ParseSourceFileRequest{mutableThis}, {})
+              .CollectedTokens;
 }
 
 bool SourceFile::shouldCollectTokens() const {
@@ -2282,7 +2283,7 @@ ArrayRef<Decl *> SourceFile::getTopLevelDecls() const {
   auto &ctx = getASTContext();
   auto *mutableThis = const_cast<SourceFile *>(this);
   return evaluateOrDefault(ctx.evaluator, ParseSourceFileRequest{mutableThis},
-                           {});
+                           {}).TopLevelDecls;
 }
 
 bool FileUnit::walk(ASTWalker &walker) {

lib/Parse/ParseDecl.cpp

@@ -236,9 +236,8 @@ void Parser::parseTopLevel(SmallVectorImpl<Decl *> &decls) {
     decls.push_back(decl);
   }
 
-  // Finalize the token receiver.
+  // Finalize the syntax context.
   SyntaxContext->addToken(Tok, LeadingTrivia, TrailingTrivia);
-  TokReceiver->finalize();
 }
 
 bool Parser::parseTopLevelSIL() {
@@ -4470,16 +4469,14 @@ Parser::parseDeclList(SourceLoc LBLoc, SourceLoc &RBLoc, Diag<> ErrorDiag,
                       ParseDeclOptions Options, IterableDeclContext *IDC,
                       bool &hadError) {
 
-  // Record the curly braces but nothing inside.
+  // If we're hashing the type body separately, record the curly braces but
+  // nothing inside for the interface hash.
+  Optional<llvm::SaveAndRestore<Optional<llvm::MD5>>> S;
   if (IDC->areTokensHashedForThisBodyInsteadOfInterfaceHash()) {
     recordTokenHash("{");
     recordTokenHash("}");
+    S.emplace(CurrentTokenHash, llvm::MD5());
   }
-  llvm::MD5 tokenHashForThisDeclList;
-  llvm::SaveAndRestore<NullablePtr<llvm::MD5>> T(
-      CurrentTokenHash, IDC->areTokensHashedForThisBodyInsteadOfInterfaceHash()
-                            ? &tokenHashForThisDeclList
-                            : CurrentTokenHash);
 
   std::vector<Decl *> decls;
   ParserStatus Status;
@@ -4514,6 +4511,7 @@ Parser::parseDeclList(SourceLoc LBLoc, SourceLoc &RBLoc, Diag<> ErrorDiag,
     return std::make_pair(decls, None);
 
   llvm::MD5::MD5Result result;
+  auto tokenHashForThisDeclList = CurrentTokenHash.getValueOr(llvm::MD5());
   tokenHashForThisDeclList.final(result);
   llvm::SmallString<32> tokenHashString;
   llvm::MD5::stringifyResult(result, tokenHashString);
@@ -6409,7 +6407,7 @@ void Parser::parseAbstractFunctionBody(AbstractFunctionDecl *AFD) {
   recordTokenHash("{");
   recordTokenHash("}");
 
-  llvm::SaveAndRestore<NullablePtr<llvm::MD5>> T(CurrentTokenHash, nullptr);
+  llvm::SaveAndRestore<Optional<llvm::MD5>> T(CurrentTokenHash, None);
 
   // If we can delay parsing this body, or this is the first pass of code
   // completion, skip until the end. If we encounter a code completion token

lib/Parse/ParseIfConfig.cpp

@@ -677,8 +677,9 @@ ParserResult<IfConfigDecl> Parser::parseIfConfig(
     llvm::SaveAndRestore<bool> S(InInactiveClauseEnvironment,
                                  InInactiveClauseEnvironment || !isActive);
     // Disable updating the interface hash inside inactive blocks.
-    llvm::SaveAndRestore<NullablePtr<llvm::MD5>> T(
-        CurrentTokenHash, isActive ? CurrentTokenHash : nullptr);
+    Optional<llvm::SaveAndRestore<Optional<llvm::MD5>>> T;
+    if (!isActive)
+      T.emplace(CurrentTokenHash, None);
 
     if (isActive || !isVersionCondition) {
       parseElements(Elements, isActive);