Requestify ExportedSourceFile parsing

Avoid parsing the syntax tree up-front, and instead
only parse it when required, which happens when either:

1. ASTGen parsing is enabled (currently disabled
   by default)
2. Round trip checking is enabled for a primary
   file (enabled by default in a debug build,
   except when dep scanning or doing an IDE
   operation)
3. We need to evaluate a macro in that file

This change therefore means that we now no longer
need to parse the syntax tree for secondary files
by default unless we specifically need to evaluate
a macro in them (e.g if we need to lookup a member
on a decl with an attached macro). And the same
for primaries in release builds.

rdar://109283847

Author: hamishknight

include/swift/AST/ParseRequests.h

@@ -116,6 +116,26 @@ class ParseSourceFileRequest
   readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
+/// Parse the ExportedSourceFile for a given SourceFile.
+class ExportedSourceFileRequest
+    : public SimpleRequest<ExportedSourceFileRequest,
+                           void *(const SourceFile *),
+                           RequestFlags::SeparatelyCached> {
+public:
+  using SimpleRequest::SimpleRequest;
+
+private:
+  friend SimpleRequest;
+
+  void *evaluate(Evaluator &evaluator, const SourceFile *SF) const;
+
+public:
+  // Separate caching.
+  bool isCached() const { return true; }
+  Optional<void *> getCachedResult() const;
+  void cacheResult(void *exportedSF) const;
+};
+
 /// Parse the top-level items of a SourceFile.
 class ParseTopLevelDeclsRequest
     : public SimpleRequest<

include/swift/AST/ParseTypeIDZone.def

@@ -28,3 +28,6 @@ SWIFT_REQUEST(Parse, ParseSourceFileRequest,
 SWIFT_REQUEST(Parse, ParseTopLevelDeclsRequest,
               ArrayRef<Decl *>(SourceFile *), Cached,
               NoLocationInfo)
+SWIFT_REQUEST(Parse, ExportedSourceFileRequest,
+              void *(const SourceFile *), SeparatelyCached,
+              NoLocationInfo)

include/swift/AST/SourceFile.h

@@ -58,7 +58,7 @@ class SourceFile final : public FileUnit {
 
 public:
   /// Flags that direct how the source file is parsed.
-  enum class ParsingFlags : uint8_t {
+  enum class ParsingFlags : uint16_t {
     /// Whether to disable delayed parsing for nominal type, extension, and
     /// function bodies.
     ///
@@ -91,9 +91,17 @@ class SourceFile final : public FileUnit {
     /// files, as they get parsed multiple times.
     SuppressWarnings = 1 << 4,
 
-    /// Whether to disable the Swift Parser ASTGen
-    /// e.g. in dependency scanning, where an AST is not needed.
-    DisableSwiftParserASTGen = 1 << 5,
+    /// Ensure that the SwiftSyntax tree round trips correctly.
+    RoundTrip = 1 << 5,
+
+    /// Validate the new SwiftSyntax parser diagnostics.
+    ValidateNewParserDiagnostics = 1 << 6,
+
+    /// Enable ASTGen for producing the AST nodes.
+    EnableASTGen = 1 << 7,
+
+    /// Enable ASTGen for producing parser diagnostics.
+    EnableASTGenDiagnostics = 1 << 8,
   };
   using ParsingOptions = OptionSet<ParsingFlags>;
 
@@ -209,7 +217,13 @@ class SourceFile final : public FileUnit {
   /// Storage for \c HasImportsMatchingFlagRequest.
   ImportOptions cachedImportOptions;
 
+  /// The \c ExportedSourceFile instance produced by ASTGen, which includes
+  /// the SourceFileSyntax node corresponding to this source file. This is
+  /// \c None if it has not yet been computed.
+  mutable Optional<void *> CachedExportedSourceFile;
+
   friend ASTContext;
+  friend class ExportedSourceFileRequest;
 
 public:
   /// Appends the given declaration to the end of the top-level decls list. Do
@@ -262,6 +276,10 @@ class SourceFile final : public FileUnit {
   /// code for it. Note this method returns \c false in WMO.
   bool isPrimary() const { return IsPrimary; }
 
+  /// Retrieve the \c ExportedSourceFile instance produced by ASTGen, which
+  /// includes the SourceFileSyntax node corresponding to this source file.
+  void *getExportedSourceFile() const;
+
   /// The list of local type declarations in the source file.
   llvm::SetVector<TypeDecl *> LocalTypeDecls;
 
@@ -334,10 +352,6 @@ class SourceFile final : public FileUnit {
   /// this source file.
   llvm::SmallVector<Located<StringRef>, 0> VirtualFilePaths;
 
-  /// The \c ExportedSourceFile instance produced by ASTGen, which includes
-  /// the SourceFileSyntax node corresponding to this source file.
-  void *exportedSourceFile = nullptr;
-
   /// Returns information about the file paths used for diagnostics and magic
   /// identifiers in this source file, including virtual filenames introduced by
   /// \c #sourceLocation(file:) declarations.

lib/AST/Module.cpp

@@ -3687,6 +3687,14 @@ SourceFile::getDefaultParsingOptions(const LangOptions &langOpts) {
     opts |= ParsingFlags::DisablePoundIfEvaluation;
   if (langOpts.CollectParsedToken)
     opts |= ParsingFlags::CollectParsedTokens;
+  if (langOpts.hasFeature(Feature::ParserRoundTrip))
+    opts |= ParsingFlags::RoundTrip;
+  if (langOpts.hasFeature(Feature::ParserValidation))
+    opts |= ParsingFlags::ValidateNewParserDiagnostics;
+  if (langOpts.hasFeature(Feature::ParserASTGen))
+    opts |= ParsingFlags::EnableASTGen;
+  if (langOpts.hasFeature(Feature::ParserDiagnostics))
+    opts |= ParsingFlags::EnableASTGenDiagnostics;
   return opts;
 }
 
@@ -3762,6 +3770,11 @@ ArrayRef<Decl *> SourceFile::getHoistedDecls() const {
   return Hoisted;
 }
 
+void *SourceFile::getExportedSourceFile() const {
+  auto &eval = getASTContext().evaluator;
+  return evaluateOrDefault(eval, ExportedSourceFileRequest{this}, nullptr);
+}
+
 void SourceFile::addDeclWithRuntimeDiscoverableAttrs(ValueDecl *decl) {
   assert(!decl->getRuntimeDiscoverableAttrs().empty());
   DeclsWithRuntimeDiscoverableAttrs.insert(decl);

lib/Frontend/Frontend.cpp

@@ -1513,6 +1513,9 @@ void CompilerInstance::finishTypeChecking() {
 
 SourceFile::ParsingOptions
 CompilerInstance::getSourceFileParsingOptions(bool forPrimary) const {
+  using ActionType = FrontendOptions::ActionType;
+  using ParsingFlags = SourceFile::ParsingFlags;
+
   const auto &frontendOpts = Invocation.getFrontendOptions();
   const auto action = frontendOpts.RequestedAction;
 
@@ -1521,42 +1524,45 @@ CompilerInstance::getSourceFileParsingOptions(bool forPrimary) const {
     // Generally in a parse-only invocation, we want to disable #if evaluation.
     // However, there are a couple of modes where we need to know which clauses
     // are active.
-    if (action != FrontendOptions::ActionType::EmitImportedModules &&
-        action != FrontendOptions::ActionType::ScanDependencies) {
-      opts |= SourceFile::ParsingFlags::DisablePoundIfEvaluation;
+    if (action != ActionType::EmitImportedModules &&
+        action != ActionType::ScanDependencies) {
+      opts |= ParsingFlags::DisablePoundIfEvaluation;
     }
 
     // If we need to dump the parse tree, disable delayed bodies as we want to
     // show everything.
-    if (action == FrontendOptions::ActionType::DumpParse)
-      opts |= SourceFile::ParsingFlags::DisableDelayedBodies;
+    if (action == ActionType::DumpParse)
+      opts |= ParsingFlags::DisableDelayedBodies;
   }
 
-  auto typeOpts = getASTContext().TypeCheckerOpts;
-  if (forPrimary || isWholeModuleCompilation()) {
+  const auto &typeOpts = getASTContext().TypeCheckerOpts;
+  const auto isEffectivelyPrimary = forPrimary || isWholeModuleCompilation();
+  if (isEffectivelyPrimary) {
     // Disable delayed body parsing for primaries and in WMO, unless
     // forcefully skipping function bodies
     if (typeOpts.SkipFunctionBodies == FunctionBodySkipping::None)
-      opts |= SourceFile::ParsingFlags::DisableDelayedBodies;
+      opts |= ParsingFlags::DisableDelayedBodies;
   } else {
     // Suppress parse warnings for non-primaries, as they'll get parsed multiple
     // times.
-    opts |= SourceFile::ParsingFlags::SuppressWarnings;
+    opts |= ParsingFlags::SuppressWarnings;
   }
 
-  // Dependency scanning does not require an AST, so disable Swift Parser
-  // ASTGen parsing completely.
-  if (frontendOpts.RequestedAction ==
-      FrontendOptions::ActionType::ScanDependencies)
-    opts |= SourceFile::ParsingFlags::DisableSwiftParserASTGen;
+  // Turn off round-trip checking for secondary files, and for dependency
+  // scanning and IDE inspection.
+  if (!isEffectivelyPrimary || SourceMgr.hasIDEInspectionTargetBuffer() ||
+      frontendOpts.RequestedAction == ActionType::ScanDependencies) {
+    opts -= ParsingFlags::RoundTrip;
+    opts -= ParsingFlags::ValidateNewParserDiagnostics;
+  }
 
   // Enable interface hash computation for primaries or emit-module-separately,
   // but not in WMO, as it's only currently needed for incremental mode.
   if (forPrimary ||
       typeOpts.SkipFunctionBodies ==
           FunctionBodySkipping::NonInlinableWithoutTypes ||
       frontendOpts.ReuseFrontendForMultipleCompilations) {
-    opts |= SourceFile::ParsingFlags::EnableInterfaceHash;
+    opts |= ParsingFlags::EnableInterfaceHash;
   }
   return opts;
 }