[SourceKit] Add a request to generate object files in SourceKit

Add 'request.compile'

Author: rintaro

include/swift/AST/ASTTypeIDZone.def

@@ -17,6 +17,7 @@
 
 SWIFT_TYPEID(ActorIsolation)
 SWIFT_TYPEID(AncestryFlags)
+SWIFT_TYPEID(BodyAndFingerprint)
 SWIFT_TYPEID(BodyInitKind)
 SWIFT_TYPEID(BodyInitKindAndExpr)
 SWIFT_TYPEID(CtorInitializerKind)

include/swift/AST/ASTTypeIDs.h

@@ -81,6 +81,7 @@ enum class AncestryFlags : uint8_t;
 enum class ImplicitMemberAction : uint8_t;
 struct FingerprintAndMembers;
 class Identifier;
+class BodyAndFingerprint;
 
 // Define the AST type zone (zone 1)
 #define SWIFT_TYPEID_ZONE AST

include/swift/AST/Decl.h

@@ -5904,6 +5904,38 @@ class ImportAsMemberStatus {
   }
 };
 
+class BodyAndFingerprint {
+  llvm::PointerIntPair<BraceStmt *, 1, bool> BodyAndHasFp;
+  Fingerprint Fp;
+
+public:
+  BodyAndFingerprint(BraceStmt *body, Optional<Fingerprint> fp)
+      : BodyAndHasFp(body, fp.hasValue()),
+        Fp(fp.hasValue() ? *fp : Fingerprint::ZERO()) {}
+  BodyAndFingerprint() : BodyAndFingerprint(nullptr, None) {}
+
+  BraceStmt *getBody() const { return BodyAndHasFp.getPointer(); }
+
+  Optional<Fingerprint> getFingerprint() const {
+    if (BodyAndHasFp.getInt())
+      return Fp;
+    else
+      return None;
+  }
+
+  void setFingerprint(Optional<Fingerprint> fp) {
+    if (fp.hasValue()) {
+      Fp = *fp;
+      BodyAndHasFp.setInt(true);
+    } else {
+      Fp = Fingerprint::ZERO();
+      BodyAndHasFp.setInt(false);
+    }
+  }
+};
+
+void simple_display(llvm::raw_ostream &out, BodyAndFingerprint value);
+
 /// Base class for function-like declarations.
 class AbstractFunctionDecl : public GenericContext, public ValueDecl {
   friend class NeedsNewVTableEntryRequest;
@@ -5986,7 +6018,7 @@ class AbstractFunctionDecl : public GenericContext, public ValueDecl {
   union {
     /// This enum member is active if getBodyKind() is BodyKind::Parsed or
     /// BodyKind::TypeChecked.
-    BraceStmt *Body;
+    BodyAndFingerprint BodyAndFP;
 
     /// This enum member is active if getBodyKind() is BodyKind::Deserialized.
     StringRef BodyStringRepresentation;
@@ -6022,9 +6054,10 @@ class AbstractFunctionDecl : public GenericContext, public ValueDecl {
                        bool Throws, SourceLoc ThrowsLoc,
                        bool HasImplicitSelfDecl,
                        GenericParamList *GenericParams)
-      : GenericContext(DeclContextKind::AbstractFunctionDecl, Parent, GenericParams),
-        ValueDecl(Kind, Parent, Name, NameLoc),
-        Body(nullptr), AsyncLoc(AsyncLoc), ThrowsLoc(ThrowsLoc) {
+      : GenericContext(DeclContextKind::AbstractFunctionDecl, Parent,
+                       GenericParams),
+        ValueDecl(Kind, Parent, Name, NameLoc), BodyAndFP(), AsyncLoc(AsyncLoc),
+        ThrowsLoc(ThrowsLoc) {
     setBodyKind(BodyKind::None);
     Bits.AbstractFunctionDecl.HasImplicitSelfDecl = HasImplicitSelfDecl;
     Bits.AbstractFunctionDecl.Overridden = false;
@@ -6195,8 +6228,9 @@ class AbstractFunctionDecl : public GenericContext, public ValueDecl {
   void setBodyToBeReparsed(SourceRange bodyRange);
 
   /// Provide the parsed body for the function.
-  void setBodyParsed(BraceStmt *S) {
+  void setBodyParsed(BraceStmt *S, Optional<Fingerprint> fp = None) {
     setBody(S, BodyKind::Parsed);
+    BodyAndFP.setFingerprint(fp);
   }
 
   /// Was there a nested type declaration detected when parsing this
@@ -6286,6 +6320,15 @@ class AbstractFunctionDecl : public GenericContext, public ValueDecl {
   /// itself.
   void keepOriginalBodySourceRange();
 
+  /// Retrieve the fingerprint of the body. Note that this is not affected by
+  /// the body of the local functions or the members of the local types in this
+  /// function.
+  Optional<Fingerprint> getBodyFingerprint() const;
+
+  /// Retrieve the fingerprint of the body including the local type members and
+  /// the local funcition bodies.
+  Optional<Fingerprint> getBodyFingerprintIncludingLocalTypeMembers() const;
+
   /// Retrieve the source range of the *original* function body.
   ///
   /// This may be different from \c getBodySourceRange() that returns the source

include/swift/AST/DiagnosticsFrontend.def

@@ -102,6 +102,8 @@ ERROR(error_immediate_mode_primary_file,none,
       "immediate mode is incompatible with -primary-file", ())
 ERROR(error_missing_frontend_action,none,
       "no frontend action was selected", ())
+ERROR(error_unsupported_frontend_action, none,
+      "unsupported action: %0", (StringRef))
 ERROR(error_invalid_source_location_str,none,
       "invalid source location string '%0'", (StringRef))
 ERROR(error_no_source_location_scope_map,none,

include/swift/AST/ParseRequests.h

@@ -63,25 +63,25 @@ class ParseMembersRequest
 };
 
 /// Parse the body of a function, initializer, or deinitializer.
-class ParseAbstractFunctionBodyRequest :
-    public SimpleRequest<ParseAbstractFunctionBodyRequest,
-                         BraceStmt *(AbstractFunctionDecl *),
-                         RequestFlags::SeparatelyCached>
-{
+class ParseAbstractFunctionBodyRequest
+    : public SimpleRequest<ParseAbstractFunctionBodyRequest,
+                           BodyAndFingerprint(AbstractFunctionDecl *),
+                           RequestFlags::SeparatelyCached> {
 public:
   using SimpleRequest::SimpleRequest;
 
 private:
   friend SimpleRequest;
 
   // Evaluation.
-  BraceStmt *evaluate(Evaluator &evaluator, AbstractFunctionDecl *afd) const;
+  BodyAndFingerprint evaluate(Evaluator &evaluator,
+                              AbstractFunctionDecl *afd) const;
 
 public:
   // Caching
   bool isCached() const { return true; }
-  Optional<BraceStmt *> getCachedResult() const;
-  void cacheResult(BraceStmt *value) const;
+  Optional<BodyAndFingerprint> getCachedResult() const;
+  void cacheResult(BodyAndFingerprint value) const;
 };
 
 struct SourceFileParsingResult {

include/swift/AST/ParseTypeIDZone.def

@@ -20,7 +20,7 @@ SWIFT_REQUEST(Parse, CodeCompletionSecondPassRequest,
 SWIFT_REQUEST(Parse, ParseMembersRequest,
               FingerprintAndMembers(IterableDeclContext *), Cached, NoLocationInfo)
 SWIFT_REQUEST(Parse, ParseAbstractFunctionBodyRequest,
-              BraceStmt *(AbstractFunctionDecl *), SeparatelyCached,
+              BodyAndFingerprint(AbstractFunctionDecl *), SeparatelyCached,
               NoLocationInfo)
 SWIFT_REQUEST(Parse, ParseSourceFileRequest,
               SourceFileParsingResult(SourceFile *), SeparatelyCached,

include/swift/Basic/SourceManager.h

@@ -50,19 +50,10 @@ class SourceManager {
   /// to speed up stats.
   mutable llvm::DenseMap<StringRef, llvm::vfs::Status> StatusCache;
 
-  struct ReplacedRangeType {
-    SourceRange Original;
-    SourceRange New;
-    ReplacedRangeType() {}
-    ReplacedRangeType(NoneType) {}
-    ReplacedRangeType(SourceRange Original, SourceRange New)
-        : Original(Original), New(New) {
-      assert(Original.isValid() && New.isValid());
-    }
-
-    explicit operator bool() const { return Original.isValid(); }
-  };
-  ReplacedRangeType ReplacedRange;
+  /// Holds replaced ranges. Keys are orignal ranges, and values are new ranges
+  /// in different buffers. This is used for code completion and ASTContext
+  /// reusing compilation.
+  llvm::DenseMap<SourceRange, SourceRange> ReplacedRanges;
 
   std::map<const char *, VirtualFile> VirtualFiles;
   mutable std::pair<const char *, const VirtualFile*> CachedVFile = {nullptr, nullptr};
@@ -109,8 +100,12 @@ class SourceManager {
 
   SourceLoc getCodeCompletionLoc() const;
 
-  const ReplacedRangeType &getReplacedRange() const { return ReplacedRange; }
-  void setReplacedRange(const ReplacedRangeType &val) { ReplacedRange = val; }
+  const llvm::DenseMap<SourceRange, SourceRange> &getReplacedRanges() const {
+    return ReplacedRanges;
+  }
+  void setReplacedRange(SourceRange Orig, SourceRange New) {
+    ReplacedRanges[Orig] = New;
+  }
 
   /// Returns true if \c LHS is before \c RHS in the source buffer.
   bool isBeforeInBuffer(SourceLoc LHS, SourceLoc RHS) const {

include/swift/Frontend/Frontend.h

@@ -53,6 +53,7 @@
 
 namespace swift {
 
+class FrontendObserver;
 class SerializedModuleLoaderBase;
 class MemoryBufferSerializedModuleLoader;
 class SILModule;
@@ -663,6 +664,9 @@ class CompilerInstance {
   /// library, returning \c false if we should continue, i.e. no error.
   bool loadStdlibIfNeeded();
 
+  /// If \p fn returns true, exits early and returns true.
+  bool forEachFileToTypeCheck(llvm::function_ref<bool(SourceFile &)> fn);
+
 private:
   /// Compute the parsing options for a source file in the main module.
   SourceFile::ParsingOptions getSourceFileParsingOptions(bool forPrimary) const;
@@ -676,8 +680,6 @@ class CompilerInstance {
   bool loadPartialModulesAndImplicitImports(
       ModuleDecl *mod, SmallVectorImpl<FileUnit *> &partialModules) const;
 
-  void forEachFileToTypeCheck(llvm::function_ref<void(SourceFile &)> fn);
-
   void finishTypeChecking();
 
 public:

include/swift/Frontend/FrontendOptions.h

@@ -438,6 +438,10 @@ class FrontendOptions {
   /// Whether to include symbols with SPI information in the symbol graph.
   bool IncludeSPISymbolsInSymbolGraph = false;
 
+  /// Whether to reuse a frontend (i.e. compiler instance) for multiple
+  /// compiletions. This prevents ASTContext being freed.
+  bool ReuseFrontendForMutipleCompilations = false;
+
   /// This is used to obfuscate the serialized search paths so we don't have
   /// to encode the actual paths into the .swiftmodule file.
   PathObfuscator serializedPathObfuscator;

include/swift/FrontendTool/FrontendTool.h

@@ -78,6 +78,8 @@ int performFrontend(ArrayRef<const char *> args,
                     void *mainAddr,
                     FrontendObserver *observer = nullptr);
 
+bool performCompileStepsPostSema(CompilerInstance &Instance, int &ReturnValue,
+                                 FrontendObserver *observer);
 
 } // namespace swift
 

include/swift/IDE/CompileInstance.h

@@ -0,0 +1,83 @@
+//===--- CompileInstance.h ------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_IDE_COMPILEINSTANCE_H
+#define SWIFT_IDE_COMPILEINSTANCE_H
+
+#include "swift/Frontend/Frontend.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Chrono.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace swift {
+
+class CompilerInstance;
+class CompilerInvocation;
+class DiagnosticConsumer;
+
+namespace ide {
+
+/// Manages \c CompilerInstance for completion like operations.
+class CompileInstance {
+  const std::string &RuntimeResourcePath;
+  const std::string &DiagnosticDocumentationPath;
+
+  struct Options {
+    unsigned MaxASTReuseCount = 100;
+  } Opts;
+
+  std::mutex mtx;
+
+  std::unique_ptr<CompilerInstance> CI;
+  llvm::hash_code CachedArgHash;
+  std::atomic<bool> CachedCIInvalidated;
+  unsigned CachedReuseCount;
+
+  /// Perform cached sema. Returns \c true if the CI is not reusable.
+  bool performCachedSemaIfPossible(DiagnosticConsumer *DiagC);
+
+  /// Setup the CI with \p Args . Returns \c true if failed.
+  bool setupCI(llvm::ArrayRef<const char *> Args,
+                 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
+                 DiagnosticConsumer *DiagC);
+
+  /// Perform Parse and Sema, potentially CI from previous compilation is
+  /// reused.
+  void performSema(llvm::ArrayRef<const char *> Args,
+                   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
+                   DiagnosticConsumer *DiagC,
+                   std::shared_ptr<std::atomic<bool>> CancellationFlag);
+
+public:
+  CompileInstance(const std::string &RuntimeResourcePath,
+                  const std::string &DiagnosticDocumentationPath)
+      : RuntimeResourcePath(RuntimeResourcePath),
+        DiagnosticDocumentationPath(DiagnosticDocumentationPath),
+        CachedCIInvalidated(false), CachedReuseCount(0) {}
+
+  /// NOTE: \p Args is only used for checking the equaity of the invocation.
+  /// Since this function assumes that it is already normalized, exact the same
+  /// arguments including their order is considered as the same invocation.
+  bool
+  performCompile(llvm::ArrayRef<const char *> Args,
+                 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FileSystem,
+                 DiagnosticConsumer *DiagC,
+                 std::shared_ptr<std::atomic<bool>> CancellationFlag);
+};
+
+} // namespace ide
+} // namespace swift
+
+#endif // SWIFT_IDE_COMPILEINSTANCE_H

include/swift/Parse/Parser.h

@@ -1195,9 +1195,11 @@ class Parser {
                                        ParseDeclOptions Flags,
                                        DeclAttributes &Attributes,
                                        bool HasFuncKeyword = true);
-  BraceStmt *parseAbstractFunctionBodyImpl(AbstractFunctionDecl *AFD);
+  BodyAndFingerprint
+  parseAbstractFunctionBodyImpl(AbstractFunctionDecl *AFD);
   void parseAbstractFunctionBody(AbstractFunctionDecl *AFD);
-  BraceStmt *parseAbstractFunctionBodyDelayed(AbstractFunctionDecl *AFD);
+  BodyAndFingerprint
+  parseAbstractFunctionBodyDelayed(AbstractFunctionDecl *AFD);
   ParserResult<ProtocolDecl> parseDeclProtocol(ParseDeclOptions Flags,
                                                DeclAttributes &Attributes);
 

lib/AST/ASTScopeLookup.cpp

@@ -78,10 +78,10 @@ ASTScopeImpl *ASTScopeImpl::findInnermostEnclosingScopeImpl(
 static SourceLoc translateLocForReplacedRange(SourceManager &sourceMgr,
                                               CharSourceRange range,
                                               SourceLoc loc) {
-  if (const auto &replacedRange = sourceMgr.getReplacedRange()) {
-    if (sourceMgr.rangeContainsTokenLoc(replacedRange.New, loc) &&
-        !sourceMgr.rangeContainsTokenLoc(replacedRange.New, range.getStart())) {
-      return replacedRange.Original.Start;
+  for (const auto &pair : sourceMgr.getReplacedRanges()) {
+    if (sourceMgr.rangeContainsTokenLoc(pair.second, loc) &&
+        !sourceMgr.rangeContainsTokenLoc(pair.second, range.getStart())) {
+      return pair.first.Start;
     }
   }
   return loc;

lib/AST/ASTScopeSourceRange.cpp

@@ -55,12 +55,11 @@ void ASTScopeImpl::checkSourceRangeBeforeAddingChild(ASTScopeImpl *child,
 
   bool childContainedInParent = [&]() {
     // HACK: For code completion. Handle replaced range.
-    if (const auto &replacedRange = sourceMgr.getReplacedRange()) {
-      auto originalRange = Lexer::getCharSourceRangeFromSourceRange(
-          sourceMgr, replacedRange.Original);
-      auto newRange = Lexer::getCharSourceRangeFromSourceRange(
-          sourceMgr, replacedRange.New);
-
+    for (const auto &pair : sourceMgr.getReplacedRanges()) {
+      auto originalRange =
+          Lexer::getCharSourceRangeFromSourceRange(sourceMgr, pair.first);
+      auto newRange =
+          Lexer::getCharSourceRangeFromSourceRange(sourceMgr, pair.second);
       if (range.contains(originalRange) &&
           newRange.contains(childCharRange))
         return true;