Merge pull request #1185 from swiftwasm/master

[pull] swiftwasm from master

Author: kateinoigakukun

include/swift/AST/ASTContext.h

@@ -407,6 +407,12 @@ class ASTContext final {
                               array.size());
   }
 
+  template <typename T>
+  MutableArrayRef<T>
+  AllocateCopy(const std::vector<T> &vec,
+               AllocationArena arena = AllocationArena::Permanent) const {
+    return AllocateCopy(ArrayRef<T>(vec), arena);
+  }
 
   template<typename T>
   ArrayRef<T> AllocateCopy(const SmallVectorImpl<T> &vec,

include/swift/AST/ParseRequests.h

@@ -19,6 +19,7 @@
 #include "swift/AST/ASTTypeIDs.h"
 #include "swift/AST/EvaluatorDependencies.h"
 #include "swift/AST/SimpleRequest.h"
+#include "swift/Syntax/SyntaxNodes.h"
 
 namespace swift {
 
@@ -81,10 +82,17 @@ class ParseAbstractFunctionBodyRequest :
   void cacheResult(BraceStmt *value) const;
 };
 
+struct SourceFileParsingResult {
+  ArrayRef<Decl *> TopLevelDecls;
+  Optional<ArrayRef<Token>> CollectedTokens;
+  Optional<llvm::MD5> InterfaceHash;
+  Optional<syntax::SourceFileSyntax> SyntaxRoot;
+};
+
 /// Parse the top-level decls of a SourceFile.
 class ParseSourceFileRequest
     : public SimpleRequest<
-          ParseSourceFileRequest, ArrayRef<Decl *>(SourceFile *),
+          ParseSourceFileRequest, SourceFileParsingResult(SourceFile *),
           RequestFlags::SeparatelyCached | RequestFlags::DependencySource> {
 public:
   using SimpleRequest::SimpleRequest;
@@ -93,13 +101,13 @@ class ParseSourceFileRequest
   friend SimpleRequest;
 
   // Evaluation.
-  ArrayRef<Decl *> evaluate(Evaluator &evaluator, SourceFile *SF) const;
+  SourceFileParsingResult evaluate(Evaluator &evaluator, SourceFile *SF) const;
 
 public:
   // Caching.
   bool isCached() const { return true; }
-  Optional<ArrayRef<Decl *>> getCachedResult() const;
-  void cacheResult(ArrayRef<Decl *> decls) const;
+  Optional<SourceFileParsingResult> getCachedResult() const;
+  void cacheResult(SourceFileParsingResult result) const;
 
 public:
   evaluator::DependencySource

include/swift/AST/ParseTypeIDZone.def

@@ -23,5 +23,5 @@ SWIFT_REQUEST(Parse, ParseAbstractFunctionBodyRequest,
               BraceStmt *(AbstractFunctionDecl *), SeparatelyCached,
               NoLocationInfo)
 SWIFT_REQUEST(Parse, ParseSourceFileRequest,
-              ArrayRef<Decl *>(SourceFile *), SeparatelyCached,
+              SourceFileParsingResult(SourceFile *), SeparatelyCached,
               NoLocationInfo)

include/swift/AST/SourceFile.h

@@ -32,8 +32,6 @@ class SourceFile final : public FileUnit {
   friend class ParseSourceFileRequest;
 
 public:
-  struct SourceFileSyntaxInfo;
-
   /// Possible attributes for imports in source files.
   enum class ImportFlags {
     /// The imported module is exposed to anyone who imports the parent module.
@@ -107,15 +105,27 @@ class SourceFile final : public FileUnit {
     /// decl.
     ///
     /// FIXME: When condition evaluation moves to a later phase, remove this
-    /// and adjust the client call 'performParseOnly'.
+    /// and the associated language option.
     DisablePoundIfEvaluation = 1 << 1,
 
+    /// Whether to build a syntax tree.
+    BuildSyntaxTree = 1 << 2,
+
+    /// Whether to save the file's parsed tokens.
+    CollectParsedTokens = 1 << 3,
+
+    /// Whether to compute the interface hash of the file.
+    EnableInterfaceHash = 1 << 4,
+
     /// Whether to suppress warnings when parsing. This is set for secondary
     /// files, as they get parsed multiple times.
-    SuppressWarnings = 1 << 2
+    SuppressWarnings = 1 << 5,
   };
   using ParsingOptions = OptionSet<ParsingFlags>;
 
+  /// Retrieve the parsing options specified in the LangOptions.
+  static ParsingOptions getDefaultParsingOptions(const LangOptions &langOpts);
+
 private:
   std::unique_ptr<SourceLookupCache> Cache;
   SourceLookupCache &getCache() const;
@@ -313,7 +323,6 @@ class SourceFile final : public FileUnit {
   llvm::StringMap<SourceFilePathInfo> getInfoForUsedFilePaths() const;
 
   SourceFile(ModuleDecl &M, SourceFileKind K, Optional<unsigned> bufferID,
-             bool KeepParsedTokens = false, bool KeepSyntaxTree = false,
              ParsingOptions parsingOpts = {}, bool isPrimary = false);
 
   ~SourceFile();
@@ -481,7 +490,7 @@ class SourceFile final : public FileUnit {
   }
 
   SWIFT_DEBUG_DUMP;
-  void dump(raw_ostream &os) const;
+  void dump(raw_ostream &os, bool parseIfNeeded = false) const;
 
   /// Pretty-print the contents of this source file.
   ///
@@ -544,50 +553,35 @@ class SourceFile final : public FileUnit {
   /// Set the root refinement context for the file.
   void setTypeRefinementContext(TypeRefinementContext *TRC);
 
-  void enableInterfaceHash() {
-    assert(!hasInterfaceHash());
-    InterfaceHash.emplace();
-  }
-
+  /// Whether this file has an interface hash available.
   bool hasInterfaceHash() const {
-    return InterfaceHash.hasValue();
+    return ParsingOpts.contains(ParsingFlags::EnableInterfaceHash);
   }
 
-  NullablePtr<llvm::MD5> getInterfaceHashPtr() {
-    return InterfaceHash ? InterfaceHash.getPointer() : nullptr;
-  }
-
-  void getInterfaceHash(llvm::SmallString<32> &str) const {
-    // Copy to preserve idempotence.
-    llvm::MD5 md5 = *InterfaceHash;
-    llvm::MD5::MD5Result result;
-    md5.final(result);
-    llvm::MD5::stringifyResult(result, str);
-  }
+  /// Output this file's interface hash into the provided string buffer.
+  void getInterfaceHash(llvm::SmallString<32> &str) const;
 
   void dumpInterfaceHash(llvm::raw_ostream &out) {
     llvm::SmallString<32> str;
     getInterfaceHash(str);
     out << str << '\n';
   }
 
-  std::vector<Token> &getTokenVector();
-
+  /// If this source file has been told to collect its parsed tokens, retrieve
+  /// those tokens.
   ArrayRef<Token> getAllTokens() const;
 
-  bool shouldCollectToken() const;
+  /// Whether the parsed tokens of this source file should be saved, allowing
+  /// them to be accessed from \c getAllTokens.
+  bool shouldCollectTokens() const;
 
   bool shouldBuildSyntaxTree() const;
 
-  bool canBeParsedInFull() const;
-
   /// Whether the bodies of types and functions within this file can be lazily
   /// parsed.
   bool hasDelayedBodyParsing() const;
 
   syntax::SourceFileSyntax getSyntaxRoot() const;
-  void setSyntaxRoot(syntax::SourceFileSyntax &&Root);
-  bool hasSyntaxRoot() const;
 
   OpaqueTypeDecl *lookupOpaqueResultType(StringRef MangledName) override;
 
@@ -602,10 +596,12 @@ class SourceFile final : public FileUnit {
 
 private:
 
-  /// If not None, the underlying vector should contain tokens of this source file.
-  Optional<std::vector<Token>> AllCorrectedTokens;
+  /// If not \c None, the underlying vector contains the parsed tokens of this
+  /// source file.
+  Optional<ArrayRef<Token>> AllCollectedTokens;
 
-  std::unique_ptr<SourceFileSyntaxInfo> SyntaxInfo;
+  /// The root of the syntax tree representing the source file.
+  std::unique_ptr<syntax::SourceFileSyntax> SyntaxRoot;
 };
 
 inline SourceFile::ParsingOptions operator|(SourceFile::ParsingFlags lhs,

include/swift/Basic/LangOptions.h

@@ -335,6 +335,10 @@ namespace swift {
     /// Whether to verify the parsed syntax tree and emit related diagnostics.
     bool VerifySyntaxTree = false;
 
+    /// Whether to disable the evaluation of '#if' decls, such that the bodies
+    /// of active clauses aren't hoisted into the enclosing scope.
+    bool DisablePoundIfEvaluation = false;
+
     /// Instead of hashing tokens inside of NominalType and ExtensionBodies into
     /// the interface hash, hash them into per-iterable-decl-context
     /// fingerprints. Fine-grained dependency types won't dirty every provides

include/swift/Frontend/Frontend.h

@@ -441,8 +441,9 @@ class CompilerInstance {
   std::vector<unsigned> InputSourceCodeBufferIDs;
 
   /// Contains \c MemoryBuffers for partial serialized module files and
-  /// corresponding partial serialized module documentation files.
-  std::vector<ModuleBuffers> PartialModules;
+  /// corresponding partial serialized module documentation files. This is
+  /// \c mutable as it is consumed by \c loadPartialModulesAndImplicitImports.
+  mutable std::vector<ModuleBuffers> PartialModules;
 
   enum : unsigned { NO_SUCH_BUFFER = ~0U };
   unsigned MainBufferID = NO_SUCH_BUFFER;
@@ -460,7 +461,7 @@ class CompilerInstance {
   /// If \p BufID is already in the set, do nothing.
   void recordPrimaryInputBuffer(unsigned BufID);
 
-  bool isWholeModuleCompilation() { return PrimaryBufferIDs.empty(); }
+  bool isWholeModuleCompilation() const { return PrimaryBufferIDs.empty(); }
 
 public:
   // Out of line to avoid having to import SILModule.h.
@@ -615,10 +616,6 @@ class CompilerInstance {
   /// Parses and type-checks all input files.
   void performSema();
 
-  /// Parses the input file but does no type-checking or module imports.
-  void performParseOnly(bool EvaluateConditionals = false,
-                        bool CanDelayBodies = true);
-
   /// Parses and performs import resolution on all input files.
   ///
   /// This is similar to a parse-only invocation, but module imports will also
@@ -631,10 +628,15 @@ class CompilerInstance {
   bool performSILProcessing(SILModule *silModule);
 
 private:
-  SourceFile *
-  createSourceFileForMainModule(SourceFileKind FileKind,
-                                Optional<unsigned> BufferID,
-                                SourceFile::ParsingOptions options = {});
+  /// Creates a new source file for the main module.
+  SourceFile *createSourceFileForMainModule(ModuleDecl *mod,
+                                            SourceFileKind FileKind,
+                                            Optional<unsigned> BufferID) const;
+
+  /// Creates all the files to be added to the main module, appending them to
+  /// \p files. If a loading error occurs, returns \c true.
+  bool createFilesForMainModule(ModuleDecl *mod,
+                                SmallVectorImpl<FileUnit *> &files) const;
 
 public:
   void freeASTContext();
@@ -644,14 +646,17 @@ class CompilerInstance {
   bool loadStdlibIfNeeded();
 
 private:
+  /// Compute the parsing options for a source file in the main module.
+  SourceFile::ParsingOptions getSourceFileParsingOptions(bool forPrimary) const;
+
   /// Retrieve a description of which modules should be implicitly imported.
   ImplicitImportInfo getImplicitImportInfo() const;
 
-  void performSemaUpTo(SourceFile::ASTStage_t LimitStage,
-                       SourceFile::ParsingOptions POpts = {});
-
-  /// Return true if had load error
-  bool loadPartialModulesAndImplicitImports();
+  /// For any serialized AST inputs, loads them in as partial module files,
+  /// appending them to \p partialModules. If a loading error occurs, returns
+  /// \c true.
+  bool loadPartialModulesAndImplicitImports(
+      ModuleDecl *mod, SmallVectorImpl<FileUnit *> &partialModules) const;
 
   void forEachFileToTypeCheck(llvm::function_ref<void(SourceFile &)> fn);
 

include/swift/Parse/Parser.h

@@ -22,6 +22,7 @@
 #include "swift/AST/Expr.h"
 #include "swift/AST/DiagnosticsParse.h"
 #include "swift/AST/LayoutConstraint.h"
+#include "swift/AST/ParseRequests.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
 #include "swift/Basic/OptionSet.h"
@@ -92,8 +93,11 @@ class ConsumeTokenReceiver {
   /// This is called to update the kind of a token whose start location is Loc.
   virtual void registerTokenKindChange(SourceLoc Loc, tok NewKind) {};
 
-  /// This is called when a source file is fully parsed.
-  virtual void finalize() {};
+  /// This is called when a source file is fully parsed. It returns the
+  /// finalized vector of tokens, or \c None if the receiver isn't configured to
+  /// record them.
+  virtual Optional<std::vector<Token>> finalize() { return None; }
+
   virtual ~ConsumeTokenReceiver() = default;
 };
 
@@ -124,7 +128,10 @@ class Parser {
   /// Tracks parsed decls that LLDB requires to be inserted at the top-level.
   std::vector<Decl *> ContextSwitchedTopLevelDecls;
 
-  NullablePtr<llvm::MD5> CurrentTokenHash;
+  /// The current token hash, or \c None if the parser isn't computing a hash
+  /// for the token stream.
+  Optional<llvm::MD5> CurrentTokenHash;
+
   void recordTokenHash(const Token Tok) {
     if (!Tok.getText().empty())
       recordTokenHash(Tok.getText());
@@ -416,6 +423,14 @@ class Parser {
     return SyntaxContext->finalizeRoot();
   }
 
+  /// Retrieve the token receiver from the parser once it has finished parsing.
+  std::unique_ptr<ConsumeTokenReceiver> takeTokenReceiver() {
+    assert(Tok.is(tok::eof) && "not done parsing yet");
+    auto *receiver = TokReceiver;
+    TokReceiver = nullptr;
+    return std::unique_ptr<ConsumeTokenReceiver>(receiver);
+  }
+
   //===--------------------------------------------------------------------===//
   // Routines to save and restore parser state.
 
@@ -472,6 +487,9 @@ class Parser {
       void receive(Token tok) override {
         delayedTokens.push_back(tok);
       }
+      Optional<std::vector<Token>> finalize() override {
+        llvm_unreachable("Cannot finalize a DelayedTokenReciever");
+      }
       ~DelayedTokenReceiver() {
         if (!shouldTransfer)
           return;

include/swift/Parse/SyntaxParseActions.h

@@ -24,11 +24,13 @@ namespace swift {
 
 class CharSourceRange;
 class ParsedTriviaPiece;
+class SourceFile;
 class SourceLoc;
 enum class tok;
 
 namespace syntax {
-  enum class SyntaxKind;
+class SourceFileSyntax;
+enum class SyntaxKind;
 }
 
 typedef void *OpaqueSyntaxNode;
@@ -55,6 +57,12 @@ class SyntaxParseActions {
                                            ArrayRef<OpaqueSyntaxNode> elements,
                                            CharSourceRange range) = 0;
 
+  /// Attempt to realize an opaque raw syntax node for a source file into a
+  /// SourceFileSyntax node. This will return \c None if the parsing action
+  /// doesn't support the realization of syntax nodes.
+  virtual Optional<syntax::SourceFileSyntax>
+  realizeSyntaxRoot(OpaqueSyntaxNode root, const SourceFile &SF) = 0;
+
   /// Discard raw syntax node.
   /// 
   /// FIXME: This breaks invariant that any recorded node will be a part of the