[clang][deps] Cherry-pick recent changes

clang/include/clang-c/Dependencies.h

@@ -200,42 +200,6 @@ CINDEX_LINKAGE void clang_experimental_DependencyScannerWorker_dispose_v0(
 typedef void CXModuleDiscoveredCallback(void *Context,
                                         CXModuleDependencySet *MDS);
 
-/**
- * Returns the list of file dependencies for a particular compiler invocation.
- *
- * \param argc the number of compiler invocation arguments (including argv[0]).
- * \param argv the compiler driver invocation arguments (including argv[0]).
- * \param WorkingDirectory the directory in which the invocation runs.
- * \param MDC a callback that is called whenever a new module is discovered.
- *            This may receive the same module on different workers. This should
- *            be NULL if
- *            \c clang_experimental_DependencyScannerService_create_v0 was
- *            called with \c CXDependencyMode_Flat. This callback will be called
- *            on the same thread that called this function.
- * \param Context the context that will be passed to \c MDC each time it is
- *                called.
- * \param [out] error the error string to pass back to client (if any).
- *
- * \returns A pointer to a CXFileDependencies on success, NULL otherwise. The
- *          CXFileDependencies must be freed by calling
- *          \c clang_experimental_FileDependencies_dispose.
- */
-CINDEX_LINKAGE CXFileDependencies *
-clang_experimental_DependencyScannerWorker_getFileDependencies_v2(
-    CXDependencyScannerWorker Worker, int argc, const char *const *argv,
-    const char *WorkingDirectory, CXModuleDiscoveredCallback *MDC,
-    void *Context, CXString *error);
-
-/**
- * Same as \c clang_experimental_DependencyScannerWorker_getFileDependencies_v2,
- * but get the dependencies by module name alone.
- */
-CINDEX_LINKAGE CXFileDependencies *
-clang_experimental_DependencyScannerWorker_getDependenciesByModuleName_v0(
-    CXDependencyScannerWorker Worker, int argc, const char *const *argv,
-    const char *ModuleName, const char *WorkingDirectory,
-    CXModuleDiscoveredCallback *MDC, void *Context, CXString *error);
-
 /**
  * A callback that is called to determine the paths of output files for each
  * module dependency. The ModuleFile (pcm) path mapping is mandatory.

clang/include/clang/Basic/Module.h

@@ -352,6 +352,10 @@ class Module {
   /// module depends.
   llvm::SmallSetVector<Module *, 2> Imports;
 
+  /// The set of top-level modules that affected the compilation of this module,
+  /// but were not imported.
+  llvm::SmallSetVector<Module *, 2> AffectingModules;
+
   /// Describes an exported module.
   ///
   /// The pointer is the module being re-exported, while the bit will be true

clang/include/clang/Frontend/CompilerInvocation.h

@@ -243,7 +243,7 @@ class CompilerInvocation : public CompilerInvocationRefBase,
   std::string getModuleHash(DiagnosticsEngine &Diags) const;
 
   using StringAllocator = llvm::function_ref<const char *(const llvm::Twine &)>;
-  /// Generate a cc1-compatible command line arguments from this instance.
+  /// Generate cc1-compatible command line arguments from this instance.
   ///
   /// \param [out] Args - The generated arguments. Note that the caller is
   /// responsible for inserting the path to the clang executable and "-cc1" if
@@ -254,6 +254,20 @@ class CompilerInvocation : public CompilerInvocationRefBase,
   void generateCC1CommandLine(llvm::SmallVectorImpl<const char *> &Args,
                               StringAllocator SA) const;
 
+  /// Generate cc1-compatible command line arguments from this instance,
+  /// wrapping the result as a std::vector<std::string>.
+  ///
+  /// This is a (less-efficient) wrapper over generateCC1CommandLine().
+  std::vector<std::string> getCC1CommandLine() const;
+
+  /// Reset all of the options that are not considered when building a
+  /// module.
+  void resetNonModularOptions();
+
+  /// Disable implicit modules and canonicalize options that are only used by
+  /// implicit modules.
+  void clearImplicitModuleBuildOptions();
+
   /// Parse command line options that map to \p CASOptions.
   static bool ParseCASArgs(CASOptions &Opts, const llvm::opt::ArgList &Args,
                            DiagnosticsEngine &Diags);

clang/include/clang/Lex/ModuleLoader.h

@@ -51,6 +51,11 @@ class ModuleLoadResult {
   ModuleLoadResult() = default;
   ModuleLoadResult(Module *M) : Storage(M, Normal) {}
   ModuleLoadResult(LoadResultKind Kind) : Storage(nullptr, Kind) {}
+  ModuleLoadResult(Module *M, LoadResultKind Kind) : Storage(M, Kind) {}
+
+  operator bool() const {
+    return Storage.getInt() == Normal && Storage.getPointer();
+  }
 
   operator Module *() const { return Storage.getPointer(); }
 

clang/include/clang/Lex/Preprocessor.h

@@ -798,6 +798,10 @@ class Preprocessor {
   /// The files that have been included.
   IncludedFilesSet IncludedFiles;
 
+  /// The set of top-level modules that affected preprocessing, but were not
+  /// imported.
+  llvm::SmallSetVector<Module *, 2> AffectingModules;
+
   /// The set of known macros exported from modules.
   llvm::FoldingSet<ModuleMacro> ModuleMacros;
 
@@ -1274,6 +1278,22 @@ class Preprocessor {
 
   /// \}
 
+  /// Mark the given module as affecting the current module or translation unit.
+  void markModuleAsAffecting(Module *M) {
+    if (!BuildingSubmoduleStack.empty()) {
+      if (M != BuildingSubmoduleStack.back().M)
+        BuildingSubmoduleStack.back().M->AffectingModules.insert(M);
+    } else {
+      AffectingModules.insert(M);
+    }
+  }
+
+  /// Get the set of top-level modules that affected preprocessing, but were not
+  /// imported.
+  const llvm::SmallSetVector<Module *, 2> &getAffectingModules() const {
+    return AffectingModules;
+  }
+
   /// Mark the file as included.
   /// Returns true if this is the first time the file was included.
   bool markIncluded(const FileEntry *File) {

clang/include/clang/Serialization/ASTBitCodes.h

@@ -835,6 +835,9 @@ enum SubmoduleRecordTypes {
   /// Specifies the name of the module that will eventually
   /// re-export the entities in this module.
   SUBMODULE_EXPORT_AS = 17,
+
+  /// Specifies affecting modules that were not imported.
+  SUBMODULE_AFFECTING_MODULES = 18,
 };
 
 /// Record types used within a comments block.

clang/include/clang/Serialization/ASTReader.h

@@ -696,7 +696,7 @@ class ASTReader
     Module *Mod;
 
     /// The kind of module reference.
-    enum { Import, Export, Conflict } Kind;
+    enum { Import, Export, Conflict, Affecting } Kind;
 
     /// The local ID of the module that is being exported.
     unsigned ID;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

@@ -61,26 +61,27 @@ class DependencyScanningService {
       ScanningMode Mode, ScanningOutputFormat Format, CASOptions CASOpts,
       std::shared_ptr<llvm::cas::ActionCache> Cache,
       IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> SharedFS,
-      bool ReuseFileManager = true, bool OptimizeArgs = false);
-
-  ~DependencyScanningService();
+      bool ReuseFileManager = true, bool OptimizeArgs = false,
+      bool EagerLoadModules = false);
 
   ScanningMode getMode() const { return Mode; }
 
   ScanningOutputFormat getFormat() const { return Format; }
 
-  const CASOptions &getCASOpts() const { return CASOpts; }
-
   bool canReuseFileManager() const { return ReuseFileManager; }
 
   bool canOptimizeArgs() const { return OptimizeArgs; }
 
+  bool shouldEagerLoadModules() const { return EagerLoadModules; }
+
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     assert(!SharedFS && "Expected not to have a CASFS");
     assert(SharedCache && "Expected a shared cache");
     return *SharedCache;
   }
 
+  const CASOptions &getCASOpts() const { return CASOpts; }
+
   llvm::cas::ActionCache &getCache() const {
     assert(Cache && "Cache is not initialized");
     return *Cache;
@@ -98,6 +99,8 @@ class DependencyScanningService {
   const bool ReuseFileManager;
   /// Whether to optimize the modules' command-line arguments.
   const bool OptimizeArgs;
+  /// Whether to set up command-lines to load PCM files eagerly.
+  const bool EagerLoadModules;
   /// Shared CachingOnDiskFileSystem. Set to nullptr to not use CAS dependency
   /// scanning.
   IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> SharedFS;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -13,9 +13,12 @@
 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Tooling/JSONCompilationDatabase.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/CAS/CASID.h"
 #include <string>
+#include <vector>
 
 namespace llvm {
 namespace cas {
@@ -30,6 +33,10 @@ class IncludeTreeRoot;
 namespace tooling {
 namespace dependencies {
 
+/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc.
+using LookupModuleOutputCallback =
+    llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>;
+
 /// The full dependencies and module graph for a specific input.
 struct FullDependencies {
   /// The identifier of the C++20 module this translation unit exports.
@@ -52,23 +59,11 @@ struct FullDependencies {
   /// determined that the differences are benign for this compilation.
   std::vector<ModuleID> ClangModuleDeps;
 
-  /// The original command line of the TU (excluding the compiler executable).
-  std::vector<std::string> OriginalCommandLine;
+  /// The command line of the TU (excluding the compiler executable).
+  std::vector<std::string> CommandLine;
 
   /// The CASID for input file dependency tree.
   llvm::Optional<llvm::cas::CASID> CASFileSystemRootID;
-
-  /// Get the full command line.
-  ///
-  /// \param LookupOutput This function is called to fill in
-  ///                     "-fmodule-file=", "-o" and other output
-  ///                     arguments for dependencies.
-  std::vector<std::string> getCommandLine(
-      llvm::function_ref<std::string(const ModuleID &, ModuleOutputKind)>
-          LookupOutput) const;
-
-  /// Get the full command line, excluding -fmodule-file=" arguments.
-  std::vector<std::string> getCommandLineWithoutModulePaths() const;
 };
 
 struct FullDependenciesResult {
@@ -134,12 +129,16 @@ class DependencyScanningTool {
   ///                    function for a single \c DependencyScanningTool in a
   ///                    single build. Use a different one for different tools,
   ///                    and clear it between builds.
+  /// \param LookupModuleOutput This function is called to fill in
+  ///                           "-fmodule-file=", "-o" and other output
+  ///                           arguments for dependencies.
   ///
   /// \returns a \c StringError with the diagnostic output if clang errors
   /// occurred, \c FullDependencies otherwise.
   llvm::Expected<FullDependenciesResult>
   getFullDependencies(const std::vector<std::string> &CommandLine,
                       StringRef CWD, const llvm::StringSet<> &AlreadySeen,
+                      LookupModuleOutputCallback LookupModuleOutput,
                       llvm::Optional<StringRef> ModuleName = None);
 
   ScanningOutputFormat getScanningFormat() const {
@@ -163,6 +162,53 @@ class DependencyScanningTool {
   DependencyScanningWorker Worker;
 };
 
+class FullDependencyConsumer : public DependencyConsumer {
+public:
+  FullDependencyConsumer(const llvm::StringSet<> &AlreadySeen,
+                         LookupModuleOutputCallback LookupModuleOutput,
+                         bool EagerLoadModules)
+      : AlreadySeen(AlreadySeen), LookupModuleOutput(LookupModuleOutput),
+        EagerLoadModules(EagerLoadModules) {}
+
+  void handleDependencyOutputOpts(const DependencyOutputOptions &) override {}
+
+  void handleFileDependency(StringRef File) override {
+    Dependencies.push_back(std::string(File));
+  }
+
+  void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {
+    PrebuiltModuleDeps.emplace_back(std::move(PMD));
+  }
+
+  void handleModuleDependency(ModuleDeps MD) override {
+    ClangModuleDeps[MD.ID.ContextHash + MD.ID.ModuleName] = std::move(MD);
+  }
+
+  void handleContextHash(std::string Hash) override {
+    ContextHash = std::move(Hash);
+  }
+
+  std::string lookupModuleOutput(const ModuleID &ID,
+                                 ModuleOutputKind Kind) override {
+    return LookupModuleOutput(ID, Kind);
+  }
+
+  FullDependenciesResult getFullDependencies(
+      const std::vector<std::string> &OriginalCommandLine,
+      Optional<cas::CASID> CASFileSystemRootID = None) const;
+
+private:
+  std::vector<std::string> Dependencies;
+  std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
+  llvm::MapVector<std::string, ModuleDeps, llvm::StringMap<unsigned>>
+      ClangModuleDeps;
+  std::string ContextHash;
+  std::vector<std::string> OutputPaths;
+  const llvm::StringSet<> &AlreadySeen;
+  LookupModuleOutputCallback LookupModuleOutput;
+  bool EagerLoadModules;
+};
+
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -45,6 +45,9 @@ class DependencyConsumer {
   virtual void handleModuleDependency(ModuleDeps MD) = 0;
 
   virtual void handleContextHash(std::string Hash) = 0;
+
+  virtual std::string lookupModuleOutput(const ModuleID &ID,
+                                         ModuleOutputKind Kind) = 0;
 };
 
 // FIXME: This may need to merge with \p DependencyConsumer in order to support
@@ -74,6 +77,9 @@ class PPIncludeActionsConsumer : public DependencyConsumer {
   void handleContextHash(std::string Hash) override {
     llvm::report_fatal_error("unexpected callback for include-tree");
   }
+  std::string lookupModuleOutput(const ModuleID &, ModuleOutputKind) override {
+    llvm::report_fatal_error("unexpected callback for include-tree");
+  }
 };
 
 /// An individual dependency scanning worker that is able to run on its own
@@ -99,6 +105,8 @@ class DependencyScanningWorker {
                                   DependencyConsumer &Consumer,
                                   llvm::Optional<StringRef> ModuleName = None);
 
+  bool shouldEagerLoadModules() const { return EagerLoadModules; }
+
   ScanningOutputFormat getFormat() const { return Format; }
 
   /// Scan from a compiler invocation.
@@ -131,21 +139,23 @@ class DependencyScanningWorker {
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
   /// The in-memory filesystem laid on top the physical filesystem in `RealFS`.
   llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFS;
-  /// The caching file system.
-  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;
   /// The file system that is used by each worker when scanning for
   /// dependencies. This filesystem persists across multiple compiler
   /// invocations.
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
-  /// The CAS Dependency Filesytem. This is not set at the sametime as DepFS;
-  llvm::IntrusiveRefCntPtr<DependencyScanningCASFilesystem> DepCASFS;
   /// The file manager that is reused across multiple invocations by this
   /// worker. If null, the file manager will not be reused.
   llvm::IntrusiveRefCntPtr<FileManager> Files;
   ScanningOutputFormat Format;
   /// Whether to optimize the modules' command-line arguments.
   bool OptimizeArgs;
+  /// Whether to set up command-lines to load PCM files eagerly.
+  bool EagerLoadModules;
 
+  /// The caching file system.
+  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS;
+  /// The CAS Dependency Filesytem. This is not set at the sametime as DepFS;
+  llvm::IntrusiveRefCntPtr<DependencyScanningCASFilesystem> DepCASFS;
   CASOptions CASOpts;
   bool UseCAS;
 };