[clang][deps] Split translation units into individual -cc1 or other commands

Instead of trying to "fix" the original driver invocation by appending
arguments to it, split it into multiple commands, and for each -cc1
command use a CompilerInvocation to give precise control over the
invocation.

This change should make it easier to (in the future) canonicalize the
command-line (e.g. to improve hits in something like ccache), apply
optimizations, or start supporting multi-arch builds, which would
require different modules for each arch.

In the long run it may make sense to treat the TU commands as a
dependency graph, each with their own dependencies on modules or earlier
TU commands, but for now they are simply a list that is executed in
order, and the dependencies are simply duplicated. Since we currently
only support single-arch builds, there is no parallelism available in
the execution.

Differential Revision: https://reviews.llvm.org/D132405

Author: benlangmuir

clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h

@@ -49,8 +49,16 @@ struct FullDependencies {
   /// determined that the differences are benign for this compilation.
   std::vector<ModuleID> ClangModuleDeps;
 
-  /// The command line of the TU (excluding the compiler executable).
-  std::vector<std::string> CommandLine;
+  /// The sequence of commands required to build the translation unit. Commands
+  /// should be executed in order.
+  ///
+  /// FIXME: If we add support for multi-arch builds in clang-scan-deps, we
+  /// should make the dependencies between commands explicit to enable parallel
+  /// builds of each architecture.
+  std::vector<Command> Commands;
+
+  /// Deprecated driver command-line. This will be removed in a future version.
+  std::vector<std::string> DriverCommandLine;
 };
 
 struct FullDependenciesResult {
@@ -99,6 +107,12 @@ class DependencyScanningTool {
                       LookupModuleOutputCallback LookupModuleOutput,
                       llvm::Optional<StringRef> ModuleName = None);
 
+  llvm::Expected<FullDependenciesResult> getFullDependenciesLegacyDriverCommand(
+      const std::vector<std::string> &CommandLine, StringRef CWD,
+      const llvm::StringSet<> &AlreadySeen,
+      LookupModuleOutputCallback LookupModuleOutput,
+      llvm::Optional<StringRef> ModuleName = None);
+
 private:
   DependencyScanningWorker Worker;
 };
@@ -111,6 +125,10 @@ class FullDependencyConsumer : public DependencyConsumer {
       : AlreadySeen(AlreadySeen), LookupModuleOutput(LookupModuleOutput),
         EagerLoadModules(EagerLoadModules) {}
 
+  void handleBuildCommand(Command Cmd) override {
+    Commands.push_back(std::move(Cmd));
+  }
+
   void handleDependencyOutputOpts(const DependencyOutputOptions &) override {}
 
   void handleFileDependency(StringRef File) override {
@@ -134,14 +152,17 @@ class FullDependencyConsumer : public DependencyConsumer {
     return LookupModuleOutput(ID, Kind);
   }
 
-  FullDependenciesResult getFullDependencies(
+  FullDependenciesResult getFullDependenciesLegacyDriverCommand(
       const std::vector<std::string> &OriginalCommandLine) const;
 
+  FullDependenciesResult takeFullDependencies();
+
 private:
   std::vector<std::string> Dependencies;
   std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
   llvm::MapVector<std::string, ModuleDeps, llvm::StringMap<unsigned>>
       ClangModuleDeps;
+  std::vector<Command> Commands;
   std::string ContextHash;
   std::vector<std::string> OutputPaths;
   const llvm::StringSet<> &AlreadySeen;

clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h

@@ -28,10 +28,20 @@ namespace dependencies {
 
 class DependencyScanningWorkerFilesystem;
 
+/// A command-line tool invocation that is part of building a TU.
+///
+/// \see FullDependencies::Commands.
+struct Command {
+  std::string Executable;
+  std::vector<std::string> Arguments;
+};
+
 class DependencyConsumer {
 public:
   virtual ~DependencyConsumer() {}
 
+  virtual void handleBuildCommand(Command Cmd) = 0;
+
   virtual void
   handleDependencyOutputOpts(const DependencyOutputOptions &Opts) = 0;
 

clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h

@@ -181,12 +181,16 @@ class ModuleDepCollector final : public DependencyCollector {
 public:
   ModuleDepCollector(std::unique_ptr<DependencyOutputOptions> Opts,
                      CompilerInstance &ScanInstance, DependencyConsumer &C,
-                     CompilerInvocation &&OriginalCI, bool OptimizeArgs,
+                     CompilerInvocation OriginalCI, bool OptimizeArgs,
                      bool EagerLoadModules);
 
   void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
 
+  /// Apply any changes implied by the discovered dependencies to the given
+  /// invocation, (e.g. disable implicit modules, add explicit module paths).
+  void applyDiscoveredDependencies(CompilerInvocation &CI);
+
 private:
   friend ModuleDepCollectorPP;
 

clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp

@@ -45,6 +45,8 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
   /// Prints out all of the gathered dependencies into a string.
   class MakeDependencyPrinterConsumer : public DependencyConsumer {
   public:
+    void handleBuildCommand(Command) override {}
+
     void
     handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override {
       this->Opts = std::make_unique<DependencyOutputOptions>(Opts);
@@ -120,34 +122,74 @@ DependencyScanningTool::getFullDependencies(
       Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
   if (Result)
     return std::move(Result);
-  return Consumer.getFullDependencies(CommandLine);
+  return Consumer.takeFullDependencies();
+}
+
+llvm::Expected<FullDependenciesResult>
+DependencyScanningTool::getFullDependenciesLegacyDriverCommand(
+    const std::vector<std::string> &CommandLine, StringRef CWD,
+    const llvm::StringSet<> &AlreadySeen,
+    LookupModuleOutputCallback LookupModuleOutput,
+    llvm::Optional<StringRef> ModuleName) {
+  FullDependencyConsumer Consumer(AlreadySeen, LookupModuleOutput,
+                                  Worker.shouldEagerLoadModules());
+  llvm::Error Result =
+      Worker.computeDependencies(CWD, CommandLine, Consumer, ModuleName);
+  if (Result)
+    return std::move(Result);
+  return Consumer.getFullDependenciesLegacyDriverCommand(CommandLine);
+}
+
+FullDependenciesResult FullDependencyConsumer::takeFullDependencies() {
+  FullDependenciesResult FDR;
+  FullDependencies &FD = FDR.FullDeps;
+
+  FD.ID.ContextHash = std::move(ContextHash);
+  FD.FileDeps = std::move(Dependencies);
+  FD.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
+  FD.Commands = std::move(Commands);
+
+  for (auto &&M : ClangModuleDeps) {
+    auto &MD = M.second;
+    if (MD.ImportedByMainFile)
+      FD.ClangModuleDeps.push_back(MD.ID);
+    // TODO: Avoid handleModuleDependency even being called for modules
+    //   we've already seen.
+    if (AlreadySeen.count(M.first))
+      continue;
+    FDR.DiscoveredModules.push_back(std::move(MD));
+  }
+
+  return FDR;
 }
 
-FullDependenciesResult FullDependencyConsumer::getFullDependencies(
+FullDependenciesResult
+FullDependencyConsumer::getFullDependenciesLegacyDriverCommand(
     const std::vector<std::string> &OriginalCommandLine) const {
   FullDependencies FD;
 
-  FD.CommandLine = makeTUCommandLineWithoutPaths(
+  FD.DriverCommandLine = makeTUCommandLineWithoutPaths(
       ArrayRef<std::string>(OriginalCommandLine).slice(1));
 
   FD.ID.ContextHash = std::move(ContextHash);
 
   FD.FileDeps.assign(Dependencies.begin(), Dependencies.end());
 
   for (const PrebuiltModuleDep &PMD : PrebuiltModuleDeps)
-    FD.CommandLine.push_back("-fmodule-file=" + PMD.PCMFile);
+    FD.DriverCommandLine.push_back("-fmodule-file=" + PMD.PCMFile);
 
   for (auto &&M : ClangModuleDeps) {
     auto &MD = M.second;
     if (MD.ImportedByMainFile) {
       FD.ClangModuleDeps.push_back(MD.ID);
       auto PCMPath = LookupModuleOutput(MD.ID, ModuleOutputKind::ModuleFile);
       if (EagerLoadModules) {
-        FD.CommandLine.push_back("-fmodule-file=" + PCMPath);
+        FD.DriverCommandLine.push_back("-fmodule-file=" + PCMPath);
       } else {
-        FD.CommandLine.push_back("-fmodule-map-file=" + MD.ClangModuleMapFile);
-        FD.CommandLine.push_back("-fmodule-file=" + MD.ID.ModuleName + "=" +
-                                 PCMPath);
+        FD.DriverCommandLine.push_back("-fmodule-map-file=" +
+                                       MD.ClangModuleMapFile);
+        FD.DriverCommandLine.push_back("-fmodule-file=" + MD.ID.ModuleName +
+                                       "=" + PCMPath);
       }
     }
   }