Merge pull request #14074 from graydon/batch-mode-driver-work

More batch mode driver work

Author: graydon

include/swift/Driver/Compilation.h

@@ -70,6 +70,10 @@ class Compilation {
   /// The DiagnosticEngine to which this Compilation should emit diagnostics.
   DiagnosticEngine &Diags;
 
+  /// The ToolChain this Compilation was built with, that it may reuse to build
+  /// subsequent BatchJobs.
+  const ToolChain &TheToolChain;
+
   /// The OutputLevel at which this Compilation should generate output.
   OutputLevel Level;
 
@@ -172,7 +176,8 @@ class Compilation {
       ArrayRefView<std::unique_ptr<T>, T *, Compilation::unwrap<T>>;
 
 public:
-  Compilation(DiagnosticEngine &Diags, OutputLevel Level,
+  Compilation(DiagnosticEngine &Diags, const ToolChain &TC,
+              OutputLevel Level,
               std::unique_ptr<llvm::opt::InputArgList> InputArgs,
               std::unique_ptr<llvm::opt::DerivedArgList> TranslatedArgs,
               InputFileList InputsWithTypes,

include/swift/Driver/Driver.h

@@ -65,6 +65,9 @@ class OutputInfo {
     /// A compilation using a single frontend invocation without -primary-file.
     SingleCompile,
 
+    /// A single process that batches together multiple StandardCompile jobs.
+    BatchModeCompile,
+
     /// Invoke the REPL
     REPL,
 
@@ -175,13 +178,28 @@ class Driver {
 
   void setCheckInputFilesExist(bool Value) { CheckInputFilesExist = Value; }
 
-  /// Construct a compilation object for a command line argument vector.
+  /// Creates an appropriate ToolChain for a given driver, given the target
+  /// specified in \p Args (or the default target). Sets the value of \c
+  /// DefaultTargetTriple from \p Args as a side effect.
+  ///
+  /// \return A ToolChain, or nullptr if an unsupported target was specified
+  /// (in which case a diagnostic error is also signalled).
+  ///
+  /// This uses a std::unique_ptr instead of returning a toolchain by value
+  /// because ToolChain has virtual methods.
+  std::unique_ptr<ToolChain>
+  buildToolChain(const llvm::opt::InputArgList &ArgList);
+
+  /// Construct a compilation object for a given ToolChain and command line
+  /// argument vector.
   ///
   /// \return A Compilation, or nullptr if none was built for the given argument
   /// vector. A null return value does not necessarily indicate an error
   /// condition; the diagnostics should be queried to determine if an error
   /// occurred.
-  std::unique_ptr<Compilation> buildCompilation(ArrayRef<const char *> Args);
+  std::unique_ptr<Compilation>
+  buildCompilation(const ToolChain &TC,
+                   std::unique_ptr<llvm::opt::InputArgList> ArgList);
 
   /// Parse the given list of strings into an InputArgList.
   std::unique_ptr<llvm::opt::InputArgList>
@@ -296,9 +314,9 @@ class Driver {
                                      llvm::SmallString<128> &Buf,
                                      CommandOutput *Output) const;
 
-  void chooseSaveOptimizationPath(Compilation &C, const OutputInfo &OI,
-                                  llvm::SmallString<128> &Buf,
-                                  CommandOutput *Output) const;
+  void chooseOptimizationRecordPath(Compilation &C, const OutputInfo &OI,
+                                    llvm::SmallString<128> &Buf,
+                                    CommandOutput *Output) const;
 
   void chooseObjectiveCHeaderOutputPath(Compilation &C, const OutputInfo &OI,
                                         const TypeToPathMap *OutputMap,

include/swift/Driver/Job.h

@@ -184,6 +184,31 @@ class Job {
                              const llvm::opt::ArgStringList &Args);
 };
 
+/// A BatchJob comprises a _set_ of jobs, each of which is sufficiently similar
+/// to the others that the whole set can be combined into a single subprocess
+/// (and thus run potentially more-efficiently than running each Job in the set
+/// individually).
+///
+/// Not all Jobs can be combined into a BatchJob: at present, only those Jobs
+/// that come from CompileJobActions, and which otherwise have the exact same
+/// input file list and arguments as one another, aside from their primary-file.
+/// See ToolChain::jobsAreBatchCombinable for details.
+
+class BatchJob : public Job {
+  SmallVector<const Job *, 4> CombinedJobs;
+public:
+  BatchJob(const JobAction &Source, SmallVectorImpl<const Job *> &&Inputs,
+           std::unique_ptr<CommandOutput> Output, const char *Executable,
+           llvm::opt::ArgStringList Arguments,
+           EnvironmentVector ExtraEnvironment,
+           std::vector<FilelistInfo> Infos,
+           ArrayRef<const Job *> Combined);
+
+  ArrayRef<const Job*> getCombinedJobs() const {
+    return CombinedJobs;
+  }
+};
+
 } // end namespace driver
 } // end namespace swift
 

include/swift/Option/Options.td

@@ -619,6 +619,10 @@ def whole_module_optimization : Flag<["-"], "whole-module-optimization">,
   HelpText<"Optimize input files together instead of individually">,
   Flags<[FrontendOption, NoInteractiveOption]>;
 
+def batch_mode : Flag<["-"], "batch-mode">,
+  Flags<[FrontendOption, NoInteractiveOption, HelpHidden]>,
+  HelpText<"Combine frontend jobs into batches">;
+
 def wmo : Flag<["-"], "wmo">, Alias<whole_module_optimization>,
   Flags<[FrontendOption, NoInteractiveOption, HelpHidden]>;
 

lib/Driver/Compilation.cpp

@@ -84,7 +84,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const LogJobSet &ljs) {
 }
 
 
-Compilation::Compilation(DiagnosticEngine &Diags, OutputLevel Level,
+Compilation::Compilation(DiagnosticEngine &Diags,
+                         const ToolChain &TC, OutputLevel Level,
                          std::unique_ptr<InputArgList> InputArgs,
                          std::unique_ptr<DerivedArgList> TranslatedArgs,
                          InputFileList InputsWithTypes,
@@ -95,7 +96,8 @@ Compilation::Compilation(DiagnosticEngine &Diags, OutputLevel Level,
                          bool SaveTemps,
                          bool ShowDriverTimeCompilation,
                          std::unique_ptr<UnifiedStatsReporter> StatsReporter)
-  : Diags(Diags), Level(Level), RawInputArgs(std::move(InputArgs)),
+  : Diags(Diags), TheToolChain(TC), Level(Level),
+    RawInputArgs(std::move(InputArgs)),
     TranslatedArgs(std::move(TranslatedArgs)), 
     InputFilesWithTypes(std::move(InputsWithTypes)), ArgsHash(ArgsHash),
     BuildStartTime(StartTime),

lib/Driver/Driver.cpp

@@ -197,42 +197,46 @@ static void validateArgs(DiagnosticEngine &diags, const ArgList &Args) {
   }
 }
 
-/// Creates an appropriate ToolChain for a given driver and target triple.
-///
-/// This uses a std::unique_ptr instead of returning a toolchain by value
-/// because ToolChain has virtual methods.
-static std::unique_ptr<const ToolChain>
-makeToolChain(Driver &driver, const llvm::Triple &target) {
+std::unique_ptr<ToolChain>
+Driver::buildToolChain(const llvm::opt::InputArgList &ArgList) {
+
+  if (const Arg *A = ArgList.getLastArg(options::OPT_target))
+    DefaultTargetTriple = llvm::Triple::normalize(A->getValue());
+
+  const llvm::Triple target(DefaultTargetTriple);
+
   switch (target.getOS()) {
   case llvm::Triple::Darwin:
   case llvm::Triple::MacOSX:
   case llvm::Triple::IOS:
   case llvm::Triple::TvOS:
   case llvm::Triple::WatchOS:
-    return llvm::make_unique<toolchains::Darwin>(driver, target);
+    return llvm::make_unique<toolchains::Darwin>(*this, target);
     break;
   case llvm::Triple::Linux:
     if (target.isAndroid()) {
-      return llvm::make_unique<toolchains::Android>(driver, target);
+      return llvm::make_unique<toolchains::Android>(*this, target);
     } else {
-      return llvm::make_unique<toolchains::GenericUnix>(driver, target);
+      return llvm::make_unique<toolchains::GenericUnix>(*this, target);
     }
     break;
   case llvm::Triple::FreeBSD:
-    return llvm::make_unique<toolchains::GenericUnix>(driver, target);
+    return llvm::make_unique<toolchains::GenericUnix>(*this, target);
     break;
   case llvm::Triple::Win32:
-    return llvm::make_unique<toolchains::Cygwin>(driver, target);
+    return llvm::make_unique<toolchains::Cygwin>(*this, target);
     break;
   case llvm::Triple::Haiku:
-    return llvm::make_unique<toolchains::GenericUnix>(driver, target);
+    return llvm::make_unique<toolchains::GenericUnix>(*this, target);
     break;
   default:
-    return nullptr;
+    Diags.diagnose(SourceLoc(), diag::error_unknown_target,
+                   ArgList.getLastArg(options::OPT_target)->getValue());
+    break;
   }
+  return nullptr;
 }
 
-
 static void computeArgsHash(SmallString<32> &out, const DerivedArgList &args) {
   SmallVector<const Arg *, 32> interestingArgs;
   interestingArgs.reserve(args.size());
@@ -501,16 +505,13 @@ static void validateEmbedBitcode(DerivedArgList &Args, OutputInfo &OI,
   }
 }
 
-std::unique_ptr<Compilation> Driver::buildCompilation(
-    ArrayRef<const char *> Args) {
+std::unique_ptr<Compilation>
+Driver::buildCompilation(const ToolChain &TC,
+                         std::unique_ptr<llvm::opt::InputArgList> ArgList) {
   llvm::PrettyStackTraceString CrashInfo("Compilation construction");
 
   llvm::sys::TimePoint<> StartTime = std::chrono::system_clock::now();
 
-  std::unique_ptr<InputArgList> ArgList(parseArgStrings(Args.slice(1)));
-  if (Diags.hadAnyError())
-    return nullptr;
-
   // Claim --driver-mode here, since it's already been handled.
   (void) ArgList->hasArg(options::OPT_driver_mode);
 
@@ -552,36 +553,25 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
   std::unique_ptr<DerivedArgList> TranslatedArgList(
     translateInputArgs(*ArgList));
 
-  if (const Arg *A = ArgList->getLastArg(options::OPT_target))
-    DefaultTargetTriple = llvm::Triple::normalize(A->getValue());
-
   validateArgs(Diags, *TranslatedArgList);
 
   if (Diags.hadAnyError())
     return nullptr;
-  
-  std::unique_ptr<const ToolChain> TC =
-      makeToolChain(*this, llvm::Triple(DefaultTargetTriple));
-  if (!TC) {
-    Diags.diagnose(SourceLoc(), diag::error_unknown_target,
-                   ArgList->getLastArg(options::OPT_target)->getValue());
-    return nullptr;
-  }
 
-  if (!handleImmediateArgs(*TranslatedArgList, *TC)) {
+  if (!handleImmediateArgs(*TranslatedArgList, TC)) {
     return nullptr;
   }
 
   // Construct the list of inputs.
   InputFileList Inputs;
-  buildInputs(*TC, *TranslatedArgList, Inputs);
+  buildInputs(TC, *TranslatedArgList, Inputs);
 
   if (Diags.hadAnyError())
     return nullptr;
 
   // Determine the OutputInfo for the driver.
   OutputInfo OI;
-  buildOutputInfo(*TC, *TranslatedArgList, Inputs, OI);
+  buildOutputInfo(TC, *TranslatedArgList, Inputs, OI);
 
   if (Diags.hadAnyError())
     return nullptr;
@@ -686,7 +676,7 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
       llvm_unreachable("Unknown OutputLevel argument!");
   }
 
-  std::unique_ptr<Compilation> C(new Compilation(Diags, Level,
+  std::unique_ptr<Compilation> C(new Compilation(Diags, TC, Level,
                                                  std::move(ArgList),
                                                  std::move(TranslatedArgList),
                                                  std::move(Inputs),
@@ -699,7 +689,7 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
                                                  std::move(StatsReporter)));
   // Construct the graph of Actions.
   SmallVector<const Action *, 8> TopLevelActions;
-  buildActions(TopLevelActions, *TC, OI, OFM.get(),
+  buildActions(TopLevelActions, TC, OI, OFM.get(),
                rebuildEverything ? nullptr : &outOfDateMap, *C);
 
   if (Diags.hadAnyError())
@@ -710,7 +700,7 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
     return nullptr;
   }
 
-  buildJobs(TopLevelActions, OI, OFM.get(), *TC, *C);
+  buildJobs(TopLevelActions, OI, OFM.get(), TC, *C);
 
   // For getting bulk fixits, or for when users explicitly request to continue
   // building despite errors.
@@ -1524,6 +1514,9 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
     AllLinkerInputs.push_back(CA);
     break;
   }
+  case OutputInfo::Mode::BatchModeCompile: {
+    llvm_unreachable("Batch mode should not be used to build actions");
+  }
   case OutputInfo::Mode::Immediate: {
     if (Inputs.empty())
       return;
@@ -2075,7 +2068,7 @@ Job *Driver::buildJobsForAction(Compilation &C, const JobAction *JA,
 
   if (C.getArgs().hasArg(options::OPT_save_optimization_record,
                          options::OPT_save_optimization_record_path))
-    chooseSaveOptimizationPath(C, OI, Buf, Output.get());
+    chooseOptimizationRecordPath(C, OI, Buf, Output.get());
 
   if ((isa<MergeModuleJobAction>(JA) ||
        (isa<CompileJobAction>(JA) &&
@@ -2396,9 +2389,9 @@ void Driver::chooseTBDPath(Compilation &C, const OutputInfo &OI,
   }
 }
 
-void Driver::chooseSaveOptimizationPath(Compilation &C, const OutputInfo &OI,
-                                        llvm::SmallString<128> &Buf,
-                                        CommandOutput *Output) const {
+void Driver::chooseOptimizationRecordPath(Compilation &C, const OutputInfo &OI,
+                                          llvm::SmallString<128> &Buf,
+                                          CommandOutput *Output) const {
   if (OI.CompilerMode == OutputInfo::Mode::SingleCompile) {
     auto filename = *getOutputFilenameFromPathArgOrAsTopLevel(
         OI, C.getArgs(), options::OPT_save_optimization_record_path,

lib/Driver/FrontendUtil.cpp

@@ -17,17 +17,18 @@
 #include "swift/Driver/Compilation.h"
 #include "swift/Driver/Driver.h"
 #include "swift/Driver/Job.h"
+#include "swift/Driver/ToolChain.h"
 #include "swift/Frontend/Frontend.h"
 
 using namespace swift;
 using namespace swift::driver;
 
 std::unique_ptr<CompilerInvocation>
-swift::driver::createCompilerInvocation(ArrayRef<const char *> ArgList,
+swift::driver::createCompilerInvocation(ArrayRef<const char *> Argv,
                                         DiagnosticEngine &Diags) {
   SmallVector<const char *, 16> Args;
   Args.push_back("<swiftc>"); // FIXME: Remove dummy argument.
-  Args.insert(Args.end(), ArgList.begin(), ArgList.end());
+  Args.insert(Args.end(), Argv.begin(), Argv.end());
 
   // When creating a CompilerInvocation, ensure that the driver creates a single
   // frontend command.
@@ -40,7 +41,17 @@ swift::driver::createCompilerInvocation(ArrayRef<const char *> ArgList,
   // CompilerInvocation may wish to remap inputs to source buffers.
   TheDriver.setCheckInputFilesExist(false);
 
-  std::unique_ptr<Compilation> C = TheDriver.buildCompilation(Args);
+  std::unique_ptr<llvm::opt::InputArgList> ArgList =
+    TheDriver.parseArgStrings(ArrayRef<const char *>(Args).slice(1));
+  if (Diags.hadAnyError())
+    return nullptr;
+
+  std::unique_ptr<ToolChain> TC = TheDriver.buildToolChain(*ArgList);
+  if (Diags.hadAnyError())
+    return nullptr;
+
+  std::unique_ptr<Compilation> C =
+      TheDriver.buildCompilation(*TC, std::move(ArgList));
   if (!C || C->getJobs().empty())
     return nullptr; // Don't emit an error; one should already have been emitted
 

lib/Driver/Job.cpp

@@ -147,3 +147,14 @@ void Job::printSummary(raw_ostream &os) const {
   }
   os << "}";
 }
+
+BatchJob::BatchJob(const JobAction &Source,
+                   SmallVectorImpl<const Job *> &&Inputs,
+                   std::unique_ptr<CommandOutput> Output,
+                   const char *Executable, llvm::opt::ArgStringList Arguments,
+                   EnvironmentVector ExtraEnvironment,
+                   std::vector<FilelistInfo> Infos,
+                   ArrayRef<const Job *> Combined)
+    : Job(Source, std::move(Inputs), std::move(Output), Executable, Arguments,
+          ExtraEnvironment, Infos),
+      CombinedJobs(Combined.begin(), Combined.end()) {}