Merge pull request #14525 from graydon/batch-mode-driver-work

Batch mode driver work

Author: graydon

include/swift/Basic/TaskQueue.h

@@ -150,7 +150,7 @@ class TaskQueue {
 
   /// Returns true if there are any tasks that have been queued but have not
   /// yet been executed.
-  bool hasRemainingTasks() {
+  virtual bool hasRemainingTasks() {
     return !QueuedTasks.empty();
   }
 };
@@ -180,14 +180,20 @@ class DummyTaskQueue : public TaskQueue {
   DummyTaskQueue(unsigned NumberOfParallelTasks = 0);
   virtual ~DummyTaskQueue();
 
-  virtual void addTask(const char *ExecPath, ArrayRef<const char *> Args,
-                       ArrayRef<const char *> Env = llvm::None,
-                       void *Context = nullptr, bool SeparateErrors = false);
+  void addTask(const char *ExecPath, ArrayRef<const char *> Args,
+               ArrayRef<const char *> Env = llvm::None,
+               void *Context = nullptr, bool SeparateErrors = false) override;
 
-  virtual bool
+  bool
   execute(TaskBeganCallback Began = TaskBeganCallback(),
           TaskFinishedCallback Finished = TaskFinishedCallback(),
-          TaskSignalledCallback Signalled = TaskSignalledCallback());
+          TaskSignalledCallback Signalled = TaskSignalledCallback()) override;
+
+  bool hasRemainingTasks() override {
+    // Need to override here because QueuedTasks is redeclared.
+    return !QueuedTasks.empty();
+  }
+
 };
 
 } // end namespace sys

include/swift/Driver/Compilation.h

@@ -17,6 +17,7 @@
 #ifndef SWIFT_DRIVER_COMPILATION_H
 #define SWIFT_DRIVER_COMPILATION_H
 
+#include "swift/Driver/Driver.h"
 #include "swift/Driver/Job.h"
 #include "swift/Driver/Util.h"
 #include "swift/Basic/ArrayRefView.h"
@@ -42,6 +43,7 @@ namespace swift {
 namespace driver {
   class Driver;
   class ToolChain;
+  class OutputInfo;
   class PerformJobsState;
 
 /// An enum providing different levels of output which should be produced
@@ -72,7 +74,12 @@ class Compilation {
 
   /// The ToolChain this Compilation was built with, that it may reuse to build
   /// subsequent BatchJobs.
-  LLVM_ATTRIBUTE_UNUSED const ToolChain &TheToolChain;
+  const ToolChain &TheToolChain;
+
+  /// The OutputInfo, which the Compilation stores a copy of upon
+  /// construction, and which it may use to build subsequent batch
+  /// jobs itself.
+  OutputInfo TheOutputInfo;
 
   /// The OutputLevel at which this Compilation should generate output.
   OutputLevel Level;
@@ -149,6 +156,11 @@ class Compilation {
   /// of date.
   bool EnableIncrementalBuild;
 
+  /// Indicates whether groups of parallel frontend jobs should be merged
+  /// together and run in composite "batch jobs" when possible, to reduce
+  /// redundant work.
+  bool EnableBatchMode;
+
   /// True if temporary files should not be deleted.
   bool SaveTemps;
 
@@ -182,19 +194,29 @@ class Compilation {
 
 public:
   Compilation(DiagnosticEngine &Diags, const ToolChain &TC,
+              OutputInfo const &OI,
               OutputLevel Level,
               std::unique_ptr<llvm::opt::InputArgList> InputArgs,
               std::unique_ptr<llvm::opt::DerivedArgList> TranslatedArgs,
               InputFileList InputsWithTypes,
               StringRef ArgsHash, llvm::sys::TimePoint<> StartTime,
               unsigned NumberOfParallelCommands = 1,
               bool EnableIncrementalBuild = false,
+              bool EnableBatchMode = false,
               bool SkipTaskExecution = false,
               bool SaveTemps = false,
               bool ShowDriverTimeCompilation = false,
               std::unique_ptr<UnifiedStatsReporter> Stats = nullptr);
   ~Compilation();
 
+  ToolChain const &getToolChain() const {
+    return TheToolChain;
+  }
+
+  OutputInfo const &getOutputInfo() const {
+    return TheOutputInfo;
+  }
+
   UnwrappedArrayView<const Action> getActions() const {
     return llvm::makeArrayRef(Actions);
   }
@@ -238,7 +260,11 @@ class Compilation {
   void disableIncrementalBuild() {
     EnableIncrementalBuild = false;
   }
-  
+
+  bool getBatchModeEnabled() const {
+    return EnableBatchMode;
+  }
+
   bool getContinueBuildingAfterErrors() const {
     return ContinueBuildingAfterErrors;
   }

include/swift/Driver/Job.h

@@ -147,6 +147,15 @@ class CommandOutput {
   /// with the provided \p Input pair of Base and Primary inputs.
   void addPrimaryOutput(CommandInputPair Input, StringRef PrimaryOutputFile);
 
+  /// Return true iff the set of additional output types in \c this is
+  /// identical to the set of additional output types in \p other.
+  bool hasSameAdditionalOutputTypes(CommandOutput const &other) const;
+
+  /// Copy all the input pairs from \p other to \c this. Assumes (and asserts)
+  /// that \p other shares output file map and PrimaryOutputType with \c this
+  /// already, as well as AdditionalOutputTypes if \c this has any.
+  void addOutputs(CommandOutput const &other);
+
   /// Assuming (and asserting) that there is only one input pair, return the
   /// primary output file associated with it. Note that the returned StringRef
   /// may be invalidated by subsequent mutations to the \c CommandOutput.
@@ -236,6 +245,8 @@ class Job {
         ExtraEnvironment(std::move(ExtraEnvironment)),
         FilelistFileInfos(std::move(Infos)) {}
 
+  virtual ~Job();
+
   const JobAction &getSource() const {
     return *SourceAndCondition.getPointer();
   }

include/swift/Driver/ToolChain.h

@@ -166,6 +166,30 @@ class ToolChain {
                                     std::unique_ptr<CommandOutput> output,
                                     const OutputInfo &OI) const;
 
+  /// Return true iff the input \c Job \p A is an acceptable candidate for
+  /// batching together into a BatchJob, via a call to \c
+  /// constructBatchJob. This is true when the \c Job is a built from a \c
+  /// CompileJobAction in a \c Compilation \p C running in \c
+  /// OutputInfo::Mode::StandardCompile output mode, with a single \c TY_Swift
+  /// \c InputAction.
+  bool jobIsBatchable(const Compilation &C, const Job *A) const;
+
+  /// Equivalence relation that holds iff the two input Jobs \p A and \p B are
+  /// acceptable candidates for combining together into a \c BatchJob, via a
+  /// call to \c constructBatchJob. This is true when each job independently
+  /// satisfies \c jobIsBatchable, and the two jobs have identical executables,
+  /// output types and environments (i.e. they are identical aside from their
+  /// inputs).
+  bool jobsAreBatchCombinable(const Compilation &C, const Job *A,
+                              const Job *B) const;
+
+  /// Construct a \c BatchJob that subsumes the work of a set of Jobs. Any pair
+  /// of elements in \p Jobs are assumed to satisfy the equivalence relation \c
+  /// jobsAreBatchCombinable, i.e. they should all be "the same" job in in all
+  /// ways other than their choices of inputs.
+  std::unique_ptr<Job> constructBatchJob(ArrayRef<const Job *> Jobs,
+                                         Compilation &C) const;
+
   /// Return the default language type to use for the given extension.
   /// If the extension is empty or is otherwise not recognized, return
   /// the invalid type \c TY_INVALID.