Merge pull request #16897 from CodaFi/taskmaster

Author: swift-ci

include/swift/Driver/Compilation.h

@@ -40,6 +40,9 @@ namespace opt {
 
 namespace swift {
   class DiagnosticEngine;
+  namespace sys {
+    class TaskQueue;
+  }
 
 namespace driver {
   class Driver;
@@ -72,6 +75,7 @@ enum class PreserveOnSignal : bool {
 
 class Compilation {
   friend class PerformJobsState;
+
 public:
   /// The filelist threshold value to pass to ensure file lists are never used
   static const size_t NEVER_USE_FILELIST = SIZE_MAX;
@@ -145,17 +149,6 @@ class Compilation {
   /// If unknown, this will be some time in the past.
   llvm::sys::TimePoint<> LastBuildTime = llvm::sys::TimePoint<>::min();
 
-  /// The number of commands which this compilation should attempt to run in
-  /// parallel.
-  const unsigned NumberOfParallelCommands;
-
-  /// Indicates whether this Compilation should use skip execution of
-  /// subtasks during performJobs() by using a dummy TaskQueue.
-  ///
-  /// \note For testing purposes only; similar user-facing features should be
-  /// implemented separately, as the dummy TaskQueue may provide faked output.
-  const bool SkipTaskExecution;
-
   /// Indicates whether this Compilation should continue execution of subtasks
   /// even if they returned an error status.
   bool ContinueBuildingAfterErrors = false;
@@ -232,12 +225,10 @@ class Compilation {
               StringRef ArgsHash, llvm::sys::TimePoint<> StartTime,
               llvm::sys::TimePoint<> LastBuildTime,
               size_t FilelistThreshold,
-              unsigned NumberOfParallelCommands = 1,
               bool EnableIncrementalBuild = false,
               bool EnableBatchMode = false,
               unsigned BatchSeed = 0,
               bool ForceOneBatchRepartition = false,
-              bool SkipTaskExecution = false,
               bool SaveTemps = false,
               bool ShowDriverTimeCompilation = false,
               std::unique_ptr<UnifiedStatsReporter> Stats = nullptr);
@@ -284,10 +275,6 @@ class Compilation {
     return DerivedOutputFileMap;
   }
 
-  unsigned getNumberOfParallelCommands() const {
-    return NumberOfParallelCommands;
-  }
-
   bool getIncrementalBuildEnabled() const {
     return EnableIncrementalBuild;
   }
@@ -320,6 +307,10 @@ class Compilation {
     return FilelistThreshold;
   }
 
+  UnifiedStatsReporter *getStatsReporter() const {
+    return Stats.get();
+  }
+
   /// Requests the path to a file containing all input source files. This can
   /// be shared across jobs.
   ///
@@ -330,9 +321,12 @@ class Compilation {
   const char *getAllSourcesPath() const;
 
   /// Asks the Compilation to perform the Jobs which it knows about.
+  ///
+  /// \param TQ The TaskQueue used to schedule jobs for execution.
+  ///
   /// \returns result code for the Compilation's Jobs; 0 indicates success and
   /// -2 indicates that one of the Compilation's Jobs crashed during execution
-  int performJobs();
+  int performJobs(std::unique_ptr<sys::TaskQueue> &&TQ);
 
   /// Returns whether the callee is permitted to pass -emit-loaded-module-trace
   /// to a frontend job.
@@ -355,10 +349,11 @@ class Compilation {
   ///
   /// \param[out] abnormalExit Set to true if any job exits abnormally (i.e.
   /// crashes).
+  /// \param TQ The task queue on which jobs will be scheduled.
   ///
   /// \returns exit code of the first failed Job, or 0 on success. If a Job
   /// crashes during execution, a negative value will be returned.
-  int performJobsImpl(bool &abnormalExit);
+  int performJobsImpl(bool &abnormalExit, std::unique_ptr<sys::TaskQueue> &&TQ);
 
   /// \brief Performs a single Job by executing in place, if possible.
   ///

include/swift/Driver/Driver.h

@@ -43,6 +43,9 @@ namespace opt {
 }
 
 namespace swift {
+  namespace sys {
+    class TaskQueue;
+  }
   class DiagnosticEngine;
 namespace driver {
   class Action;
@@ -213,6 +216,13 @@ class Driver {
   std::unique_ptr<ToolChain>
   buildToolChain(const llvm::opt::InputArgList &ArgList);
 
+  /// Compute the task queue for this compilation and command line argument
+  /// vector.
+  ///
+  /// \return A TaskQueue, or nullptr if an invalid number of parallel jobs is
+  /// specified.  This condition is signalled by a diagnostic.
+  std::unique_ptr<sys::TaskQueue> buildTaskQueue(const Compilation &C);
+
   /// Construct a compilation object for a given ToolChain and command line
   /// argument vector.
   ///

lib/Driver/Compilation.cpp

@@ -108,12 +108,10 @@ Compilation::Compilation(DiagnosticEngine &Diags,
                          llvm::sys::TimePoint<> StartTime,
                          llvm::sys::TimePoint<> LastBuildTime,
                          size_t FilelistThreshold,
-                         unsigned NumberOfParallelCommands,
                          bool EnableIncrementalBuild,
                          bool EnableBatchMode,
                          unsigned BatchSeed,
                          bool ForceOneBatchRepartition,
-                         bool SkipTaskExecution,
                          bool SaveTemps,
                          bool ShowDriverTimeCompilation,
                          std::unique_ptr<UnifiedStatsReporter> StatsReporter)
@@ -127,8 +125,6 @@ Compilation::Compilation(DiagnosticEngine &Diags,
     ArgsHash(ArgsHash),
     BuildStartTime(StartTime),
     LastBuildTime(LastBuildTime),
-    NumberOfParallelCommands(NumberOfParallelCommands),
-    SkipTaskExecution(SkipTaskExecution),
     EnableIncrementalBuild(EnableIncrementalBuild),
     OutputCompilationRecordForModuleOnlyBuild(
         OutputCompilationRecordForModuleOnlyBuild),
@@ -619,14 +615,9 @@ namespace driver {
     }
 
   public:
-    PerformJobsState(Compilation &Comp)
-      : Comp(Comp),
-        ActualIncrementalTracer(Comp.Stats.get()) {
-      if (Comp.SkipTaskExecution)
-        TQ.reset(new DummyTaskQueue(Comp.NumberOfParallelCommands));
-      else
-        TQ.reset(new TaskQueue(Comp.NumberOfParallelCommands,
-                               Comp.Stats.get()));
+    PerformJobsState(Compilation &Comp, std::unique_ptr<TaskQueue> &&TaskQueue)
+      : Comp(Comp), ActualIncrementalTracer(Comp.Stats.get()),
+        TQ(std::move(TaskQueue)) {
       if (Comp.ShowIncrementalBuildDecisions || Comp.Stats)
         IncrementalTracer = &ActualIncrementalTracer;
     }
@@ -889,7 +880,7 @@ namespace driver {
         return;
       }
 
-      size_t NumPartitions = Comp.NumberOfParallelCommands;
+      size_t NumPartitions = TQ->getNumberOfParallelTasks();
       CommandSetVector Batchable, NonBatchable;
       std::vector<const Job *> Batches;
       bool PretendTheCommandLineIsTooLongOnce =
@@ -1220,16 +1211,17 @@ static bool writeFilelistIfNecessary(const Job *job, const ArgList &args,
   return ok;
 }
 
-int Compilation::performJobsImpl(bool &abnormalExit) {
-  PerformJobsState State(*this);
+int Compilation::performJobsImpl(bool &abnormalExit,
+                                 std::unique_ptr<TaskQueue> &&TQ) {
+  PerformJobsState State(*this, std::move(TQ));
 
   State.scheduleInitialJobs();
   State.scheduleAdditionalJobs();
   State.formBatchJobsAndAddPendingJobsToTaskQueue();
   State.runTaskQueueToCompletion();
   State.checkUnfinishedJobs();
 
-  if (!CompilationRecordPath.empty() && !SkipTaskExecution) {
+  if (!CompilationRecordPath.empty()) {
     InputInfoMap InputInfo;
     State.populateInputInfoMap(InputInfo);
     checkForOutOfDateInputs(Diags, InputInfo);
@@ -1320,7 +1312,7 @@ static bool writeAllSourcesFile(DiagnosticEngine &diags, StringRef path,
   return true;
 }
 
-int Compilation::performJobs() {
+int Compilation::performJobs(std::unique_ptr<TaskQueue> &&TQ) {
   if (AllSourceFilesPath)
     if (!writeAllSourcesFile(Diags, AllSourceFilesPath, getInputFiles()))
       return EXIT_FAILURE;
@@ -1334,12 +1326,12 @@ int Compilation::performJobs() {
     return performSingleCommand(Jobs.front().get());
   }
 
-  if (!TaskQueue::supportsParallelExecution() && NumberOfParallelCommands > 1) {
+  if (!TaskQueue::supportsParallelExecution() && TQ->getNumberOfParallelTasks() > 1) {
     Diags.diagnose(SourceLoc(), diag::warning_parallel_execution_not_supported);
   }
 
   bool abnormalExit;
-  int result = performJobsImpl(abnormalExit);
+  int result = performJobsImpl(abnormalExit, std::move(TQ));
 
   if (!SaveTemps) {
     for (const auto &pathPair : TempFilePaths) {

lib/Driver/Driver.cpp

@@ -285,6 +285,28 @@ Driver::buildToolChain(const llvm::opt::InputArgList &ArgList) {
   return nullptr;
 }
 
+std::unique_ptr<sys::TaskQueue> Driver::buildTaskQueue(const Compilation &C) {
+  const auto &ArgList = C.getArgs();
+  unsigned NumberOfParallelCommands = 1;
+  if (const Arg *A = ArgList.getLastArg(options::OPT_j)) {
+    if (StringRef(A->getValue()).getAsInteger(10, NumberOfParallelCommands)) {
+      Diags.diagnose(SourceLoc(), diag::error_invalid_arg_value,
+                     A->getAsString(ArgList), A->getValue());
+      return nullptr;
+    }
+  }
+
+  const bool DriverSkipExecution =
+    ArgList.hasArg(options::OPT_driver_skip_execution,
+                   options::OPT_driver_print_jobs);
+  if (DriverSkipExecution) {
+    return llvm::make_unique<sys::DummyTaskQueue>(NumberOfParallelCommands);
+  } else {
+    return llvm::make_unique<sys::TaskQueue>(NumberOfParallelCommands,
+                                             C.getStatsReporter());
+  }
+}
+
 static void computeArgsHash(SmallString<32> &out, const DerivedArgList &args) {
   SmallVector<const Arg *, 32> interestingArgs;
   interestingArgs.reserve(args.size());
@@ -637,9 +659,6 @@ Driver::buildCompilation(const ToolChain &TC,
   bool DriverPrintDerivedOutputFileMap =
     ArgList->hasArg(options::OPT_driver_print_derived_output_file_map);
   DriverPrintBindings = ArgList->hasArg(options::OPT_driver_print_bindings);
-  bool DriverSkipExecution =
-    ArgList->hasArg(options::OPT_driver_skip_execution,
-                    options::OPT_driver_print_jobs);
   bool ShowIncrementalBuildDecisions =
     ArgList->hasArg(options::OPT_driver_show_incremental);
   bool ShowJobLifecycle =
@@ -791,15 +810,6 @@ Driver::buildCompilation(const ToolChain &TC,
     }
   }
 
-  unsigned NumberOfParallelCommands = 1;
-  if (const Arg *A = ArgList->getLastArg(options::OPT_j)) {
-    if (StringRef(A->getValue()).getAsInteger(10, NumberOfParallelCommands)) {
-      Diags.diagnose(SourceLoc(), diag::error_invalid_arg_value,
-                     A->getAsString(*ArgList), A->getValue());
-      return nullptr;
-    }
-  }
-
   size_t DriverFilelistThreshold;
   if (getFilelistThreshold(*TranslatedArgList, DriverFilelistThreshold, Diags))
     return nullptr;
@@ -829,12 +839,10 @@ Driver::buildCompilation(const ToolChain &TC,
                       StartTime,
                       LastBuildTime,
                       DriverFilelistThreshold,
-                      NumberOfParallelCommands,
                       Incremental,
                       BatchMode,
                       DriverBatchSeed,
                       DriverForceOneBatchRepartition,
-                      DriverSkipExecution,
                       SaveTemps,
                       ShowDriverTimeCompilation,
                       std::move(StatsReporter)));

tools/driver/driver.cpp

@@ -17,6 +17,7 @@
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/Basic/LLVMInitialize.h"
 #include "swift/Basic/Program.h"
+#include "swift/Basic/TaskQueue.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/Driver/Compilation.h"
 #include "swift/Driver/Driver.h"
@@ -219,11 +220,13 @@ int main(int argc_, const char **argv_) {
 
   std::unique_ptr<Compilation> C =
       TheDriver.buildCompilation(*TC, std::move(ArgList));
+
   if (Diags.hadAnyError())
     return 1;
 
   if (C) {
-    return C->performJobs();
+    std::unique_ptr<sys::TaskQueue> TQ = TheDriver.buildTaskQueue(*C);
+    return C->performJobs(std::move(TQ));
   }
 
   return 0;