[llvm-exegesis] Add option to specify the number of measurement repetitions (#74276)

Currently, the llvm-exegesis LatencyBenchmarkRunner repeats the
benchmark several times (currently 30) and then aggregates the result to
deal with noise in the measurement process. With this patch, the number
of repetitions to perform is made configurable rather than left as a
static number. This allows for significantly faster execution in
situations where someone is performing a task like experimenting with
memory annotations where the exact cycle counts might not be useful, and
also allows for increased precision when desired.

Author: boomanaiden154

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp

@@ -21,11 +21,13 @@ namespace exegesis {
 LatencyBenchmarkRunner::LatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode)
+    Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode,
+    unsigned BenchmarkRepeatCount)
     : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) {
   assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) &&
          "invalid mode");
   ResultAggMode = ResultAgg;
+  NumMeasurements = BenchmarkRepeatCount;
 }
 
 LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default;
@@ -68,7 +70,6 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
   // Cycle measurements include some overhead from the kernel. Repeat the
   // measure several times and return the aggregated value, as specified by
   // ResultAggMode.
-  constexpr const int NumMeasurements = 30;
   llvm::SmallVector<int64_t, 4> AccumulatedValues;
   double MinVariance = std::numeric_limits<double>::infinity();
   const char *CounterName = State.getPfmCounters().CycleCounter;

llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h

@@ -24,14 +24,16 @@ class LatencyBenchmarkRunner : public BenchmarkRunner {
   LatencyBenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
                          BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
                          Benchmark::ResultAggregationModeE ResultAggMode,
-                         ExecutionModeE ExecutionMode);
+                         ExecutionModeE ExecutionMode,
+                         unsigned BenchmarkRepeatCount);
   ~LatencyBenchmarkRunner() override;
 
 private:
   Expected<std::vector<BenchmarkMeasure>>
   runMeasurements(const FunctionExecutor &Executor) const override;
 
   Benchmark::ResultAggregationModeE ResultAggMode;
+  unsigned NumMeasurements;
 };
 } // namespace exegesis
 } // namespace llvm

llvm/tools/llvm-exegesis/lib/Target.cpp

@@ -79,6 +79,7 @@ ExegesisTarget::createBenchmarkRunner(
     Benchmark::ModeE Mode, const LLVMState &State,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
     BenchmarkRunner::ExecutionModeE ExecutionMode,
+    unsigned BenchmarkRepeatCount,
     Benchmark::ResultAggregationModeE ResultAggMode) const {
   PfmCountersInfo PfmCounters = State.getPfmCounters();
   switch (Mode) {
@@ -101,7 +102,8 @@ ExegesisTarget::createBenchmarkRunner(
                   "the kernel for real event counts."));
     }
     return createLatencyBenchmarkRunner(State, Mode, BenchmarkPhaseSelector,
-                                        ResultAggMode, ExecutionMode);
+                                        ResultAggMode, ExecutionMode,
+                                        BenchmarkRepeatCount);
   case Benchmark::Uops:
     if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
         !PfmCounters.UopsCounter && !PfmCounters.IssueCounters)
@@ -130,9 +132,11 @@ std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
     Benchmark::ResultAggregationModeE ResultAggMode,
-    BenchmarkRunner::ExecutionModeE ExecutionMode) const {
+    BenchmarkRunner::ExecutionModeE ExecutionMode,
+    unsigned BenchmarkRepeatCount) const {
   return std::make_unique<LatencyBenchmarkRunner>(
-      State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode);
+      State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode,
+      BenchmarkRepeatCount);
 }
 
 std::unique_ptr<BenchmarkRunner> ExegesisTarget::createUopsBenchmarkRunner(

llvm/tools/llvm-exegesis/lib/Target.h

@@ -262,6 +262,7 @@ class ExegesisTarget {
       Benchmark::ModeE Mode, const LLVMState &State,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
       BenchmarkRunner::ExecutionModeE ExecutionMode,
+      unsigned BenchmarkRepeatCount,
       Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const;
 
   // Returns the ExegesisTarget for the given triple or nullptr if the target
@@ -305,7 +306,8 @@ class ExegesisTarget {
       const LLVMState &State, Benchmark::ModeE Mode,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
       Benchmark::ResultAggregationModeE ResultAggMode,
-      BenchmarkRunner::ExecutionModeE ExecutionMode) const;
+      BenchmarkRunner::ExecutionModeE ExecutionMode,
+      unsigned BenchmarkRepeatCount) const;
   std::unique_ptr<BenchmarkRunner> virtual createUopsBenchmarkRunner(
       const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
       Benchmark::ResultAggregationModeE ResultAggMode,

llvm/tools/llvm-exegesis/llvm-exegesis.cpp

@@ -262,6 +262,12 @@ static cl::opt<BenchmarkRunner::ExecutionModeE> ExecutionMode(
                           "allows for the use of memory annotations")),
     cl::init(BenchmarkRunner::ExecutionModeE::InProcess));
 
+static cl::opt<unsigned> BenchmarkRepeatCount(
+    "benchmark-repeat-count",
+    cl::desc("The number of times to repeat measurements on the benchmark k "
+             "before aggregating the results"),
+    cl::cat(BenchmarkOptions), cl::init(30));
+
 static ExitOnError ExitOnErr("llvm-exegesis error: ");
 
 // Helper function that logs the error(s) and exits.
@@ -485,7 +491,7 @@ void benchmarkMain() {
   const std::unique_ptr<BenchmarkRunner> Runner =
       ExitOnErr(State.getExegesisTarget().createBenchmarkRunner(
           BenchmarkMode, State, BenchmarkPhaseSelector, ExecutionMode,
-          ResultAggMode));
+          BenchmarkRepeatCount, ResultAggMode));
   if (!Runner) {
     ExitWithError("cannot create benchmark runner");
   }