Make perf. counter stats optional in BenchmarkStats

This enables to run runtime benchmarks even in environments that do not support perf. event profiling fully (like CI).

Author: Kobzol

collector/benchlib/src/comm/messages.rs

@@ -16,12 +16,14 @@ pub struct BenchmarkResult {
 }
 
 /// The stats gathered by a single benchmark execution.
+/// Some of the perf. counters may be missing if the machine that executes the benchmark is unable
+/// to gather them.
 #[derive(Debug, serde::Serialize, serde::Deserialize)]
 pub struct BenchmarkStats {
-    pub cycles: u64,
-    pub instructions: u64,
-    pub branch_misses: u64,
-    pub cache_misses: u64,
-    pub cache_references: u64,
+    pub cycles: Option<u64>,
+    pub instructions: Option<u64>,
+    pub branch_misses: Option<u64>,
+    pub cache_misses: Option<u64>,
+    pub cache_references: Option<u64>,
     pub wall_time: Duration,
 }

collector/benchlib/src/measure/perf_counter/unix.rs

@@ -4,12 +4,14 @@ use perf_event::events::Hardware;
 use perf_event::{Builder, Counter, Group};
 use std::time::Instant;
 
+/// A collection of CPU performance counters.
+/// The counters are optional, because some CPUs are not able to record them.
 struct Counters {
-    cycles: Counter,
-    instructions: Counter,
-    branch_misses: Counter,
-    cache_misses: Counter,
-    cache_references: Counter,
+    cycles: Option<Counter>,
+    instructions: Option<Counter>,
+    branch_misses: Option<Counter>,
+    cache_misses: Option<Counter>,
+    cache_references: Option<Counter>,
 }
 
 /// Benchmarks a single function generated by `benchmark_constructor`.
@@ -48,11 +50,11 @@ pub fn benchmark_function<F: Fn() -> Bench + 'static, R, Bench: FnOnce() -> R +
     black_box(output);
 
     let result = BenchmarkStats {
-        cycles: measurement[&counters.cycles],
-        instructions: measurement[&counters.instructions],
-        branch_misses: measurement[&counters.branch_misses],
-        cache_misses: measurement[&counters.cache_misses],
-        cache_references: measurement[&counters.cache_references],
+        cycles: counters.cycles.map(|c| measurement[&c]),
+        instructions: counters.instructions.map(|c| measurement[&c]),
+        branch_misses: counters.branch_misses.map(|c| measurement[&c]),
+        cache_misses: counters.cache_misses.map(|c| measurement[&c]),
+        cache_references: counters.cache_references.map(|c| measurement[&c]),
         wall_time: duration,
     };
     Ok(result)
@@ -77,19 +79,23 @@ Try lowering it with `sudo bash -c 'echo -1 > /proc/sys/kernel/perf_event_parano
 }
 
 fn prepare_counters(group: &mut Group) -> anyhow::Result<Counters> {
-    let mut add_event = |event: Hardware| {
-        Builder::new()
-            .group(group)
-            .kind(event)
-            .build()
-            .map_err(|error| anyhow::anyhow!("Could not add counter {:?}: {:?}", event, error))
+    let mut add_event = |event: Hardware| match Builder::new().group(group).kind(event).build() {
+        Ok(counter) => Some(counter),
+        Err(error) => {
+            log::warn!(
+                "Could not add counter {:?}: {:?}. Maybe the CPU doesn't support it?",
+                event,
+                error
+            );
+            None
+        }
     };
 
-    let cycles = add_event(Hardware::CPU_CYCLES)?;
-    let instructions = add_event(Hardware::INSTRUCTIONS)?;
-    let branch_misses = add_event(Hardware::BRANCH_MISSES)?;
-    let cache_misses = add_event(Hardware::CACHE_MISSES)?;
-    let cache_references = add_event(Hardware::CACHE_REFERENCES)?;
+    let cycles = add_event(Hardware::CPU_CYCLES);
+    let instructions = add_event(Hardware::INSTRUCTIONS);
+    let branch_misses = add_event(Hardware::BRANCH_MISSES);
+    let cache_misses = add_event(Hardware::CACHE_MISSES);
+    let cache_references = add_event(Hardware::CACHE_REFERENCES);
 
     Ok(Counters {
         cycles,

collector/src/runtime/mod.rs

@@ -125,24 +125,36 @@ fn calculate_mean<I: Iterator<Item = f64> + Clone>(iter: I) -> f64 {
 }
 
 fn print_stats(result: &BenchmarkResult) {
-    fn print_metric<F: Fn(&BenchmarkStats) -> u64>(result: &BenchmarkResult, name: &str, f: F) {
-        let mean = calculate_mean(result.stats.iter().map(&f).map(|v| v as f64));
-        let stddev = calculate_mean(
-            result
-                .stats
-                .iter()
-                .map(&f)
-                .map(|v| (v as f64 - mean).powf(2.0)),
-        )
-        .sqrt();
-
+    fn print_metric<F: Fn(&BenchmarkStats) -> Option<u64>>(
+        result: &BenchmarkResult,
+        name: &str,
+        f: F,
+    ) {
         let name = format!("[{name}]");
-        println!("{name:>20}: {:>16} (+/- {:>8})", mean as u64, stddev as u64);
+        let has_data = result.stats.iter().map(&f).all(|v| v.is_some());
+        if has_data {
+            let f = |stats: &BenchmarkStats| -> u64 { f(stats).unwrap() };
+            let mean = calculate_mean(result.stats.iter().map(&f).map(|v| v as f64));
+            let stddev = calculate_mean(
+                result
+                    .stats
+                    .iter()
+                    .map(&f)
+                    .map(|v| (v as f64 - mean).powf(2.0)),
+            )
+            .sqrt();
+
+            println!("{name:>20}: {:>16} (+/- {:>8})", mean as u64, stddev as u64);
+        } else {
+            println!("{name:>20}: Not available");
+        }
     }
 
     print_metric(result, "Instructions", |m| m.instructions);
     print_metric(result, "Cycles", |m| m.cycles);
-    print_metric(result, "Wall time [us]", |m| m.wall_time.as_micros() as u64);
+    print_metric(result, "Wall time [us]", |m| {
+        Some(m.wall_time.as_micros() as u64)
+    });
     print_metric(result, "Branch misses", |m| m.branch_misses);
     print_metric(result, "Cache misses", |m| m.cache_misses);
 }