feat(codspeed): run the warmup with the simulated cache kept warm

crates/bencher_compat/README.md

@@ -72,8 +72,8 @@ $ cargo codspeed run
 Using codspeed-bencher-compat v1.0.0 compatibility layer
 NOTICE: codspeed is enabled, but no performance measurement will
 be made since it's running in an unknown environment.
-Checked: benches/example.rs::a (group: benches)
-Checked: benches/example.rs::b (group: benches)
+Checked: benches/example.rs::a (group: benches) (warmup: 5 iterations)
+Checked: benches/example.rs::b (group: benches) (warmup: 5 iterations)
         Done running bencher_example
     Finished running 1 benchmark suite(s)
 ```

crates/bencher_compat/src/compat.rs

@@ -1,5 +1,5 @@
 use codspeed::{
-    codspeed::{black_box, CodSpeed, WARMUP_RUNS},
+    codspeed::{black_box, CodSpeed},
     utils::{get_formated_function_path, get_git_relative_path},
 };
 
@@ -34,13 +34,20 @@ impl Bencher {
         let file = get_git_relative_path(self.current_file.as_str());
         let bench_path = get_formated_function_path(self.current_bench_path.as_str());
         let uri = format!("{}::{}", file.to_string_lossy(), bench_path);
-        for _ in 0..WARMUP_RUNS {
-            black_box(inner());
+        loop {
+            let warming_up = self.codspeed.warmup_iteration();
+            if !warming_up {
+                self.codspeed.start_benchmark(uri.as_str());
+            }
+            let output = inner();
+            if !warming_up {
+                self.codspeed.end_benchmark();
+            }
+            black_box(output);
+            if !warming_up {
+                break;
+            }
         }
-        self.codspeed.start_benchmark(uri.as_str());
-        let output = inner();
-        self.codspeed.end_benchmark();
-        black_box(output);
     }
 }
 

crates/codspeed/src/codspeed.rs

@@ -1,10 +1,48 @@
 use crate::measurement;
 use colored::Colorize;
 use std::ffi::CString;
+use std::time::{Duration, Instant};
 
 pub use std::hint::black_box;
 
-pub const WARMUP_RUNS: u32 = 5;
+/// Every benchmark runs at least this many unmeasured iterations before the measured one.
+pub const WARMUP_MIN_ITERATIONS: u32 = 1;
+/// Upper bound on unmeasured iterations, even when [`WARMUP_MAX_DURATION`] is not reached.
+pub const WARMUP_MAX_ITERATIONS: u32 = 5;
+/// Once the warmup has run for this long, no further unmeasured iteration is started.
+/// The clock runs under the simulator, so this is simulator time, not native time.
+pub const WARMUP_MAX_DURATION: Duration = Duration::from_secs(1);
+
+/// Progress of the unmeasured iterations that precede a measured iteration.
+///
+/// Instrumentation (and therefore cache simulation) is already running while these
+/// iterations execute, so the measured iteration starts with the caches populated by
+/// the benchmark itself instead of with the flushed state a fresh
+/// `CALLGRIND_START_INSTRUMENTATION` would leave behind.
+struct Warmup {
+    started: Instant,
+    iterations: u32,
+}
+
+impl Warmup {
+    fn begin() -> Self {
+        measurement::start_instrumentation();
+        Self {
+            started: Instant::now(),
+            iterations: 0,
+        }
+    }
+
+    fn should_iterate(&self) -> bool {
+        if self.iterations < WARMUP_MIN_ITERATIONS {
+            return true;
+        }
+        if self.iterations >= WARMUP_MAX_ITERATIONS {
+            return false;
+        }
+        self.started.elapsed() < WARMUP_MAX_DURATION
+    }
+}
 
 pub fn display_native_harness() {
     eprintln!("Harness: codspeed v{}", env!("CARGO_PKG_VERSION"),);
@@ -30,6 +68,8 @@ pub struct CodSpeed {
     current_benchmark: CString,
     group_stack: Vec<String>,
     instrumentation_status: InstrumentationStatus,
+    warmup: Option<Warmup>,
+    warmup_iterations: u32,
 }
 
 impl CodSpeed {
@@ -67,6 +107,8 @@ impl CodSpeed {
             current_benchmark: CString::new("").expect("CString::new failed"),
             group_stack: Vec::new(),
             instrumentation_status,
+            warmup: None,
+            warmup_iterations: 0,
         }
     }
 
@@ -78,6 +120,31 @@ impl CodSpeed {
         self.group_stack.pop();
     }
 
+    /// Returns whether the harness should run one more unmeasured iteration before the
+    /// measured one. The first call switches instrumentation on; the following
+    /// [`CodSpeed::start_benchmark`] then only zeroes the counters, keeping the simulated
+    /// cache state the warmup iterations built up.
+    ///
+    /// Call it before every candidate iteration:
+    ///
+    /// ```ignore
+    /// while codspeed.warmup_iteration() {
+    ///     black_box(routine());
+    /// }
+    /// codspeed.start_benchmark(uri);
+    /// black_box(routine());
+    /// codspeed.end_benchmark();
+    /// ```
+    #[inline(always)]
+    pub fn warmup_iteration(&mut self) -> bool {
+        let warmup = self.warmup.get_or_insert_with(Warmup::begin);
+        if !warmup.should_iterate() {
+            return false;
+        }
+        warmup.iterations += 1;
+        true
+    }
+
     #[inline(always)]
     pub fn start_benchmark(&mut self, name: &str) {
         self.current_benchmark = CString::new(name).expect("CString::new failed");
@@ -90,7 +157,17 @@ impl CodSpeed {
         // We intentionally do this no matter the instrumentation status.
         // The overhead in case we are not running valgrind is extremely low, and we do not want to
         // add a conditionnal branch to valgrind measurements.
-        measurement::start();
+        //
+        // Everything executed before `zero_statistics` is discarded from the measurement, so the
+        // branch below is invisible to the counters.
+        match self.warmup.take() {
+            Some(warmup) => self.warmup_iterations = warmup.iterations,
+            None => {
+                self.warmup_iterations = 0;
+                measurement::start_instrumentation();
+            }
+        }
+        measurement::zero_statistics();
     }
 
     #[inline(always)]
@@ -120,10 +197,11 @@ impl CodSpeed {
             format!(" (group: {})", self.group_stack.join("/"))
         };
         println!(
-            "{}: {}{}",
+            "{}: {}{} (warmup: {} iterations)",
             action_str,
             self.current_benchmark.to_string_lossy(),
-            group_str
+            group_str,
+            self.warmup_iterations,
         );
     }
 }
@@ -133,3 +211,32 @@ impl Default for CodSpeed {
         Self::new()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn warmup_runs_between_min_and_max_iterations_per_benchmark() {
+        let mut codspeed = CodSpeed::new();
+        for _ in 0..2 {
+            let mut iterations = 0;
+            while codspeed.warmup_iteration() {
+                iterations += 1;
+            }
+            assert!(iterations >= WARMUP_MIN_ITERATIONS);
+            assert!(iterations <= WARMUP_MAX_ITERATIONS);
+            codspeed.start_benchmark("warmup_test");
+            assert_eq!(codspeed.warmup_iterations, iterations);
+            codspeed.end_benchmark();
+        }
+    }
+
+    #[test]
+    fn measuring_without_warmup_reports_zero_warmup_iterations() {
+        let mut codspeed = CodSpeed::new();
+        codspeed.start_benchmark("no_warmup");
+        assert_eq!(codspeed.warmup_iterations, 0);
+        codspeed.end_benchmark();
+    }
+}

crates/codspeed/src/measurement.rs

@@ -35,17 +35,27 @@ pub fn set_metadata() {
     }
 }
 
+/// Switches Callgrind instrumentation on. With `--cache-sim=yes` this flushes the
+/// simulated cache, so it is issued once before the warmup and never between the
+/// warmup and the measured iteration.
 #[inline(always)]
-pub fn start() {
+pub fn start_instrumentation() {
     unsafe {
-        send_client_request(0, &[ClientRequest::ZeroStatistics as Value, 0, 0, 0, 0, 0]);
         send_client_request(
             0,
             &[ClientRequest::StartInstrumentation as Value, 0, 0, 0, 0, 0],
         );
     }
 }
 
+/// Zeroes the cost centers without touching the simulated cache state.
+#[inline(always)]
+pub fn zero_statistics() {
+    unsafe {
+        send_client_request(0, &[ClientRequest::ZeroStatistics as Value, 0, 0, 0, 0, 0]);
+    }
+}
+
 #[inline(always)]
 pub fn stop(name: &CString) {
     unsafe {

crates/criterion_compat/README.md

@@ -68,7 +68,7 @@ $ cargo codspeed run
      Running my_benchmark
 Using codspeed-criterion-compat v1.0.0 compatibility layer
 NOTICE: codspeed is enabled, but no performance measurement will be made since it's running in an unknown environment.
-Checked: benches/bencher_example.rs::fib_20 (group: benches)
+Checked: benches/bencher_example.rs::fib_20 (group: benches) (warmup: 5 iterations)
         Done running bencher_example
     Finished running 1 benchmark suite(s)
 ```