feat(divan): run multiple rounds of analysis mode

Author: not-matthias

crates/codspeed/src/instrument_hooks/mod.rs

@@ -3,8 +3,8 @@ mod ffi;
 
 #[cfg(use_instrument_hooks)]
 mod linux_impl {
-
     use super::ffi;
+    use crate::measurement;
     use std::ffi::CString;
     use std::sync::OnceLock;
 
@@ -44,6 +44,12 @@ mod linux_impl {
             unsafe { ffi::instrument_hooks_is_instrumented(self.0) }
         }
 
+        // FIXME: Should we move this to instrument-hooks native library?
+        #[inline(always)]
+        pub fn toggle_collect() {
+            measurement::toggle_collect();
+        }
+
         #[inline(always)]
         pub fn start_benchmark(&self) -> Result<(), u8> {
             let result = unsafe { ffi::instrument_hooks_start_benchmark(self.0) };
@@ -165,6 +171,9 @@ mod other_impl {
             false
         }
 
+        #[inline(always)]
+        pub fn toggle_collect() {}
+
         pub fn start_benchmark(&self) -> Result<(), u8> {
             Ok(())
         }

crates/codspeed/src/measurement.rs

@@ -35,6 +35,11 @@ pub fn set_metadata() {
     }
 }
 
+#[inline(always)]
+pub fn toggle_collect() {
+    unsafe { send_client_request(0, &[ClientRequest::ToggleCollect as Value, 0, 0, 0, 0, 0]) };
+}
+
 #[inline(always)]
 pub fn start() {
     unsafe {

crates/codspeed/src/request/mod.rs

@@ -5,6 +5,7 @@ const CG_BASE: u32 = ((b'C' as u32) << 24) + ((b'T' as u32) << 16);
 pub enum ClientRequest {
     RunningOnValgrind = 0x1001,
     ZeroStatistics = CG_BASE + 1,
+    ToggleCollect = CG_BASE + 2,
     DumpStatisticsAt = CG_BASE + 3,
     StartInstrumentation = CG_BASE + 4,
     StopInstrumentation = CG_BASE + 5,

crates/divan_compat/src/compat/bench/mod.rs

@@ -10,7 +10,8 @@ pub use self::{
     options::BenchOptions,
 };
 
-use codspeed::codspeed::CodSpeed;
+use ::codspeed::codspeed::CodSpeed;
+use ::codspeed::instrument_hooks::InstrumentHooks;
 use std::cell::RefCell;
 
 /// Using this in place of `()` for `GenI` prevents `Bencher::with_inputs` from
@@ -136,11 +137,15 @@ where
     {
         let mut codspeed = self.codspeed.borrow_mut();
         let mut gen_input = self.config.gen_input.borrow_mut();
-        let input = gen_input();
-        codspeed.start_benchmark(self.uri.as_str());
-        let output = benched(input);
-        codspeed.end_benchmark();
-        divan::black_box(output);
+
+        codspeed::run_rounds(&mut codspeed, self.uri.as_str(), || {
+            // FIXME: We could also run multiple rounds here
+            let input = gen_input();
+            InstrumentHooks::toggle_collect();
+            let output = benched(divan::black_box(input));
+            InstrumentHooks::toggle_collect();
+            divan::black_box(output);
+        });
     }
 
     pub fn bench_local_refs<O, B>(self, mut benched: B)
@@ -149,11 +154,49 @@ where
     {
         let mut codspeed = self.codspeed.borrow_mut();
         let mut gen_input = self.config.gen_input.borrow_mut();
-        let mut input = gen_input();
 
-        codspeed.start_benchmark(self.uri.as_str());
-        let output = benched(&mut input);
+        codspeed::run_rounds(&mut codspeed, self.uri.as_str(), || {
+            let mut input = gen_input();
+            InstrumentHooks::toggle_collect();
+            let output = benched(&mut input);
+            InstrumentHooks::toggle_collect();
+            divan::black_box(input);
+            divan::black_box(output);
+        });
+    }
+}
+
+mod codspeed {
+    use super::*;
+    use std::time::{Duration, Instant};
+
+    pub fn run_rounds(codspeed: &mut CodSpeed, uri: &str, mut run_iteration: impl FnMut()) {
+        // FIXME: Maybe move this to codspeed
+        let (max_rounds, max_duration) = match std::env::var("CODSPEED_RUNNER_MODE").as_deref() {
+            Ok("simulation") | Ok("instrumentation") => (None, Some(Duration::from_millis(100))),
+            Ok("memory") => (Some(1), None),
+            Ok(m) => unreachable!("Invalid runner mode: {m}"),
+            Err(err) => panic!("Failed to get runner mode: {err}"),
+        };
+        let mut rounds = 0;
+        let rounds_start_time = Instant::now();
+
+        codspeed.start_benchmark(uri);
+        InstrumentHooks::toggle_collect(); // Pause collection
+
+        loop {
+            rounds += 1;
+
+            run_iteration();
+
+            let within_rounds = max_rounds.map_or(true, |max| rounds < max);
+            let within_duration =
+                max_duration.map_or(true, |max| rounds_start_time.elapsed() < max);
+            if !(within_rounds && within_duration) {
+                break;
+            }
+        }
+
         codspeed.end_benchmark();
-        divan::black_box(output);
     }
 }