Define targets in codspeed.yaml to execute with codspeed run

.gitignore

@@ -1,2 +1,3 @@
 /target
 .DS_Store
+.codspeed

crates/exec-harness/src/analysis.rs

@@ -1,24 +1,29 @@
 use crate::prelude::*;
 
-use crate::uri::NameAndUri;
+use crate::BenchmarkCommand;
+use crate::uri;
 use codspeed::instrument_hooks::InstrumentHooks;
 use std::process::Command;
 
-pub fn perform(name_and_uri: NameAndUri, command: Vec<String>) -> Result<()> {
+pub fn perform(commands: Vec<BenchmarkCommand>) -> Result<()> {
     let hooks = InstrumentHooks::instance();
 
-    let mut cmd = Command::new(&command[0]);
-    cmd.args(&command[1..]);
-    hooks.start_benchmark().unwrap();
-    let status = cmd.status();
-    hooks.stop_benchmark().unwrap();
-    let status = status.context("Failed to execute command")?;
+    for benchmark_cmd in commands {
+        let name_and_uri = uri::generate_name_and_uri(&benchmark_cmd.name, &benchmark_cmd.command);
 
-    if !status.success() {
-        bail!("Command exited with non-zero status: {status}");
-    }
+        let mut cmd = Command::new(&benchmark_cmd.command[0]);
+        cmd.args(&benchmark_cmd.command[1..]);
+        hooks.start_benchmark().unwrap();
+        let status = cmd.status();
+        hooks.stop_benchmark().unwrap();
+        let status = status.context("Failed to execute command")?;
+
+        if !status.success() {
+            bail!("Command exited with non-zero status: {status}");
+        }
 
-    hooks.set_executed_benchmark(&name_and_uri.uri).unwrap();
+        hooks.set_executed_benchmark(&name_and_uri.uri).unwrap();
+    }
 
     Ok(())
 }

crates/exec-harness/src/lib.rs

@@ -1,15 +1,83 @@
 use clap::ValueEnum;
+use prelude::*;
 use serde::{Deserialize, Serialize};
+use std::io::{self, BufRead};
 
 pub mod analysis;
 pub mod prelude;
-pub mod uri;
+mod uri;
 pub mod walltime;
 
-#[derive(ValueEnum, Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(ValueEnum, Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum MeasurementMode {
     Walltime,
     Memory,
     Simulation,
 }
+
+/// A single benchmark command for stdin mode input.
+///
+/// This struct defines the JSON format for passing benchmark commands to exec-harness
+/// via stdin (when invoked with `-`). The runner uses this same struct to serialize
+/// targets from codspeed.yaml.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BenchmarkCommand {
+    /// The command and arguments to execute
+    pub command: Vec<String>,
+
+    /// Optional benchmark name
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+
+    /// Walltime execution options (flattened into the JSON object)
+    #[serde(default)]
+    pub walltime_args: walltime::WalltimeExecutionArgs,
+}
+
+/// Read and parse benchmark commands from stdin as JSON
+pub fn read_commands_from_stdin() -> Result<Vec<BenchmarkCommand>> {
+    let stdin = io::stdin();
+    let mut input = String::new();
+
+    for line in stdin.lock().lines() {
+        let line = line.context("Failed to read line from stdin")?;
+        input.push_str(&line);
+        input.push('\n');
+    }
+
+    let commands: Vec<BenchmarkCommand> =
+        serde_json::from_str(&input).context("Failed to parse JSON from stdin")?;
+
+    if commands.is_empty() {
+        bail!("No commands provided in stdin input");
+    }
+
+    for cmd in &commands {
+        if cmd.command.is_empty() {
+            bail!("Empty command in stdin input");
+        }
+    }
+
+    Ok(commands)
+}
+
+/// Execute benchmark commands
+pub fn execute_benchmarks(
+    commands: Vec<BenchmarkCommand>,
+    measurement_mode: Option<MeasurementMode>,
+) -> Result<()> {
+    match measurement_mode {
+        Some(MeasurementMode::Walltime) | None => {
+            walltime::perform(commands)?;
+        }
+        Some(MeasurementMode::Memory) => {
+            analysis::perform(commands)?;
+        }
+        Some(MeasurementMode::Simulation) => {
+            bail!("Simulation measurement mode is not yet supported by exec-harness");
+        }
+    }
+
+    Ok(())
+}

crates/exec-harness/src/main.rs

@@ -1,9 +1,9 @@
 use clap::Parser;
-use exec_harness::MeasurementMode;
-use exec_harness::analysis;
 use exec_harness::prelude::*;
-use exec_harness::uri;
-use exec_harness::walltime;
+use exec_harness::walltime::WalltimeExecutionArgs;
+use exec_harness::{
+    BenchmarkCommand, MeasurementMode, execute_benchmarks, read_commands_from_stdin,
+};
 
 #[derive(Parser, Debug)]
 #[command(name = "exec-harness")]
@@ -21,9 +21,10 @@ struct Args {
     measurement_mode: Option<MeasurementMode>,
 
     #[command(flatten)]
-    execution_args: walltime::WalltimeExecutionArgs,
+    walltime_args: WalltimeExecutionArgs,
 
-    /// The command and arguments to execute
+    /// The command and arguments to execute.
+    /// Use "-" as the only argument to read a JSON payload from stdin.
     #[arg(trailing_var_arg = true, allow_hyphen_values = true)]
     command: Vec<String>,
 }
@@ -37,26 +38,20 @@ fn main() -> Result<()> {
     debug!("Starting exec-harness with pid {}", std::process::id());
 
     let args = Args::parse();
-
-    if args.command.is_empty() {
-        bail!("Error: No command provided");
-    }
-
-    let bench_name_and_uri = uri::generate_name_and_uri(&args.name, &args.command);
-
-    match args.measurement_mode {
-        Some(MeasurementMode::Walltime) | None => {
-            let execution_options: walltime::ExecutionOptions = args.execution_args.try_into()?;
-
-            walltime::perform(bench_name_and_uri, args.command, &execution_options)?;
-        }
-        Some(MeasurementMode::Memory) => {
-            analysis::perform(bench_name_and_uri, args.command)?;
-        }
-        Some(MeasurementMode::Simulation) => {
-            bail!("Simulation measurement mode is not yet supported by exec-harness");
-        }
-    }
+    let measurement_mode = args.measurement_mode;
+
+    // Determine if we're in stdin mode or CLI mode
+    let commands = match args.command.as_slice() {
+        [single] if single == "-" => read_commands_from_stdin()?,
+        [] => bail!("No command provided"),
+        _ => vec![BenchmarkCommand {
+            command: args.command,
+            name: args.name,
+            walltime_args: args.walltime_args,
+        }],
+    };
+
+    execute_benchmarks(commands, measurement_mode)?;
 
     Ok(())
 }

crates/exec-harness/src/walltime/benchmark_loop.rs

@@ -13,7 +13,7 @@ pub fn run_rounds(
     let warmup_time_ns = config.warmup_time_ns;
     let hooks = InstrumentHooks::instance();
 
-    let do_one_round = |times_per_round_ns: &mut Vec<u128>| {
+    let do_one_round = || -> Result<(u64, u64)> {
         let mut child = Command::new(&command[0])
             .args(&command[1..])
             .spawn()
@@ -24,21 +24,25 @@ pub fn run_rounds(
             .context("Failed to wait for command to finish")?;
 
         let bench_round_end_ts_ns = InstrumentHooks::current_timestamp();
-        hooks.add_benchmark_timestamps(bench_round_start_ts_ns, bench_round_end_ts_ns);
 
         if !status.success() {
             bail!("Command exited with non-zero status: {status}");
         }
 
-        times_per_round_ns.push((bench_round_end_ts_ns - bench_round_start_ts_ns) as u128);
-
-        Ok(())
+        Ok((bench_round_start_ts_ns, bench_round_end_ts_ns))
     };
 
     // Compute the number of rounds to perform (potentially undefined if no warmup and only time constraints)
+    hooks.start_benchmark().unwrap();
     let rounds_to_perform: Option<u64> = if warmup_time_ns > 0 {
-        match compute_rounds_from_warmup(config, hooks, &bench_uri, do_one_round)? {
-            WarmupResult::EarlyReturn(times) => return Ok(times),
+        match compute_rounds_from_warmup(config, do_one_round)? {
+            WarmupResult::EarlyReturn { start, end } => {
+                // Add marker for the single warmup round so the run still gets profiling data
+                hooks.add_benchmark_timestamps(start, end);
+                hooks.stop_benchmark().unwrap();
+                hooks.set_executed_benchmark(&bench_uri).unwrap();
+                return Ok(vec![(end - start) as u128]);
+            }
             WarmupResult::Rounds(rounds) => Some(rounds),
         }
     } else {
@@ -77,17 +81,25 @@ pub fn run_rounds(
         .unwrap_or_default();
     let mut current_round: u64 = 0;
 
-    hooks.start_benchmark().unwrap();
-
     debug!(
         "Starting loop with ending conditions: \
         rounds {rounds_to_perform:?}, \
         min_time_ns {min_time_ns:?}, \
         max_time_ns {max_time_ns:?}"
     );
+
     let round_start_ts_ns = InstrumentHooks::current_timestamp();
+
+    let mut round_timestamps: Vec<(u64, u64)> = if let Some(rounds) = rounds_to_perform {
+        Vec::with_capacity(rounds as usize)
+    } else {
+        Vec::new()
+    };
+
     loop {
-        do_one_round(&mut times_per_round_ns)?;
+        let current_round_timestamps = do_one_round()?;
+        // Only store timestamps for later processing in order to avoid overhead during the loop
+        round_timestamps.push(current_round_timestamps);
         current_round += 1;
 
         let elapsed_ns = InstrumentHooks::current_timestamp() - round_start_ts_ns;
@@ -122,61 +134,65 @@ pub fn run_rounds(
             break;
         }
     }
+
+    // Record timestamps
+    for (start, end) in round_timestamps {
+        hooks.add_benchmark_timestamps(start, end);
+        times_per_round_ns.push((end - start) as u128);
+    }
+
     hooks.stop_benchmark().unwrap();
     hooks.set_executed_benchmark(&bench_uri).unwrap();
 
     Ok(times_per_round_ns)
 }
 
 enum WarmupResult {
-    /// Warmup satisfied max_time constraint, return early with these times
-    EarlyReturn(Vec<u128>),
+    /// Warmup exceeded max_time constraint with a single run, return early with this single timestamp pair
+    EarlyReturn { start: u64, end: u64 },
     /// Continue with this many rounds
     Rounds(u64),
 }
 
 /// Run warmup rounds and compute the number of benchmark rounds to perform
-fn compute_rounds_from_warmup<F>(
-    config: &ExecutionOptions,
-    hooks: &InstrumentHooks,
-    bench_uri: &str,
-    do_one_round: F,
-) -> Result<WarmupResult>
+fn compute_rounds_from_warmup<F>(config: &ExecutionOptions, do_one_round: F) -> Result<WarmupResult>
 where
-    F: Fn(&mut Vec<u128>) -> Result<()>,
+    F: Fn() -> Result<(u64, u64)>,
 {
-    let mut warmup_times_ns = Vec::new();
+    let mut warmup_timestamps: Vec<(u64, u64)> = Vec::new();
     let warmup_start_ts_ns = InstrumentHooks::current_timestamp();
 
-    hooks.start_benchmark().unwrap();
     while InstrumentHooks::current_timestamp() < warmup_start_ts_ns + config.warmup_time_ns {
-        do_one_round(&mut warmup_times_ns)?;
+        let (start, end) = do_one_round()?;
+        warmup_timestamps.push((start, end));
     }
-    hooks.stop_benchmark().unwrap();
     let warmup_end_ts_ns = InstrumentHooks::current_timestamp();
 
     // Check if single warmup round already exceeded max_time
-    if let [single_warmup_round_duration_ns] = warmup_times_ns.as_slice() {
+    if let [(start, end)] = warmup_timestamps.as_slice() {
+        let single_warmup_round_duration_ns = end - start;
         match config.max {
             Some(RoundOrTime::TimeNs(time_ns)) | Some(RoundOrTime::Both { time_ns, .. }) => {
-                if time_ns <= *single_warmup_round_duration_ns as u64 {
+                if time_ns <= single_warmup_round_duration_ns {
                     info!(
-                        "Warmup duration ({}) exceeded or met max_time ({}). No more rounds will be performed.",
-                        format_ns(*single_warmup_round_duration_ns as u64),
+                        "A single warmup execution ({}) exceeded or met max_time ({}). No more rounds will be performed.",
+                        format_ns(single_warmup_round_duration_ns),
                         format_ns(time_ns)
                     );
-                    hooks.set_executed_benchmark(bench_uri).unwrap();
-                    return Ok(WarmupResult::EarlyReturn(warmup_times_ns));
+                    return Ok(WarmupResult::EarlyReturn {
+                        start: *start,
+                        end: *end,
+                    });
                 }
             }
             _ => { /* No max time constraint */ }
         }
     }
 
-    info!("Completed {} warmup rounds", warmup_times_ns.len());
+    info!("Completed {} warmup rounds", warmup_timestamps.len());
 
     let average_time_per_round_ns =
-        (warmup_end_ts_ns - warmup_start_ts_ns) / warmup_times_ns.len() as u64;
+        (warmup_end_ts_ns - warmup_start_ts_ns) / warmup_timestamps.len() as u64;
 
     let actual_min_rounds = compute_min_rounds(config, average_time_per_round_ns);
     let actual_max_rounds = compute_max_rounds(config, average_time_per_round_ns);

crates/exec-harness/src/walltime/config.rs

@@ -1,4 +1,5 @@
 use crate::prelude::*;
+use serde::{Deserialize, Serialize};
 use std::time::Duration;
 
 const DEFAULT_WARMUP_TIME_NS: u64 = 1_000_000_000; // 1 second
@@ -27,7 +28,7 @@ fn parse_duration_to_ns(s: &str) -> Result<u64> {
 ///
 /// ⚠️ Make sure to update WalltimeExecutionArgs::to_cli_args() when fields change, else the runner
 /// will not properly forward arguments
-#[derive(Debug, Clone, Default, clap::Args)]
+#[derive(Debug, Clone, Default, clap::Args, Serialize, Deserialize)]
 pub struct WalltimeExecutionArgs {
     /// Duration of the warmup phase before measurement starts.
     /// During warmup, the benchmark runs to stabilize performance (e.g., JIT compilation, cache warming).