From ca5d25c3414e18fb37f31dd4c13fba92ac51eede Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 24 Jul 2025 10:40:41 +0200 Subject: [PATCH 1/6] feat: parse `go test` json output --- .gitattributes | 2 + src/run/runner/wall_time/golang/mod.rs | 1 + src/run/runner/wall_time/golang/parser.rs | 180 ++++++++++++++++++ .../wall_time/golang/testdata/fuego.txt | 3 + .../wall_time/golang/testdata/simple.txt | 3 + src/run/runner/wall_time/mod.rs | 1 + 6 files changed, 190 insertions(+) create mode 100644 .gitattributes create mode 100644 src/run/runner/wall_time/golang/mod.rs create mode 100644 src/run/runner/wall_time/golang/parser.rs create mode 100644 src/run/runner/wall_time/golang/testdata/fuego.txt create mode 100644 src/run/runner/wall_time/golang/testdata/simple.txt diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..c2c7404c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +src/run/runner/wall_time/golang/testdata/fuego.txt filter=lfs diff=lfs merge=lfs -text +src/run/runner/wall_time/golang/testdata/simple.txt filter=lfs diff=lfs merge=lfs -text diff --git a/src/run/runner/wall_time/golang/mod.rs b/src/run/runner/wall_time/golang/mod.rs new file mode 100644 index 00000000..b93e263b --- /dev/null +++ b/src/run/runner/wall_time/golang/mod.rs @@ -0,0 +1 @@ +mod parser; diff --git a/src/run/runner/wall_time/golang/parser.rs b/src/run/runner/wall_time/golang/parser.rs new file mode 100644 index 00000000..f58a6958 --- /dev/null +++ b/src/run/runner/wall_time/golang/parser.rs @@ -0,0 +1,180 @@ +use std::collections::HashMap; + +use crate::prelude::*; +use itertools::Itertools; +use regex::Regex; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RawTestOutput { + #[serde(rename = "Time")] + pub time: Option, + #[serde(rename = "Action")] + pub action: String, + #[serde(rename = "Package")] + pub package: String, + #[serde(rename = "Test")] + pub test: Option, + #[serde(rename = "Output")] + pub output: Option, + #[serde(rename = "Elapsed")] + pub elapsed: Option, +} + +pub struct RawOutput { + name: String, + time: f64, + iters: u64, +} + +impl RawOutput { + fn parse_output(line: &str) -> Result> { + lazy_static::lazy_static! { + static ref BENCHMARK_REGEX: Regex = Regex::new( + r"^(Benchmark[\w/]+)(?:-\d+)?\s+(\d+)\s+([0-9.]+)\s*ns/op" + ).unwrap(); + } + + if let Some(captures) = BENCHMARK_REGEX.captures(line.trim()) { + let name = captures + .get(1) + .context("Failed to get benchmark name")? + .as_str() + .to_string(); + let iters: u64 = captures + .get(2) + .context("Failed to get iterations")? + .as_str() + .parse()?; + let time: f64 = captures + .get(3) + .context("Failed to get time")? + .as_str() + .parse()?; + + Ok(Some(RawOutput { name, time, iters })) + } else { + Ok(None) + } + } + + pub fn parse(output: &str) -> Result> { + let mut results = Vec::new(); + for line in output.lines() { + let event: RawTestOutput = serde_json::from_str(line)?; + + if event.action != "output" { + continue; + } + let Some(output_text) = &event.output else { + continue; + }; + let Some(measurement) = Self::parse_output(output_text)? else { + continue; + }; + + results.push((event.package, measurement)); + } + + Ok(results) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BenchmarkData { + pub package: String, + pub name: String, + pub times: Vec, + pub iters: Vec, +} + +impl BenchmarkData { + pub fn process_raw_results(raw_results: Vec<(String, RawOutput)>) -> Vec { + let grouped: HashMap<(String, String), Vec<&(String, RawOutput)>> = raw_results + .iter() + .into_group_map_by(|(package, bench)| (package.clone(), bench.name.clone())); + + grouped + .into_iter() + .map(|((package, name), measurements)| BenchmarkData { + package, + name, + times: measurements.iter().map(|(_, m)| m.time).collect(), + iters: measurements.iter().map(|(_, m)| m.iters).collect(), + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_output() { + let line = "BenchmarkFibonacci10-16 \t 15564\t 755.2 ns/op\n"; + + let result = RawOutput::parse_output(line).unwrap().unwrap(); + assert_eq!(result.name, "BenchmarkFibonacci10"); + assert_eq!(result.iters, 15564); + assert!((result.time - 755.2).abs() < 0.1); + + let line = + "BenchmarkOutTransform/pointer_to_value-16 \t 8257989\t 162.7 ns/op\n"; + let result = RawOutput::parse_output(line).unwrap().unwrap(); + assert_eq!(result.name, "BenchmarkOutTransform/pointer_to_value"); + assert_eq!(result.iters, 8257989); + assert!((result.time - 162.7).abs() < 0.1); + } + + #[test] + fn test_parse_output_no_match() { + // Test line that doesn't match benchmark pattern + let line = "=== RUN BenchmarkFibonacci10\n"; + + let result = RawOutput::parse_output(line).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_parse_and_process_benchmark_data() { + const RESULT: &str = include_str!("testdata/simple.txt"); + + let raw_results = RawOutput::parse(RESULT).unwrap(); + let processed = BenchmarkData::process_raw_results(raw_results); + assert_eq!(processed.len(), 3); + + let fib10 = processed + .iter() + .find(|b| b.name == "BenchmarkFibonacci10") + .unwrap(); + + let fib20 = processed + .iter() + .find(|b| b.name == "BenchmarkFibonacci20") + .unwrap(); + + let fib30 = processed + .iter() + .find(|b| b.name == "BenchmarkFibonacci30") + .unwrap(); + + assert_eq!(fib10.package, "example"); + assert_eq!(fib10.times.len(), 10); + assert_eq!(fib10.iters.len(), 10); + assert_eq!(fib20.package, "example"); + assert_eq!(fib20.times.len(), 10); + assert_eq!(fib20.iters.len(), 10); + assert_eq!(fib30.package, "example"); + assert_eq!(fib30.times.len(), 10); + assert_eq!(fib30.iters.len(), 10); + } + + #[test] + fn test_parse_fuego() { + let content = include_str!("testdata/fuego.txt"); + let raw_results = RawOutput::parse(content).unwrap(); + let processed = BenchmarkData::process_raw_results(raw_results); + assert_eq!(processed.len(), 19); + } +} diff --git a/src/run/runner/wall_time/golang/testdata/fuego.txt b/src/run/runner/wall_time/golang/testdata/fuego.txt new file mode 100644 index 00000000..74200870 --- /dev/null +++ b/src/run/runner/wall_time/golang/testdata/fuego.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842a09f7689fc1e8a78a3fa9454bb9d1c9a3fbc89696b5e7403104a76f7ad15c +size 545001 diff --git a/src/run/runner/wall_time/golang/testdata/simple.txt b/src/run/runner/wall_time/golang/testdata/simple.txt new file mode 100644 index 00000000..0d1101e2 --- /dev/null +++ b/src/run/runner/wall_time/golang/testdata/simple.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e94f9348ab435ae3dcc08cb94e55b3910ea9447b4e47f174dba5f4e9731312 +size 6862 diff --git a/src/run/runner/wall_time/mod.rs b/src/run/runner/wall_time/mod.rs index ca152f25..c9cc4b35 100644 --- a/src/run/runner/wall_time/mod.rs +++ b/src/run/runner/wall_time/mod.rs @@ -1,2 +1,3 @@ pub mod executor; +pub mod golang; pub mod perf; From 3d18a8f0ef9503e8f7396394765d8f6fc5dddc86 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 24 Jul 2025 10:43:18 +0200 Subject: [PATCH 2/6] chore: add walltime_result from codspeed-rust --- .../wall_time/golang/walltime_results.rs | 326 ++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 src/run/runner/wall_time/golang/walltime_results.rs diff --git a/src/run/runner/wall_time/golang/walltime_results.rs b/src/run/runner/wall_time/golang/walltime_results.rs new file mode 100644 index 00000000..b7f0c491 --- /dev/null +++ b/src/run/runner/wall_time/golang/walltime_results.rs @@ -0,0 +1,326 @@ +use anyhow::{Context, Result}; +use std::{ + io::Write, + path::{Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; +use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics}; + +const IQR_OUTLIER_FACTOR: f64 = 1.5; +const STDEV_OUTLIER_FACTOR: f64 = 3.0; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkMetadata { + pub name: String, + pub uri: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct BenchmarkStats { + min_ns: f64, + max_ns: f64, + mean_ns: f64, + stdev_ns: f64, + + q1_ns: f64, + median_ns: f64, + q3_ns: f64, + + rounds: u64, + total_time: f64, + iqr_outlier_rounds: u64, + stdev_outlier_rounds: u64, + iter_per_round: u64, + warmup_iters: u64, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +struct BenchmarkConfig { + warmup_time_ns: Option, + min_round_time_ns: Option, + max_time_ns: Option, + max_rounds: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct WalltimeBenchmark { + #[serde(flatten)] + metadata: BenchmarkMetadata, + + config: BenchmarkConfig, + stats: BenchmarkStats, +} + +impl WalltimeBenchmark { + /// Entry point called in patched integration to harvest raw walltime data + /// + /// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work + /// + /// # Arguments + /// + /// - `scope`: The used integration, e.g. "divan" or "criterion" + /// - `name`: The name of the benchmark + /// - `uri`: The URI of the benchmark + /// - `iters_per_round`: The number of iterations for each round (=sample_size), e.g. `[1, 2, 3]` (variable) or `[2, 2, 2, 2]` (constant). + /// - `times_per_round_ns`: The measured time for each round in nanoseconds, e.g. `[1000, 2000, 3000]` + /// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined) + /// + /// # Pseudo-code + /// + /// ```text + /// let sample_count = /* The number of executions for the same benchmark. */ + /// let sample_size = iters_per_round = vec![/* The number of iterations within each sample. */]; + /// for round in 0..sample_count { + /// let times_per_round_ns = 0; + /// for iteration in 0..sample_size[round] { + /// run_benchmark(); + /// times_per_round_ns += /* measured execution time */; + /// } + /// } + /// ``` + /// + pub fn collect_raw_walltime_results( + scope: &str, + name: String, + uri: String, + iters_per_round: Vec, + times_per_round_ns: Vec, + max_time_ns: Option, + ) { + if !crate::utils::running_with_codspeed_runner() { + return; + } + let workspace_root = std::env::var("CODSPEED_CARGO_WORKSPACE_ROOT").map(PathBuf::from); + let Ok(workspace_root) = workspace_root else { + eprintln!("codspeed failed to get workspace root. skipping"); + return; + }; + let data = WalltimeBenchmark::from_runtime_data( + name, + uri, + iters_per_round, + times_per_round_ns, + max_time_ns, + ); + data.dump_to_results(&workspace_root, scope); + } + + pub fn from_runtime_data( + name: String, + uri: String, + iters_per_round: Vec, + times_per_round_ns: Vec, + max_time_ns: Option, + ) -> Self { + let total_time = times_per_round_ns.iter().sum::() as f64 / 1_000_000_000.0; + let time_per_iteration_per_round_ns: Vec<_> = times_per_round_ns + .into_iter() + .zip(&iters_per_round) + .map(|(time_per_round, iter_per_round)| time_per_round / iter_per_round) + .map(|t| t as f64) + .collect::>(); + + let mut data = Data::new(time_per_iteration_per_round_ns); + let rounds = data.len() as u64; + + let mean_ns = data.mean().unwrap(); + + let stdev_ns = if data.len() < 2 { + // std_dev() returns f64::NAN if data has less than two entries, so we have to + // manually handle this case. + 0.0 + } else { + data.std_dev().unwrap() + }; + + let q1_ns = data.quantile(0.25); + let median_ns = data.median(); + let q3_ns = data.quantile(0.75); + + let iqr_ns = q3_ns - q1_ns; + let iqr_outlier_rounds = data + .iter() + .filter(|&&t| { + t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns + }) + .count() as u64; + + let stdev_outlier_rounds = data + .iter() + .filter(|&&t| { + t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns + || t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns + }) + .count() as u64; + + let min_ns = data.min(); + let max_ns = data.max(); + + // TODO(COD-1056): We currently only support single iteration count per round + let iter_per_round = + (iters_per_round.iter().sum::() / iters_per_round.len() as u128) as u64; + let warmup_iters = 0; // FIXME: add warmup detection + + let stats = BenchmarkStats { + min_ns, + max_ns, + mean_ns, + stdev_ns, + q1_ns, + median_ns, + q3_ns, + rounds, + total_time, + iqr_outlier_rounds, + stdev_outlier_rounds, + iter_per_round, + warmup_iters, + }; + + WalltimeBenchmark { + metadata: BenchmarkMetadata { name, uri }, + config: BenchmarkConfig { + max_time_ns: max_time_ns.map(|t| t as f64), + ..Default::default() + }, + stats, + } + } + + fn dump_to_results(&self, workspace_root: &Path, scope: &str) { + let output_dir = result_dir_from_workspace_root(workspace_root).join(scope); + std::fs::create_dir_all(&output_dir).unwrap(); + let bench_id = uuid::Uuid::new_v4().to_string(); + let output_path = output_dir.join(format!("{bench_id}.json")); + let mut writer = std::fs::File::create(&output_path).expect("Failed to create the file"); + serde_json::to_writer_pretty(&mut writer, self).expect("Failed to write the data"); + writer.flush().expect("Failed to flush the writer"); + } + + pub fn is_invalid(&self) -> bool { + self.stats.min_ns < f64::EPSILON + } + + pub fn name(&self) -> &str { + &self.metadata.name + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct Instrument { + #[serde(rename = "type")] + type_: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Creator { + name: String, + version: String, + pid: u32, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct WalltimeResults { + creator: Creator, + instrument: Instrument, + benchmarks: Vec, +} + +impl WalltimeResults { + pub fn collect_walltime_results(workspace_root: &Path) -> Result { + // retrieve data from `{workspace_root}/target/codspeed/raw_results/{scope}/*.json + let benchmarks = glob::glob(&format!( + "{}/**/*.json", + result_dir_from_workspace_root(workspace_root) + .to_str() + .unwrap(), + ))? + .map(|sample| -> Result<_> { + let sample = sample?; + serde_json::from_reader::<_, WalltimeBenchmark>(std::fs::File::open(&sample)?) + .context("Failed to read benchmark data") + }) + .collect::>>()?; + + Ok(WalltimeResults { + instrument: Instrument { + type_: "walltime".to_string(), + }, + creator: Creator { + name: "codspeed-rust".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + pid: std::process::id(), + }, + benchmarks, + }) + } + + pub fn clear(workspace_root: &Path) -> Result<()> { + let raw_results_dir = result_dir_from_workspace_root(workspace_root); + std::fs::remove_dir_all(&raw_results_dir).ok(); // ignore errors when the directory does not exist + std::fs::create_dir_all(&raw_results_dir) + .context("Failed to create raw_results directory")?; + Ok(()) + } + + pub fn benchmarks(&self) -> &[WalltimeBenchmark] { + &self.benchmarks + } +} + +// FIXME: This assumes that the cargo target dir is `target`, and duplicates information with +// `cargo-codspeed::helpers::get_codspeed_target_dir` +fn result_dir_from_workspace_root(workspace_root: &Path) -> PathBuf { + workspace_root + .join("target") + .join("codspeed") + .join("walltime") + .join("raw_results") +} + +#[cfg(test)] +mod tests { + use super::*; + + const NAME: &str = "benchmark"; + const URI: &str = "test::benchmark"; + + #[test] + fn test_parse_single_benchmark() { + let benchmark = WalltimeBenchmark::from_runtime_data( + NAME.to_string(), + URI.to_string(), + vec![1], + vec![42], + None, + ); + assert_eq!(benchmark.stats.stdev_ns, 0.); + assert_eq!(benchmark.stats.min_ns, 42.); + assert_eq!(benchmark.stats.max_ns, 42.); + assert_eq!(benchmark.stats.mean_ns, 42.); + } + + #[test] + fn test_parse_bench_with_variable_iterations() { + let iters_per_round = vec![1, 2, 3, 4, 5, 6]; + let total_rounds = iters_per_round.iter().sum::() as f64; + + let benchmark = WalltimeBenchmark::from_runtime_data( + NAME.to_string(), + URI.to_string(), + iters_per_round, + vec![42, 42 * 2, 42 * 3, 42 * 4, 42 * 5, 42 * 6], + None, + ); + + assert_eq!(benchmark.stats.stdev_ns, 0.); + assert_eq!(benchmark.stats.min_ns, 42.); + assert_eq!(benchmark.stats.max_ns, 42.); + assert_eq!(benchmark.stats.mean_ns, 42.); + assert_eq!( + benchmark.stats.total_time, + 42. * total_rounds / 1_000_000_000.0 + ); + } +} From 2af536f0fdd115c0e5479096933a056565fe752a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 24 Jul 2025 10:46:40 +0200 Subject: [PATCH 3/6] refactor: adapt walltime_results to runner --- Cargo.lock | 20 +++ Cargo.toml | 1 + src/run/runner/wall_time/golang/mod.rs | 34 +++++ .../wall_time/golang/walltime_results.rs | 131 ++---------------- 4 files changed, 64 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65659ee1..f257f7e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,15 @@ version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "async-compression" version = "0.4.18" @@ -323,6 +332,7 @@ dependencies = [ "sha256", "shell-quote", "simplelog", + "statrs", "sysinfo", "temp-env", "tempfile", @@ -2143,6 +2153,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strsim" version = "0.11.1" diff --git a/Cargo.toml b/Cargo.toml index 2a45c774..7225d12a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ debugid = "0.8.0" memmap2 = "0.9.5" nix = { version = "0.29.0", features = ["fs", "user"] } futures = "0.3.31" +statrs = { version = "0.18.0", default-features = false } [target.'cfg(target_os = "linux")'.dependencies] procfs = "0.17.0" diff --git a/src/run/runner/wall_time/golang/mod.rs b/src/run/runner/wall_time/golang/mod.rs index b93e263b..4f1db06a 100644 --- a/src/run/runner/wall_time/golang/mod.rs +++ b/src/run/runner/wall_time/golang/mod.rs @@ -1 +1,35 @@ +use crate::prelude::*; +use std::path::Path; + mod parser; +mod walltime_results; + +pub fn collect_walltime_results(stdout: &str, dst_dir: &Path) -> Result<()> { + let benchmarks = parser::BenchmarkData::process_raw_results(parser::RawOutput::parse(stdout)?) + .into_iter() + .map(|result| { + let uri = format!("{}::{}", result.package, result.name); + walltime_results::WalltimeBenchmark::from_runtime_data( + result.name, + uri, + result.iters.into_iter().map(|i| i as u128).collect(), + result.times.into_iter().map(|t| t as u128).collect(), + None, + ) + }) + .collect::>(); + debug!("Parsed {} benchmarks", benchmarks.len()); + + let pid = std::process::id(); + let creator = walltime_results::Creator { + name: "runner".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + pid, + }; + let results = walltime_results::WalltimeResults::new(benchmarks, creator)?; + + let mut file = std::fs::File::create(dst_dir.join(format!("{pid}.json")))?; + serde_json::to_writer_pretty(&mut file, &results)?; + + Ok(()) +} diff --git a/src/run/runner/wall_time/golang/walltime_results.rs b/src/run/runner/wall_time/golang/walltime_results.rs index b7f0c491..730a278c 100644 --- a/src/run/runner/wall_time/golang/walltime_results.rs +++ b/src/run/runner/wall_time/golang/walltime_results.rs @@ -1,8 +1,6 @@ -use anyhow::{Context, Result}; -use std::{ - io::Write, - path::{Path, PathBuf}, -}; +// NOTE: This file was taken from `codspeed-rust` and modified a bit to fit this project. + +use anyhow::Result; use serde::{Deserialize, Serialize}; use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics}; @@ -53,59 +51,6 @@ pub struct WalltimeBenchmark { } impl WalltimeBenchmark { - /// Entry point called in patched integration to harvest raw walltime data - /// - /// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work - /// - /// # Arguments - /// - /// - `scope`: The used integration, e.g. "divan" or "criterion" - /// - `name`: The name of the benchmark - /// - `uri`: The URI of the benchmark - /// - `iters_per_round`: The number of iterations for each round (=sample_size), e.g. `[1, 2, 3]` (variable) or `[2, 2, 2, 2]` (constant). - /// - `times_per_round_ns`: The measured time for each round in nanoseconds, e.g. `[1000, 2000, 3000]` - /// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined) - /// - /// # Pseudo-code - /// - /// ```text - /// let sample_count = /* The number of executions for the same benchmark. */ - /// let sample_size = iters_per_round = vec![/* The number of iterations within each sample. */]; - /// for round in 0..sample_count { - /// let times_per_round_ns = 0; - /// for iteration in 0..sample_size[round] { - /// run_benchmark(); - /// times_per_round_ns += /* measured execution time */; - /// } - /// } - /// ``` - /// - pub fn collect_raw_walltime_results( - scope: &str, - name: String, - uri: String, - iters_per_round: Vec, - times_per_round_ns: Vec, - max_time_ns: Option, - ) { - if !crate::utils::running_with_codspeed_runner() { - return; - } - let workspace_root = std::env::var("CODSPEED_CARGO_WORKSPACE_ROOT").map(PathBuf::from); - let Ok(workspace_root) = workspace_root else { - eprintln!("codspeed failed to get workspace root. skipping"); - return; - }; - let data = WalltimeBenchmark::from_runtime_data( - name, - uri, - iters_per_round, - times_per_round_ns, - max_time_ns, - ); - data.dump_to_results(&workspace_root, scope); - } - pub fn from_runtime_data( name: String, uri: String, @@ -187,24 +132,6 @@ impl WalltimeBenchmark { stats, } } - - fn dump_to_results(&self, workspace_root: &Path, scope: &str) { - let output_dir = result_dir_from_workspace_root(workspace_root).join(scope); - std::fs::create_dir_all(&output_dir).unwrap(); - let bench_id = uuid::Uuid::new_v4().to_string(); - let output_path = output_dir.join(format!("{bench_id}.json")); - let mut writer = std::fs::File::create(&output_path).expect("Failed to create the file"); - serde_json::to_writer_pretty(&mut writer, self).expect("Failed to write the data"); - writer.flush().expect("Failed to flush the writer"); - } - - pub fn is_invalid(&self) -> bool { - self.stats.min_ns < f64::EPSILON - } - - pub fn name(&self) -> &str { - &self.metadata.name - } } #[derive(Debug, Serialize, Deserialize)] @@ -214,10 +141,10 @@ struct Instrument { } #[derive(Debug, Serialize, Deserialize)] -struct Creator { - name: String, - version: String, - pid: u32, +pub struct Creator { + pub name: String, + pub version: String, + pub pid: u32, } #[derive(Debug, Serialize, Deserialize)] @@ -228,55 +155,15 @@ pub struct WalltimeResults { } impl WalltimeResults { - pub fn collect_walltime_results(workspace_root: &Path) -> Result { - // retrieve data from `{workspace_root}/target/codspeed/raw_results/{scope}/*.json - let benchmarks = glob::glob(&format!( - "{}/**/*.json", - result_dir_from_workspace_root(workspace_root) - .to_str() - .unwrap(), - ))? - .map(|sample| -> Result<_> { - let sample = sample?; - serde_json::from_reader::<_, WalltimeBenchmark>(std::fs::File::open(&sample)?) - .context("Failed to read benchmark data") - }) - .collect::>>()?; - + pub fn new(benchmarks: Vec, creator: Creator) -> Result { Ok(WalltimeResults { instrument: Instrument { type_: "walltime".to_string(), }, - creator: Creator { - name: "codspeed-rust".to_string(), - version: env!("CARGO_PKG_VERSION").to_string(), - pid: std::process::id(), - }, + creator, benchmarks, }) } - - pub fn clear(workspace_root: &Path) -> Result<()> { - let raw_results_dir = result_dir_from_workspace_root(workspace_root); - std::fs::remove_dir_all(&raw_results_dir).ok(); // ignore errors when the directory does not exist - std::fs::create_dir_all(&raw_results_dir) - .context("Failed to create raw_results directory")?; - Ok(()) - } - - pub fn benchmarks(&self) -> &[WalltimeBenchmark] { - &self.benchmarks - } -} - -// FIXME: This assumes that the cargo target dir is `target`, and duplicates information with -// `cargo-codspeed::helpers::get_codspeed_target_dir` -fn result_dir_from_workspace_root(workspace_root: &Path) -> PathBuf { - workspace_root - .join("target") - .join("codspeed") - .join("walltime") - .join("raw_results") } #[cfg(test)] From ac4c59d25132b47f106033b38da8742575b19943 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 24 Jul 2025 10:49:08 +0200 Subject: [PATCH 4/6] feat: parse golang test output --- .../helpers/run_command_with_log_pipe.rs | 113 +++++++++++++++--- src/run/runner/wall_time/executor.rs | 14 ++- src/run/runner/wall_time/perf/mod.rs | 4 +- 3 files changed, 112 insertions(+), 19 deletions(-) diff --git a/src/run/runner/helpers/run_command_with_log_pipe.rs b/src/run/runner/helpers/run_command_with_log_pipe.rs index 2b034450..4491db23 100644 --- a/src/run/runner/helpers/run_command_with_log_pipe.rs +++ b/src/run/runner/helpers/run_command_with_log_pipe.rs @@ -5,22 +5,37 @@ use std::future::Future; use std::io::{Read, Write}; use std::process::Command; use std::process::ExitStatus; +use std::sync::{Arc, Mutex}; use std::thread; +struct CmdRunnerOptions { + on_process_spawned: Option, + capture_stdout: bool, +} + +impl Default for CmdRunnerOptions { + fn default() -> Self { + Self { + on_process_spawned: None, + capture_stdout: false, + } + } +} + /// Run a command and log its output to stdout and stderr /// /// # Arguments /// - `cmd`: The command to run. -/// - `cb`: A callback function that takes the process ID and returns a result. +/// - `options`: Configuration options for the runner (e.g. capture output, run callback) /// /// # Returns -/// -/// The exit status of the command. -/// -pub async fn run_command_with_log_pipe_and_callback( +/// A tuple containing: +/// - `ExitStatus`: The exit status of the executed command +/// - `Option`: Captured stdout if `capture_stdout` was true, otherwise None +async fn run_command_with_log_pipe_and_options( mut cmd: Command, - cb: F, -) -> Result + options: CmdRunnerOptions, +) -> Result<(ExitStatus, Option)> where F: FnOnce(u32) -> Fut, Fut: Future>, @@ -29,14 +44,23 @@ where mut reader: impl Read, mut writer: impl Write, log_prefix: Option<&str>, + captured_output: Option>>>, ) -> Result<()> { let prefix = log_prefix.unwrap_or(""); let mut buffer = [0; 1024]; + let mut capture_guard = captured_output + .as_ref() + .map(|capture| capture.lock().unwrap()); loop { let bytes_read = reader.read(&mut buffer)?; if bytes_read == 0 { break; } + + if let Some(ref mut output) = capture_guard { + output.extend_from_slice(&buffer[..bytes_read]); + } + suspend_progress_bar(|| { writer.write_all(&buffer[..bytes_read]).unwrap(); trace!( @@ -57,19 +81,76 @@ where .context("failed to spawn the process")?; let stdout = process.stdout.take().expect("unable to get stdout"); let stderr = process.stderr.take().expect("unable to get stderr"); - thread::spawn(move || { - log_tee(stdout, std::io::stdout(), None).unwrap(); - }); - thread::spawn(move || { - log_tee(stderr, std::io::stderr(), Some("[stderr]")).unwrap(); - }); + let captured_stdout = if options.capture_stdout { + Some(Arc::new(Mutex::new(Vec::new()))) + } else { + None + }; + let (stdout_handle, stderr_handle) = { + let stdout_capture = captured_stdout.clone(); + let stdout_handle = thread::spawn(move || { + log_tee(stdout, std::io::stdout(), None, stdout_capture).unwrap(); + }); + let stderr_handle = thread::spawn(move || { + log_tee(stderr, std::io::stderr(), Some("[stderr]"), None).unwrap(); + }); - cb(process.id()).await?; + (stdout_handle, stderr_handle) + }; - process.wait().context("failed to wait for the process") + if let Some(cb) = options.on_process_spawned { + cb(process.id()).await?; + } + + let exit_status = process.wait().context("failed to wait for the process")?; + let _ = (stdout_handle.join().unwrap(), stderr_handle.join().unwrap()); + + let stdout_output = captured_stdout + .map(|capture| String::from_utf8_lossy(&capture.lock().unwrap()).to_string()); + Ok((exit_status, stdout_output)) +} + +pub async fn run_command_with_log_pipe_and_callback( + cmd: Command, + cb: F, +) -> Result<(ExitStatus, Option)> +where + F: FnOnce(u32) -> Fut, + Fut: Future>, +{ + run_command_with_log_pipe_and_options( + cmd, + CmdRunnerOptions { + on_process_spawned: Some(cb), + capture_stdout: false, + }, + ) + .await } pub async fn run_command_with_log_pipe(cmd: Command) -> Result { - run_command_with_log_pipe_and_callback(cmd, async |_| Ok(())).await + let (exit_status, _) = run_command_with_log_pipe_and_options( + cmd, + CmdRunnerOptions:: futures::future::Ready>> { + on_process_spawned: None, + capture_stdout: false, + }, + ) + .await?; + Ok(exit_status) +} + +pub async fn run_command_with_log_pipe_capture_stdout( + cmd: Command, +) -> Result<(ExitStatus, String)> { + let (exit_status, stdout) = run_command_with_log_pipe_and_options( + cmd, + CmdRunnerOptions:: futures::future::Ready>> { + on_process_spawned: None, + capture_stdout: true, + }, + ) + .await?; + Ok((exit_status, stdout.unwrap_or_default())) } diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index 1b29e81c..90b53b55 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -5,7 +5,8 @@ use crate::run::instruments::mongo_tracer::MongoTracer; use crate::run::runner::executor::Executor; use crate::run::runner::helpers::env::{get_base_injected_env, is_codspeed_debug_enabled}; use crate::run::runner::helpers::get_bench_command::get_bench_command; -use crate::run::runner::helpers::run_command_with_log_pipe::run_command_with_log_pipe; +use crate::run::runner::helpers::run_command_with_log_pipe::run_command_with_log_pipe_capture_stdout; +use crate::run::runner::wall_time::golang; use crate::run::runner::{ExecutorName, RunData}; use crate::run::{check_system::SystemInfo, config::Config}; use async_trait::async_trait; @@ -181,7 +182,16 @@ impl Executor for WallTimeExecutor { cmd.args(["sh", "-c", &bench_cmd]); debug!("cmd: {cmd:?}"); - run_command_with_log_pipe(cmd).await + let (status, stdout) = run_command_with_log_pipe_capture_stdout(cmd).await?; + + if config.command.trim().starts_with("go test") { + let results_folder = run_data.profile_folder.join("results"); + std::fs::create_dir_all(&results_folder)?; + + golang::collect_walltime_results(&stdout, &results_folder)?; + } + + Ok(status) } }; diff --git a/src/run/runner/wall_time/perf/mod.rs b/src/run/runner/wall_time/perf/mod.rs index 33a71aa8..9301eef6 100644 --- a/src/run/runner/wall_time/perf/mod.rs +++ b/src/run/runner/wall_time/perf/mod.rs @@ -133,7 +133,9 @@ impl PerfRunner { Ok(()) }; - run_command_with_log_pipe_and_callback(cmd, on_process_started).await + run_command_with_log_pipe_and_callback(cmd, on_process_started) + .await + .map(|(exit_status, _)| exit_status) } pub async fn save_files_to(&self, profile_folder: &PathBuf) -> anyhow::Result<()> { From 36831615d1f09e25e70d69601bbb5e5bae0e57fd Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 24 Jul 2025 11:44:54 +0200 Subject: [PATCH 5/6] fix: use correct times for golang --- src/run/runner/wall_time/golang/parser.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/run/runner/wall_time/golang/parser.rs b/src/run/runner/wall_time/golang/parser.rs index f58a6958..8df22588 100644 --- a/src/run/runner/wall_time/golang/parser.rs +++ b/src/run/runner/wall_time/golang/parser.rs @@ -99,7 +99,11 @@ impl BenchmarkData { .map(|((package, name), measurements)| BenchmarkData { package, name, - times: measurements.iter().map(|(_, m)| m.time).collect(), + // WalltimeResults expects times to be the _total time_ for each round. + times: measurements + .iter() + .map(|(_, m)| m.time * m.iters as f64) + .collect(), iters: measurements.iter().map(|(_, m)| m.iters).collect(), }) .collect() From 43f127132e902ccce282ea75f8cd4260c5563162 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 24 Jul 2025 11:50:55 +0200 Subject: [PATCH 6/6] fix(ci): enable lfs for tests --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ce9434b..1362cb12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,5 +21,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + lfs: true - uses: moonrepo/setup-rust@v1 - run: cargo test --all