Validate walltime results before uploading and error out if no bench found

AGENTS.md

@@ -16,16 +16,21 @@ cargo build
 # Build in release mode
 cargo build --release
 
-# Run tests
-cargo test
+# Run tests (prefer nextest if available)
+cargo nextest run  # preferred if installed
+cargo test         # fallback if nextest is not available
 
 # Run specific test
-cargo test <test_name>
+cargo nextest run <test_name>  # with nextest
+cargo test <test_name>         # with cargo test
 
 # Run tests with output
-cargo test -- --nocapture
+cargo nextest run --nocapture  # with nextest
+cargo test -- --nocapture      # with cargo test
 ```
 
+**Note**: Always check if `cargo nextest` is available first (with `cargo nextest --version` or `which cargo-nextest`). If available, use it instead of `cargo test` as it provides faster and more reliable test execution.
+
 ### Running the Application
 ```bash
 # Build and run
@@ -85,9 +90,13 @@ The core functionality for running benchmarks:
 ## Testing
 
 The project uses:
-- Standard Rust `cargo test`
+- `cargo nextest` (preferred) or standard Rust `cargo test`
 - `insta` for snapshot testing
 - `rstest` for parameterized tests
 - `temp-env` for environment variable testing
 
 Test files include snapshots in `snapshots/` directories for various run environment providers.
+
+**Important**:
+- Always prefer `cargo nextest run` over `cargo test` when running tests, as it provides better performance and reliability.
+- Some walltime executor tests require `sudo` access and will fail in non-interactive environments (e.g., `test_walltime_executor::*`). These failures are expected if sudo is not available.

crates/runner-shared/src/lib.rs

@@ -2,3 +2,4 @@ pub mod debug_info;
 pub mod fifo;
 pub mod metadata;
 pub mod unwind_data;
+pub mod walltime_results;

crates/runner-shared/src/walltime_results.rs

@@ -0,0 +1,65 @@
+// WARN: Keep in sync with codspeed-rust
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct BenchmarkMetadata {
+    pub name: String,
+    pub uri: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct BenchmarkStats {
+    pub min_ns: f64,
+    pub max_ns: f64,
+    pub mean_ns: f64,
+    pub stdev_ns: f64,
+
+    pub q1_ns: f64,
+    pub median_ns: f64,
+    pub q3_ns: f64,
+
+    pub rounds: u64,
+    pub total_time: f64,
+    pub iqr_outlier_rounds: u64,
+    pub stdev_outlier_rounds: u64,
+    pub iter_per_round: u64,
+    pub warmup_iters: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+struct BenchmarkConfig {
+    warmup_time_ns: Option<f64>,
+    min_round_time_ns: Option<f64>,
+    max_time_ns: Option<f64>,
+    max_rounds: Option<u64>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct WalltimeBenchmark {
+    #[serde(flatten)]
+    pub metadata: BenchmarkMetadata,
+
+    config: BenchmarkConfig,
+    pub stats: BenchmarkStats,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Instrument {
+    #[serde(rename = "type")]
+    type_: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Creator {
+    name: String,
+    version: String,
+    pub pid: u32,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeResults {
+    pub creator: Creator,
+    pub instrument: Instrument,
+    pub benchmarks: Vec<WalltimeBenchmark>,
+}

src/run/runner/helpers/run_command_with_log_pipe.rs

@@ -32,20 +32,48 @@ where
     ) -> Result<()> {
         let prefix = log_prefix.unwrap_or("");
         let mut buffer = [0; 1024];
+        let mut line_buffer = Vec::new();
+
         loop {
             let bytes_read = reader.read(&mut buffer)?;
             if bytes_read == 0 {
+                // Flush any remaining data in the line buffer
+                if !line_buffer.is_empty() {
+                    suspend_progress_bar(|| {
+                        writer.write_all(&line_buffer).unwrap();
+                        trace!(
+                            target: EXECUTOR_TARGET,
+                            "{}{}",
+                            prefix,
+                            String::from_utf8_lossy(&line_buffer)
+                        );
+                    });
+                }
                 break;
             }
-            suspend_progress_bar(|| {
-                writer.write_all(&buffer[..bytes_read]).unwrap();
-                trace!(
-                    target: EXECUTOR_TARGET,
-                    "{}{}",
-                    prefix,
-                    String::from_utf8_lossy(&buffer[..bytes_read])
-                );
-            });
+
+            // Add the chunk to our line buffer
+            line_buffer.extend_from_slice(&buffer[..bytes_read]);
+
+            // Check if we have any complete lines (ending with \n or \r)
+            if let Some(last_newline_pos) =
+                line_buffer.iter().rposition(|&b| b == b'\n' || b == b'\r')
+            {
+                // We have at least one complete line, flush up to and including the last newline
+                let to_flush = &line_buffer[..=last_newline_pos];
+                suspend_progress_bar(|| {
+                    writer.write_all(to_flush).unwrap();
+                    trace!(
+                        target: EXECUTOR_TARGET,
+                        "{}{}",
+                        prefix,
+                        String::from_utf8_lossy(to_flush)
+                    );
+                });
+
+                // Keep the remainder in the buffer
+                line_buffer = line_buffer[last_newline_pos + 1..].to_vec();
+            }
         }
         Ok(())
     }

src/run/runner/valgrind/executor.rs

@@ -50,6 +50,13 @@ impl Executor for ValgrindExecutor {
     ) -> Result<()> {
         harvest_perf_maps(&run_data.profile_folder).await?;
 
+        // No matter the command in input, at this point valgrind will have been run and have produced output files.
+        //
+        // Contrary to walltime, checking that benchmarks have been detected here would require
+        // parsing the valgrind output files, which is not ideal at this stage.
+        // A comprehensive message will be sent to the user if no benchmarks are detected,
+        // even if it's later in the process than technically possible.
+
         Ok(())
     }
 }

src/run/runner/wall_time/executor.rs

@@ -1,3 +1,4 @@
+use super::helpers::validate_walltime_results;
 use super::perf::PerfRunner;
 use crate::prelude::*;
 use crate::run::RunnerMode;
@@ -224,6 +225,8 @@ impl Executor for WallTimeExecutor {
             perf.save_files_to(&run_data.profile_folder).await?;
         }
 
+        validate_walltime_results(&run_data.profile_folder)?;
+
         Ok(())
     }
 }