From 9829a0cb06da7bad3e4187232a11e2a9efa0ab65 Mon Sep 17 00:00:00 2001 From: Adrien Cacciaguerra Date: Tue, 11 Mar 2025 18:00:22 +0100 Subject: [PATCH] feat(run): add mode command arg --- README.md | 17 +++++++++++++ src/run/config.rs | 7 +++++ src/run/mod.rs | 19 +++++++++++--- src/run/runner/mod.rs | 38 +++++----------------------- src/run/runner/wall_time/executor.rs | 2 +- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 9dc9211e..6fcddc5f 100644 --- a/README.md +++ b/README.md @@ -78,3 +78,20 @@ Use the `CODSPEED_LOG` environment variable to set the logging level: ```bash CODSPEED_LOG=debug codspeed run ... ``` + +### Changing the mode of the runner + +By default, the runner will run the benchmark in the `instrumentation` mode. You can specify the mode with the `--mode` flag of the `run` command: + +```bash +# Run in the `instrumentation` mode +codspeed run --mode instrumentation + +# Run in the `walltime` mode +codspeed run --mode walltime +``` + +> [!WARNING] +> We strongly recommend not changing this mode unless you know what you are doing. +> Using the `walltime` mode on traditional VMs/Hosted Runners will lead to inconsistent data. For the best results, we recommend using CodSpeed Hosted Macro Runners, which are fine-tuned for performance measurement consistency. +> Check out the [Walltime Instrument Documentation](https://docs.codspeed.io/instruments/walltime/) for more details. diff --git a/src/run/config.rs b/src/run/config.rs index 6ab959a6..87b28f72 100644 --- a/src/run/config.rs +++ b/src/run/config.rs @@ -4,6 +4,8 @@ use url::Url; use crate::run::RunArgs; +use super::RunnerMode; + #[derive(Debug)] pub struct Config { pub upload_url: Url, @@ -11,6 +13,7 @@ pub struct Config { pub working_directory: Option, pub command: String, + pub mode: RunnerMode, pub instruments: Instruments, pub skip_upload: bool, @@ -32,6 +35,7 @@ impl Config { token: None, working_directory: None, command: "".into(), + mode: RunnerMode::Instrumentation, instruments: Instruments::test(), skip_upload: false, skip_setup: false, @@ -52,6 +56,7 @@ impl TryFrom for Config { upload_url, token: args.token, working_directory: args.working_directory, + mode: args.mode, instruments, command: args.command.join(" "), skip_upload: args.skip_upload, @@ -72,6 +77,7 @@ mod tests { upload_url: None, token: None, working_directory: None, + mode: RunnerMode::Instrumentation, instruments: vec![], mongo_uri_env_name: None, skip_upload: false, @@ -94,6 +100,7 @@ mod tests { upload_url: Some("https://example.com/upload".into()), token: Some("token".into()), working_directory: Some("/tmp".into()), + mode: RunnerMode::Instrumentation, instruments: vec!["mongodb".into()], mongo_uri_env_name: Some("MONGODB_URI".into()), skip_upload: true, diff --git a/src/run/mod.rs b/src/run/mod.rs index 960524c0..72b43579 100644 --- a/src/run/mod.rs +++ b/src/run/mod.rs @@ -4,10 +4,11 @@ use crate::prelude::*; use crate::run::{config::Config, logger::Logger}; use crate::VERSION; use check_system::SystemInfo; -use clap::Args; +use clap::{Args, ValueEnum}; use instruments::mongo_tracer::{install_mongodb_tracer, MongoTracer}; use run_environment::interfaces::RunEnvironment; use runner::get_run_data; +use serde::Serialize; pub mod check_system; pub mod helpers; @@ -36,6 +37,14 @@ fn show_banner() { debug!("codspeed v{}", VERSION); } +#[derive(ValueEnum, Clone, Default, Debug, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum RunnerMode { + #[default] + Instrumentation, + Walltime, +} + #[derive(Args, Debug)] pub struct RunArgs { /// The upload URL to use for uploading the results, useful for on-premises installations @@ -50,6 +59,10 @@ pub struct RunArgs { #[arg(long)] pub working_directory: Option, + /// The mode to run the benchmarks in. + #[arg(long, default_value_t, value_enum, env = "CODSPEED_RUNNER_MODE")] + pub mode: RunnerMode, + /// Comma-separated list of instruments to enable. Possible values: mongodb. #[arg(long, value_delimiter = ',')] pub instruments: Vec, @@ -86,6 +99,7 @@ impl RunArgs { upload_url: None, token: None, working_directory: None, + mode: RunnerMode::Instrumentation, instruments: vec![], mongo_uri_env_name: None, skip_upload: false, @@ -117,8 +131,7 @@ pub async fn run(args: RunArgs, api_client: &CodSpeedAPIClient) -> Result<()> { let system_info = SystemInfo::new()?; check_system::check_system(&system_info)?; - let mode = runner::get_mode()?; - let executor = runner::get_executor_from_mode(mode); + let executor = runner::get_executor_from_mode(&config.mode); if !config.skip_setup { start_group!("Preparing the environment"); diff --git a/src/run/runner/mod.rs b/src/run/runner/mod.rs index 1f7e0b50..52800bb7 100644 --- a/src/run/runner/mod.rs +++ b/src/run/runner/mod.rs @@ -1,62 +1,36 @@ -use std::{env, fmt::Display}; +use std::fmt::Display; use crate::prelude::*; +use super::RunnerMode; + mod executor; mod helpers; mod interfaces; mod valgrind; mod wall_time; -use anyhow::bail; use executor::Executor; use helpers::profile_folder::create_profile_folder; pub use interfaces::{ExecutorName, RunData}; use valgrind::executor::ValgrindExecutor; use wall_time::executor::WallTimeExecutor; -pub enum RunnerMode { - Instrumentation, - WallTime, -} - impl Display for RunnerMode { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { RunnerMode::Instrumentation => write!(f, "instrumentation"), - RunnerMode::WallTime => write!(f, "walltime"), - } - } -} - -impl TryFrom<&str> for RunnerMode { - type Error = anyhow::Error; - - fn try_from(value: &str) -> Result { - match value { - "instrumentation" => Ok(RunnerMode::Instrumentation), - "walltime" => Ok(RunnerMode::WallTime), - _ => bail!("Unknown runner mode: {}", value), + RunnerMode::Walltime => write!(f, "walltime"), } } } pub const EXECUTOR_TARGET: &str = "executor"; -pub fn get_mode() -> Result { - if let Ok(runner_mode) = env::var("CODSPEED_RUNNER_MODE") { - debug!("CODSPEED_RUNNER_MODE is set to {}", runner_mode); - RunnerMode::try_from(runner_mode.as_str()) - } else { - debug!("CODSPEED_RUNNER_MODE is not set, using instrumentation"); - Ok(RunnerMode::Instrumentation) - } -} - -pub fn get_executor_from_mode(mode: RunnerMode) -> Box { +pub fn get_executor_from_mode(mode: &RunnerMode) -> Box { match mode { RunnerMode::Instrumentation => Box::new(ValgrindExecutor), - RunnerMode::WallTime => Box::new(WallTimeExecutor), + RunnerMode::Walltime => Box::new(WallTimeExecutor), } } diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index 6bb03520..51ca1a8c 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -29,7 +29,7 @@ impl Executor for WallTimeExecutor { let mut cmd = Command::new("sh"); cmd.envs(get_base_injected_env( - RunnerMode::WallTime, + RunnerMode::Walltime, &run_data.profile_folder, ));