feat(run): add mode command arg

README.md

@@ -78,3 +78,20 @@ Use the `CODSPEED_LOG` environment variable to set the logging level:
 ```bash
 CODSPEED_LOG=debug codspeed run ...
 ```
+
+### Changing the mode of the runner
+
+By default, the runner will run the benchmark in the `instrumentation` mode. You can specify the mode with the `--mode` flag of the `run` command:
+
+```bash
+# Run in the `instrumentation` mode
+codspeed run --mode instrumentation <my-benchmark-command>
+
+# Run in the `walltime` mode
+codspeed run --mode walltime <my-benchmark-command>
+```
+
+> [!WARNING]
+> We strongly recommend not changing this mode unless you know what you are doing.
+> Using the `walltime` mode on traditional VMs/Hosted Runners will lead to inconsistent data. For the best results, we recommend using CodSpeed Hosted Macro Runners, which are fine-tuned for performance measurement consistency.
+> Check out the [Walltime Instrument Documentation](https://docs.codspeed.io/instruments/walltime/) for more details.

src/run/config.rs

@@ -4,13 +4,16 @@ use url::Url;
 
 use crate::run::RunArgs;
 
+use super::RunnerMode;
+
 #[derive(Debug)]
 pub struct Config {
     pub upload_url: Url,
     pub token: Option<String>,
     pub working_directory: Option<String>,
     pub command: String,
 
+    pub mode: RunnerMode,
     pub instruments: Instruments,
 
     pub skip_upload: bool,
@@ -32,6 +35,7 @@ impl Config {
             token: None,
             working_directory: None,
             command: "".into(),
+            mode: RunnerMode::Instrumentation,
             instruments: Instruments::test(),
             skip_upload: false,
             skip_setup: false,
@@ -52,6 +56,7 @@ impl TryFrom<RunArgs> for Config {
             upload_url,
             token: args.token,
             working_directory: args.working_directory,
+            mode: args.mode,
             instruments,
             command: args.command.join(" "),
             skip_upload: args.skip_upload,
@@ -72,6 +77,7 @@ mod tests {
             upload_url: None,
             token: None,
             working_directory: None,
+            mode: RunnerMode::Instrumentation,
             instruments: vec![],
             mongo_uri_env_name: None,
             skip_upload: false,
@@ -94,6 +100,7 @@ mod tests {
             upload_url: Some("https://example.com/upload".into()),
             token: Some("token".into()),
             working_directory: Some("/tmp".into()),
+            mode: RunnerMode::Instrumentation,
             instruments: vec!["mongodb".into()],
             mongo_uri_env_name: Some("MONGODB_URI".into()),
             skip_upload: true,

src/run/mod.rs

@@ -4,10 +4,11 @@ use crate::prelude::*;
 use crate::run::{config::Config, logger::Logger};
 use crate::VERSION;
 use check_system::SystemInfo;
-use clap::Args;
+use clap::{Args, ValueEnum};
 use instruments::mongo_tracer::{install_mongodb_tracer, MongoTracer};
 use run_environment::interfaces::RunEnvironment;
 use runner::get_run_data;
+use serde::Serialize;
 
 pub mod check_system;
 pub mod helpers;
@@ -36,6 +37,14 @@ fn show_banner() {
     debug!("codspeed v{}", VERSION);
 }
 
+#[derive(ValueEnum, Clone, Default, Debug, Serialize)]
+#[serde(rename_all = "lowercase")]
+pub enum RunnerMode {
+    #[default]
+    Instrumentation,
+    Walltime,
+}
+
 #[derive(Args, Debug)]
 pub struct RunArgs {
     /// The upload URL to use for uploading the results, useful for on-premises installations
@@ -50,6 +59,10 @@ pub struct RunArgs {
     #[arg(long)]
     pub working_directory: Option<String>,
 
+    /// The mode to run the benchmarks in.
+    #[arg(long, default_value_t, value_enum, env = "CODSPEED_RUNNER_MODE")]
+    pub mode: RunnerMode,
+
     /// Comma-separated list of instruments to enable. Possible values: mongodb.
     #[arg(long, value_delimiter = ',')]
     pub instruments: Vec<String>,
@@ -86,6 +99,7 @@ impl RunArgs {
             upload_url: None,
             token: None,
             working_directory: None,
+            mode: RunnerMode::Instrumentation,
             instruments: vec![],
             mongo_uri_env_name: None,
             skip_upload: false,
@@ -117,8 +131,7 @@ pub async fn run(args: RunArgs, api_client: &CodSpeedAPIClient) -> Result<()> {
     let system_info = SystemInfo::new()?;
     check_system::check_system(&system_info)?;
 
-    let mode = runner::get_mode()?;
-    let executor = runner::get_executor_from_mode(mode);
+    let executor = runner::get_executor_from_mode(&config.mode);
 
     if !config.skip_setup {
         start_group!("Preparing the environment");

src/run/runner/mod.rs

@@ -1,62 +1,36 @@
-use std::{env, fmt::Display};
+use std::fmt::Display;
 
 use crate::prelude::*;
 
+use super::RunnerMode;
+
 mod executor;
 mod helpers;
 mod interfaces;
 mod valgrind;
 mod wall_time;
 
-use anyhow::bail;
 use executor::Executor;
 use helpers::profile_folder::create_profile_folder;
 pub use interfaces::{ExecutorName, RunData};
 use valgrind::executor::ValgrindExecutor;
 use wall_time::executor::WallTimeExecutor;
 
-pub enum RunnerMode {
-    Instrumentation,
-    WallTime,
-}
-
 impl Display for RunnerMode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             RunnerMode::Instrumentation => write!(f, "instrumentation"),
-            RunnerMode::WallTime => write!(f, "walltime"),
-        }
-    }
-}
-
-impl TryFrom<&str> for RunnerMode {
-    type Error = anyhow::Error;
-
-    fn try_from(value: &str) -> Result<Self> {
-        match value {
-            "instrumentation" => Ok(RunnerMode::Instrumentation),
-            "walltime" => Ok(RunnerMode::WallTime),
-            _ => bail!("Unknown runner mode: {}", value),
+            RunnerMode::Walltime => write!(f, "walltime"),
         }
     }
 }
 
 pub const EXECUTOR_TARGET: &str = "executor";
 
-pub fn get_mode() -> Result<RunnerMode> {
-    if let Ok(runner_mode) = env::var("CODSPEED_RUNNER_MODE") {
-        debug!("CODSPEED_RUNNER_MODE is set to {}", runner_mode);
-        RunnerMode::try_from(runner_mode.as_str())
-    } else {
-        debug!("CODSPEED_RUNNER_MODE is not set, using instrumentation");
-        Ok(RunnerMode::Instrumentation)
-    }
-}
-
-pub fn get_executor_from_mode(mode: RunnerMode) -> Box<dyn Executor> {
+pub fn get_executor_from_mode(mode: &RunnerMode) -> Box<dyn Executor> {
     match mode {
         RunnerMode::Instrumentation => Box::new(ValgrindExecutor),
-        RunnerMode::WallTime => Box::new(WallTimeExecutor),
+        RunnerMode::Walltime => Box::new(WallTimeExecutor),
     }
 }
 

src/run/runner/wall_time/executor.rs

@@ -29,7 +29,7 @@ impl Executor for WallTimeExecutor {
         let mut cmd = Command::new("sh");
 
         cmd.envs(get_base_injected_env(
-            RunnerMode::WallTime,
+            RunnerMode::Walltime,
             &run_data.profile_folder,
         ));