diff --git a/README.md b/README.md index ab996214..9dc9211e 100644 --- a/README.md +++ b/README.md @@ -59,22 +59,17 @@ codspeed run pytest ./tests --codspeed codspeed run pnpm vitest bench ``` +## Advanced usage + +### Installing tools before running + +You can install executors and instruments before running the benchmark with the `setup` command: + +```bash +codspeed setup ``` -Usage: codspeed run [OPTIONS] [COMMAND]... - -Arguments: - [COMMAND]... The bench command to run - -Options: - --upload-url - The upload URL to use for uploading the results, useful for on-premises installations - --token - The token to use for uploading the results, if not provided it will be read from the CODSPEED_TOKEN environment variable - --working-directory - The directory where the command will be executed - -h, --help - Print help -``` + +This is especially useful when configuring environments with tools such as docker. ### Logging level diff --git a/src/app.rs b/src/app.rs index 6a3a807c..2386180a 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,5 +1,9 @@ use crate::{ - api_client::CodSpeedAPIClient, auth, local_logger::CODSPEED_U8_COLOR_CODE, prelude::*, run, + api_client::CodSpeedAPIClient, + auth, + local_logger::{init_local_logger, CODSPEED_U8_COLOR_CODE}, + prelude::*, + run, setup, }; use clap::{ builder::{styling, Styles}, @@ -37,17 +41,27 @@ pub struct Cli { enum Commands { /// Run the bench command and upload the results to CodSpeed Run(run::RunArgs), - /// Commands related to authentication with CodSpeed + /// Manage the CLI authentication state Auth(auth::AuthArgs), + /// Pre-install the codspeed executors + Setup, } pub async fn run() -> Result<()> { let cli = Cli::parse(); let api_client = CodSpeedAPIClient::try_from(&cli)?; + match cli.command { + Commands::Run(_) => {} // Run is responsible for its own logger initialization + _ => { + init_local_logger()?; + } + } + match cli.command { Commands::Run(args) => run::run(args, &api_client).await?, Commands::Auth(args) => auth::run(args, &api_client).await?, + Commands::Setup => setup::setup().await?, } Ok(()) } diff --git a/src/auth.rs b/src/auth.rs index 23f27fc4..582a6628 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -1,10 +1,8 @@ use std::time::Duration; -use crate::local_logger::get_local_logger; use crate::{api_client::CodSpeedAPIClient, config::CodSpeedConfig, prelude::*}; use clap::{Args, Subcommand}; use console::style; -use simplelog::CombinedLogger; use tokio::time::{sleep, Instant}; #[derive(Debug, Args)] @@ -19,15 +17,7 @@ enum AuthCommands { Login, } -fn init_logger() -> Result<()> { - let logger = get_local_logger(); - CombinedLogger::init(vec![logger])?; - Ok(()) -} - pub async fn run(args: AuthArgs, api_client: &CodSpeedAPIClient) -> Result<()> { - init_logger()?; - match args.command { AuthCommands::Login => login(api_client).await?, } diff --git a/src/local_logger.rs b/src/local_logger.rs index 652d49bb..fabb8f6e 100644 --- a/src/local_logger.rs +++ b/src/local_logger.rs @@ -4,17 +4,17 @@ use std::{ time::Duration, }; +use crate::prelude::*; use console::{style, Style}; use indicatif::{ProgressBar, ProgressStyle}; use lazy_static::lazy_static; use log::Log; -use simplelog::SharedLogger; +use simplelog::{CombinedLogger, SharedLogger}; use std::io::Write; use crate::logger::{get_group_event, GroupEvent}; pub const CODSPEED_U8_COLOR_CODE: u8 = 208; // #FF8700 -const BLACK_U8_COLOR_CODE: u8 = 16; // #000 lazy_static! { pub static ref SPINNER: Arc>> = Arc::new(Mutex::new(None)); @@ -67,13 +67,11 @@ impl Log for LocalLogger { match group_event { GroupEvent::Start(name) | GroupEvent::StartOpened(name) => { println!( - " {}", - style(format!(" {} ", name.to_uppercase())) + "\n{}", + style(format!("►►► {} ", name)) .bold() - .color256(BLACK_U8_COLOR_CODE) - .on_color256(CODSPEED_U8_COLOR_CODE) + .color256(CODSPEED_U8_COLOR_CODE) ); - println!(); if *IS_TTY { let spinner = ProgressBar::new_spinner(); @@ -99,10 +97,8 @@ impl Log for LocalLogger { let mut spinner = SPINNER.lock().unwrap(); if let Some(spinner) = spinner.as_mut() { spinner.finish_and_clear(); - println!(); } } - println!(); } } @@ -158,6 +154,12 @@ pub fn get_local_logger() -> Box { Box::new(LocalLogger::new()) } +pub fn init_local_logger() -> Result<()> { + let logger = get_local_logger(); + CombinedLogger::init(vec![logger])?; + Ok(()) +} + pub fn clean_logger() { let mut spinner = SPINNER.lock().unwrap(); if let Some(spinner) = spinner.as_mut() { diff --git a/src/main.rs b/src/main.rs index ff7686a6..ddc57bef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ mod logger; mod prelude; mod request_client; mod run; +mod setup; use console::style; use lazy_static::lazy_static; diff --git a/src/run/check_system.rs b/src/run/check_system.rs index 9b4cca17..75b4256b 100644 --- a/src/run/check_system.rs +++ b/src/run/check_system.rs @@ -130,12 +130,18 @@ pub fn check_system(system_info: &SystemInfo) -> Result<()> { match system_info.arch.as_str() { "x86_64" | "aarch64" => { - warn!("Unsupported system: {:?}", system_info); - warn!("Continuing with best effort support"); + warn!( + "Unofficially supported system: {} {}. Continuing with best effort support.", + system_info.os, system_info.os_version + ); return Ok(()); } _ => {} } - bail!("Unsupported system: {:?}", system_info); + bail!( + "Unsupported system: {} {}", + system_info.os, + system_info.os_version + ); } diff --git a/src/run/runner/valgrind/helpers/download_file.rs b/src/run/helpers/download_file.rs similarity index 100% rename from src/run/runner/valgrind/helpers/download_file.rs rename to src/run/helpers/download_file.rs diff --git a/src/run/helpers/mod.rs b/src/run/helpers/mod.rs index 868c6b99..cc26c283 100644 --- a/src/run/helpers/mod.rs +++ b/src/run/helpers/mod.rs @@ -1,7 +1,9 @@ +mod download_file; mod find_repository_root; mod get_env_var; mod parse_git_remote; +pub use download_file::download_file; pub use find_repository_root::find_repository_root; pub use get_env_var::get_env_variable; pub use parse_git_remote::*; diff --git a/src/run/instruments/mongo_tracer.rs b/src/run/instruments/mongo_tracer.rs index 7783a1d8..298b1b9b 100644 --- a/src/run/instruments/mongo_tracer.rs +++ b/src/run/instruments/mongo_tracer.rs @@ -1,4 +1,5 @@ use std::{ + env, io::Read, path::{Path, PathBuf}, process::{Child, Command, Stdio}, @@ -10,8 +11,8 @@ use reqwest::Client; use tokio::fs; use url::Url; -use crate::prelude::*; -use crate::run::helpers::get_env_variable; +use crate::{prelude::*, run::helpers::download_file}; +use crate::{run::helpers::get_env_variable, MONGODB_TRACER_VERSION}; use super::MongoDBConfig; @@ -228,6 +229,31 @@ impl MongoTracer { } } +pub async fn install_mongodb_tracer() -> Result<()> { + debug!("Installing mongodb-tracer"); + // TODO: release the tracer and update this url + let installer_url = format!("https://codspeed-public-assets.s3.eu-west-1.amazonaws.com/mongo-tracer/{MONGODB_TRACER_VERSION}/cs-mongo-tracer-installer.sh"); + let installer_path = env::temp_dir().join("cs-mongo-tracer-installer.sh"); + download_file( + &Url::parse(installer_url.as_str()).unwrap(), + &installer_path, + ) + .await?; + + let output = Command::new("bash") + .arg(installer_path.to_str().unwrap()) + .stdout(Stdio::piped()) + .output() + .map_err(|_| anyhow!("Failed to install mongo-tracer"))?; + + if !output.status.success() { + info!("stdout: {}", String::from_utf8_lossy(&output.stdout)); + error!("stderr: {}", String::from_utf8_lossy(&output.stderr)); + bail!("Failed to install mongo-tracer"); + } + Ok(()) +} + #[cfg(test)] mod tests { use std::ffi::OsStr; diff --git a/src/run/mod.rs b/src/run/mod.rs index 8baa52ac..960524c0 100644 --- a/src/run/mod.rs +++ b/src/run/mod.rs @@ -5,16 +5,16 @@ use crate::run::{config::Config, logger::Logger}; use crate::VERSION; use check_system::SystemInfo; use clap::Args; -use instruments::mongo_tracer::MongoTracer; +use instruments::mongo_tracer::{install_mongodb_tracer, MongoTracer}; use run_environment::interfaces::RunEnvironment; use runner::get_run_data; -mod check_system; -mod helpers; +pub mod check_system; +pub mod helpers; mod instruments; mod poll_results; pub mod run_environment; -mod runner; +pub mod runner; mod uploader; pub mod config; @@ -120,14 +120,19 @@ pub async fn run(args: RunArgs, api_client: &CodSpeedAPIClient) -> Result<()> { let mode = runner::get_mode()?; let executor = runner::get_executor_from_mode(mode); - let run_data = get_run_data()?; - if !config.skip_setup { start_group!("Preparing the environment"); - executor.setup(&config, &system_info, &run_data).await?; + executor.setup(&system_info).await?; + // TODO: refactor and move directly in the Instruments struct as a `setup` method + if config.instruments.is_mongodb_enabled() { + install_mongodb_tracer().await?; + } + info!("Environment ready"); end_group!(); } + let run_data = get_run_data()?; + start_opened_group!("Running the benchmarks"); // TODO: refactor and move directly in the Instruments struct as a `start` method diff --git a/src/run/runner/executor.rs b/src/run/runner/executor.rs index 2e27ef3c..885cca57 100644 --- a/src/run/runner/executor.rs +++ b/src/run/runner/executor.rs @@ -8,12 +8,9 @@ use async_trait::async_trait; pub trait Executor { fn name(&self) -> ExecutorName; - async fn setup( - &self, - config: &Config, - system_info: &SystemInfo, - run_data: &RunData, - ) -> Result<()>; + async fn setup(&self, _system_info: &SystemInfo) -> Result<()> { + Ok(()) + } /// Runs the executor async fn run( diff --git a/src/run/runner/mod.rs b/src/run/runner/mod.rs index ca434c45..1f7e0b50 100644 --- a/src/run/runner/mod.rs +++ b/src/run/runner/mod.rs @@ -60,6 +60,10 @@ pub fn get_executor_from_mode(mode: RunnerMode) -> Box { } } +pub fn get_all_executors() -> Vec> { + vec![Box::new(ValgrindExecutor), Box::new(WallTimeExecutor)] +} + pub fn get_run_data() -> Result { let profile_folder = create_profile_folder()?; Ok(RunData { profile_folder }) diff --git a/src/run/runner/valgrind/executor.rs b/src/run/runner/valgrind/executor.rs index 1798bfab..409fea69 100644 --- a/src/run/runner/valgrind/executor.rs +++ b/src/run/runner/valgrind/executor.rs @@ -6,7 +6,8 @@ use crate::run::runner::executor::Executor; use crate::run::runner::{ExecutorName, RunData}; use crate::run::{check_system::SystemInfo, config::Config}; -use super::{helpers::perf_maps::harvest_perf_maps, measure, setup::setup}; +use super::setup::install_valgrind; +use super::{helpers::perf_maps::harvest_perf_maps, measure}; pub struct ValgrindExecutor; @@ -16,14 +17,8 @@ impl Executor for ValgrindExecutor { ExecutorName::Valgrind } - async fn setup( - &self, - config: &Config, - system_info: &SystemInfo, - _run_data: &RunData, - ) -> Result<()> { - setup(system_info, config).await?; - + async fn setup(&self, system_info: &SystemInfo) -> Result<()> { + install_valgrind(system_info).await?; Ok(()) } diff --git a/src/run/runner/valgrind/helpers/mod.rs b/src/run/runner/valgrind/helpers/mod.rs index 9072b60a..4c80afa0 100644 --- a/src/run/runner/valgrind/helpers/mod.rs +++ b/src/run/runner/valgrind/helpers/mod.rs @@ -1,4 +1,3 @@ -pub mod download_file; pub mod ignored_objects_path; pub mod introspected_nodejs; pub mod perf_maps; diff --git a/src/run/runner/valgrind/setup.rs b/src/run/runner/valgrind/setup.rs index 684b4017..41fcd1ad 100644 --- a/src/run/runner/valgrind/setup.rs +++ b/src/run/runner/valgrind/setup.rs @@ -5,12 +5,8 @@ use std::{ use url::Url; -use super::helpers::download_file::download_file; -use crate::{prelude::*, MONGODB_TRACER_VERSION, VALGRIND_CODSPEED_VERSION}; -use crate::{ - run::{check_system::SystemInfo, config::Config}, - VALGRIND_CODSPEED_DEB_VERSION, -}; +use crate::{prelude::*, run::helpers::download_file, VALGRIND_CODSPEED_VERSION}; +use crate::{run::check_system::SystemInfo, VALGRIND_CODSPEED_DEB_VERSION}; /// Run a command with sudo if available fn run_with_sudo(command_args: &[&str]) -> Result<()> { @@ -77,15 +73,23 @@ fn is_valgrind_installed() -> bool { } let version = String::from_utf8_lossy(&version_output.stdout); - version.contains(VALGRIND_CODSPEED_VERSION.as_str()) + let result = version.contains(VALGRIND_CODSPEED_VERSION.as_str()); + if !result { + warn!( + "Valgrind is installed but the version is not the expected one. expecting {} but found installed: {}", + VALGRIND_CODSPEED_VERSION.as_str(), + version + ); + } + result } else { false } } -async fn install_valgrind(system_info: &SystemInfo) -> Result<()> { +pub async fn install_valgrind(system_info: &SystemInfo) -> Result<()> { if is_valgrind_installed() { - debug!("Valgrind is already installed with the correct version, skipping installation"); + info!("Valgrind is already installed with the correct version, skipping installation"); return Ok(()); } debug!("Installing valgrind"); @@ -106,44 +110,8 @@ async fn install_valgrind(system_info: &SystemInfo) -> Result<()> { deb_path.to_str().unwrap(), ])?; - Ok(()) -} - -async fn install_mongodb_tracer() -> Result<()> { - debug!("Installing mongodb-tracer"); - // TODO: release the tracer and update this url - let installer_url = format!("https://codspeed-public-assets.s3.eu-west-1.amazonaws.com/mongo-tracer/{MONGODB_TRACER_VERSION}/cs-mongo-tracer-installer.sh"); - let installer_path = env::temp_dir().join("cs-mongo-tracer-installer.sh"); - download_file( - &Url::parse(installer_url.as_str()).unwrap(), - &installer_path, - ) - .await?; - - let output = Command::new("bash") - .arg(installer_path.to_str().unwrap()) - .stdout(Stdio::piped()) - .output() - .map_err(|_| anyhow!("Failed to install mongo-tracer"))?; - - if !output.status.success() { - info!("stdout: {}", String::from_utf8_lossy(&output.stdout)); - error!("stderr: {}", String::from_utf8_lossy(&output.stderr)); - bail!("Failed to install mongo-tracer"); - } - - Ok(()) -} - -pub async fn setup(system_info: &SystemInfo, config: &Config) -> Result<()> { - install_valgrind(system_info).await?; - - // TODO: move into setup of the Instruments struct - if config.instruments.is_mongodb_enabled() { - install_mongodb_tracer().await?; - } + info!("Valgrind installation completed successfully"); - info!("Environment ready"); Ok(()) } diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs index f2aee6a0..6bb03520 100644 --- a/src/run/runner/wall_time/executor.rs +++ b/src/run/runner/wall_time/executor.rs @@ -19,15 +19,6 @@ impl Executor for WallTimeExecutor { ExecutorName::WallTime } - async fn setup( - &self, - _config: &Config, - _system_info: &SystemInfo, - _run_data: &RunData, - ) -> Result<()> { - Ok(()) - } - async fn run( &self, config: &Config, diff --git a/src/setup.rs b/src/setup.rs new file mode 100644 index 00000000..2257577c --- /dev/null +++ b/src/setup.rs @@ -0,0 +1,19 @@ +use crate::prelude::*; +use crate::run::check_system::SystemInfo; +use crate::run::runner::get_all_executors; + +pub async fn setup() -> Result<()> { + let system_info = SystemInfo::new()?; + let executors = get_all_executors(); + start_group!("Setting up the environment for all executors"); + for executor in executors { + info!( + "Setting up the environment for the executor: {}", + executor.name().to_string() + ); + executor.setup(&system_info).await?; + } + info!("Environment setup completed"); + end_group!(); + Ok(()) +}