fix: runner doesnt work with in GitHub action

Cargo.toml

@@ -62,6 +62,7 @@ procfs = "0.17.0"
 temp-env = { version = "0.3.6", features = ["async_closure"] }
 insta = { version = "1.29.0", features = ["json", "redactions"] }
 test-with = { version = "0.15", default-features = false, features = [] }
+rstest = { version = "0.25.0", default-features = false }
 
 [workspace.metadata.release]
 sign-tag = true

src/run/runner/helpers/env.rs

@@ -17,3 +17,12 @@ pub fn get_base_injected_env(
         ),
     ])
 }
+
+pub fn is_codspeed_debug_enabled() -> bool {
+    let log_level = std::env::var("CODSPEED_LOG")
+        .ok()
+        .and_then(|log_level| log_level.parse::<log::LevelFilter>().ok())
+        .unwrap_or(log::LevelFilter::Info);
+
+    log_level < log::LevelFilter::Debug
+}

src/run/runner/mod.rs

@@ -7,6 +7,8 @@ use super::{RunnerMode, config::Config};
 mod executor;
 mod helpers;
 mod interfaces;
+#[cfg(test)]
+mod tests;
 mod valgrind;
 mod wall_time;
 

src/run/runner/tests.rs

@@ -0,0 +1,187 @@
+use crate::run::check_system::SystemInfo;
+use crate::run::config::Config;
+use crate::run::runner::executor::Executor;
+use crate::run::runner::interfaces::RunData;
+use crate::run::runner::valgrind::executor::ValgrindExecutor;
+use crate::run::{RunnerMode, runner::wall_time::executor::WallTimeExecutor};
+use tempfile::TempDir;
+use tokio::sync::{OnceCell, Semaphore, SemaphorePermit};
+
+const SIMPLE_ECHO_SCRIPT: &str = "echo 'Hello, World!'";
+const MULTILINE_ECHO_SCRIPT: &str = "echo \"Working\"
+echo \"with\"
+echo \"multiple lines\"";
+const MULTILINE_ECHO_WITH_SEMICOLONS: &str = "echo \"Working\";
+echo \"with\";
+echo \"multiple lines\";";
+const DIRECTORY_CHECK_SCRIPT: &str = "cd /tmp
+# Check that the directory is actually changed
+if [ $(basename $(pwd)) != \"tmp\" ]; then
+  exit 1
+fi";
+const ENV_VAR_VALIDATION_SCRIPT: &str = "
+output=$(echo \"$MY_ENV_VAR\")
+if [ \"$output\" != \"Hello\" ]; then
+  echo \"Assertion failed: Expected 'Hello' but got '$output'\"
+  exit 1
+fi";
+
+const TESTS: [&str; 5] = [
+    SIMPLE_ECHO_SCRIPT,
+    MULTILINE_ECHO_SCRIPT,
+    MULTILINE_ECHO_WITH_SEMICOLONS,
+    DIRECTORY_CHECK_SCRIPT,
+    ENV_VAR_VALIDATION_SCRIPT,
+];
+
+async fn create_test_setup() -> (SystemInfo, RunData, TempDir) {
+    let system_info = SystemInfo::new().unwrap();
+
+    let temp_dir = TempDir::new().unwrap();
+    let run_data = RunData {
+        profile_folder: temp_dir.path().to_path_buf(),
+    };
+    (system_info, run_data, temp_dir)
+}
+
+mod valgrind {
+    use super::*;
+
+    async fn get_valgrind_executor() -> &'static ValgrindExecutor {
+        static VALGRIND_EXECUTOR: OnceCell<ValgrindExecutor> = OnceCell::const_new();
+
+        VALGRIND_EXECUTOR
+            .get_or_init(|| async {
+                let executor = ValgrindExecutor;
+                let system_info = SystemInfo::new().unwrap();
+                executor.setup(&system_info).await.unwrap();
+                executor
+            })
+            .await
+    }
+
+    fn valgrind_config(command: &str) -> Config {
+        Config {
+            mode: RunnerMode::Instrumentation,
+            command: command.to_string(),
+            ..Config::test()
+        }
+    }
+
+    #[rstest::rstest]
+    #[case(TESTS[0])]
+    #[case(TESTS[1])]
+    #[case(TESTS[2])]
+    #[case(TESTS[3])]
+    #[tokio::test]
+    async fn test_valgrind_executor(#[case] cmd: &str) {
+        let (system_info, run_data, _temp_dir) = create_test_setup().await;
+        let executor = get_valgrind_executor().await;
+
+        let config = valgrind_config(cmd);
+        executor
+            .run(&config, &system_info, &run_data, &None)
+            .await
+            .unwrap();
+    }
+
+    #[rstest::rstest]
+    #[case("MY_ENV_VAR", "Hello", ENV_VAR_VALIDATION_SCRIPT)]
+    #[tokio::test]
+    async fn test_valgrind_executor_with_env(
+        #[case] env_var: &str,
+        #[case] env_value: &str,
+        #[case] cmd: &str,
+    ) {
+        let (system_info, run_data, _temp_dir) = create_test_setup().await;
+        let executor = get_valgrind_executor().await;
+
+        temp_env::async_with_vars(&[(env_var, Some(env_value))], async {
+            let config = valgrind_config(cmd);
+            executor
+                .run(&config, &system_info, &run_data, &None)
+                .await
+                .unwrap();
+        })
+        .await;
+    }
+}
+
+mod walltime {
+    use super::*;
+
+    async fn get_walltime_executor() -> (SemaphorePermit<'static>, WallTimeExecutor) {
+        static WALLTIME_INIT: OnceCell<()> = OnceCell::const_new();
+        static WALLTIME_SEMAPHORE: OnceCell<Semaphore> = OnceCell::const_new();
+
+        WALLTIME_INIT
+            .get_or_init(|| async {
+                let executor = WallTimeExecutor::new();
+                let system_info = SystemInfo::new().unwrap();
+                executor.setup(&system_info).await.unwrap();
+            })
+            .await;
+
+        // We can't execute multiple walltime executors in parallel because perf isn't thread-safe (yet). We have to
+        // use a semaphore to limit concurrent access.
+        let semaphore = WALLTIME_SEMAPHORE
+            .get_or_init(|| async { Semaphore::new(1) })
+            .await;
+        let permit = semaphore.acquire().await.unwrap();
+
+        (permit, WallTimeExecutor::new())
+    }
+
+    fn walltime_config(command: &str, enable_perf: bool) -> Config {
+        Config {
+            mode: RunnerMode::Walltime,
+            command: command.to_string(),
+            enable_perf,
+            ..Config::test()
+        }
+    }
+
+    #[rstest::rstest]
+    #[case(TESTS[0], false)]
+    #[case(TESTS[0], true)]
+    #[case(TESTS[1], false)]
+    #[case(TESTS[1], true)]
+    #[case(TESTS[2], false)]
+    #[case(TESTS[2], true)]
+    #[case(TESTS[3], false)]
+    #[case(TESTS[3], true)]
+    #[tokio::test]
+    async fn test_walltime_executor(#[case] cmd: &str, #[case] enable_perf: bool) {
+        let (system_info, run_data, _temp_dir) = create_test_setup().await;
+        let (_permit, executor) = get_walltime_executor().await;
+
+        let config = walltime_config(cmd, enable_perf);
+        executor
+            .run(&config, &system_info, &run_data, &None)
+            .await
+            .unwrap();
+    }
+
+    #[rstest::rstest]
+    #[case("MY_ENV_VAR", "Hello", ENV_VAR_VALIDATION_SCRIPT, false)]
+    #[case("MY_ENV_VAR", "Hello", ENV_VAR_VALIDATION_SCRIPT, true)]
+    #[tokio::test]
+    async fn test_walltime_executor_with_env(
+        #[case] env_var: &str,
+        #[case] env_value: &str,
+        #[case] cmd: &str,
+        #[case] enable_perf: bool,
+    ) {
+        let (system_info, run_data, _temp_dir) = create_test_setup().await;
+        let (_permit, executor) = get_walltime_executor().await;
+
+        temp_env::async_with_vars(&[(env_var, Some(env_value))], async {
+            let config = walltime_config(cmd, enable_perf);
+            executor
+                .run(&config, &system_info, &run_data, &None)
+                .await
+                .unwrap();
+        })
+        .await;
+    }
+}

src/run/runner/valgrind/measure.rs

@@ -56,7 +56,7 @@ fn create_run_script() -> anyhow::Result<TempPath> {
     // 1. The command to execute
     // 2. The path to the file where the exit code will be written
     const WRAPPER_SCRIPT: &str = r#"#!/bin/sh
-           (eval $1)
+           sh -c "$1"
            status=$?
            echo -n "$status" > "$2"
             "#;
@@ -189,6 +189,10 @@ mod tests {
 
     #[test]
     fn test_run_wrapper_script() {
+        temp_env::with_var("TEST_ENV_VAR", "test_value".into(), || {
+            assert_eq!(safe_run("echo $TEST_ENV_VAR"), (0, 0));
+        });
+
         assert_eq!(safe_run("ls"), (0, 0));
         assert_eq!(safe_run("exit 0"), (0, 0));
         assert_eq!(safe_run("exit 1"), (0, 1));
@@ -201,5 +205,38 @@ mod tests {
         assert_eq!(safe_run("test 1 = 2 && exit 42"), (0, 1));
         assert_eq!(safe_run("test 1 = 1 || exit 42"), (0, 0));
         assert_eq!(safe_run("test 1 = 2 || exit 42"), (0, 42));
+
+        const MULTILINE_ECHO_SCRIPT: &str = "echo \"Working\"
+echo \"with\"
+echo \"multiple lines\"";
+
+        const MULTILINE_ECHO_WITH_SEMICOLONS: &str = "echo \"Working\";
+echo \"with\";
+echo \"multiple lines\";";
+
+        const ENV_VAR_VALIDATION_SCRIPT: &str = "export MY_ENV_VAR=\"Hello\"
+output=$(echo \"$MY_ENV_VAR\")
+if [ \"$output\" != \"Hello\" ]; then
+  echo \"Assertion failed: Expected 'Hello' but got '$output'\"
+  exit 1
+fi";
+
+        const ENV_VAR_VALIDATION_FAIL_SCRIPT: &str = "MY_ENV_VAR=\"Wrong\"
+output=$(echo \"$MY_ENV_VAR\")
+if [ \"$output\" != \"Hello\" ]; then
+  echo \"Assertion failed: Expected 'Hello' but got '$output'\"
+  exit 1
+fi";
+
+        const DIRECTORY_CHECK_SCRIPT: &str = "cd /tmp
+# Check that the directory is actually changed
+if [ $(basename $(pwd)) != \"tmp\" ]; then
+  exit 1
+fi";
+        assert_eq!(safe_run(MULTILINE_ECHO_SCRIPT), (0, 0));
+        assert_eq!(safe_run(MULTILINE_ECHO_WITH_SEMICOLONS), (0, 0));
+        assert_eq!(safe_run(DIRECTORY_CHECK_SCRIPT), (0, 0));
+        assert_eq!(safe_run(ENV_VAR_VALIDATION_SCRIPT), (0, 0));
+        assert_eq!(safe_run(ENV_VAR_VALIDATION_FAIL_SCRIPT), (0, 1));
     }
 }