From df2a91892097dc2da52cfa216e601fde7d8034e3 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 14:00:26 +0200 Subject: [PATCH 01/11] fix: find all installed python versions --- .../valgrind/helpers/ignored_objects_path.rs | 103 +++++++++++++++--- 1 file changed, 87 insertions(+), 16 deletions(-) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index df2d4471..bf9ff322 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -1,28 +1,99 @@ use crate::prelude::*; use std::{path::PathBuf, process::Command}; -fn get_python_objects() -> Vec { - let output = Command::new("python") - .arg("-c") - .arg("import sysconfig; print('/'.join(sysconfig.get_config_vars('LIBDIR', 'INSTSONAME')))") - .output(); - - if output.is_err() { - let err = output.err().unwrap().to_string(); - debug!("Failed to get python shared objects: {err}"); - return vec![]; +fn find_uv_python_paths() -> anyhow::Result> { + let output = Command::new("uv") + .args([ + "python", + "list", + "--only-installed", + "--output-format", + "json", + ]) + .output()?; + if !output.status.success() { + bail!( + "Failed to get uv python paths: {}", + String::from_utf8_lossy(&output.stderr) + ); } - let output = output.unwrap(); + + let json_output = String::from_utf8_lossy(&output.stdout); + let json: serde_json::Value = serde_json::from_str(&json_output).unwrap_or_default(); + let arr = json + .as_array() + .context("Failed to parse uv python paths: not an array")?; + let paths: Vec = arr + .iter() + .filter_map(|obj| obj.get("path")) + .filter_map(|p| p.as_str()) + .map(|s| s.to_string()) + .collect(); + Ok(paths) +} + +fn find_system_python_paths() -> anyhow::Result> { + let output = Command::new("which").args(["-a", "python"]).output()?; if !output.status.success() { - debug!( - "Failed to get python shared objects: {}", + bail!( + "Failed to get system python path: {}", String::from_utf8_lossy(&output.stderr) ); - return vec![]; } - let so_output = String::from_utf8_lossy(&output.stdout).trim().to_string(); - vec![so_output] + let paths = String::from_utf8_lossy(&output.stdout) + .lines() + .map(|line| line.trim().to_string()) + .collect(); + Ok(paths) +} + +fn find_python_paths() -> anyhow::Result> { + let uv_paths = find_uv_python_paths().unwrap_or_default(); + let system_paths = find_system_python_paths().unwrap_or_default(); + + let mut paths = uv_paths; + paths.extend(system_paths); + paths.sort(); + paths.dedup(); + Ok(paths) +} + +fn get_python_objects() -> Vec { + let mut python_objects = Vec::new(); + for path in find_python_paths().unwrap_or_default() { + // Get the parent directory of the python binary, then join with lib + let python_path = PathBuf::from(&path); + let Some(parent_dir) = python_path.parent() else { + continue; + }; + let Some(install_dir) = parent_dir.parent() else { + continue; + }; + + let lib_dir = install_dir.join("lib"); + let Ok(entries) = std::fs::read_dir(&lib_dir) else { + continue; + }; + + for entry in entries.flatten() { + let file_name = entry.file_name(); + let file_name_str = file_name.to_string_lossy(); + + // Check if filename matches libpython*.so pattern + if !file_name_str.starts_with("libpython") || !file_name_str.ends_with(".so") { + continue; + } + + let entry_path = entry.path(); + let Some(full_path) = entry_path.to_str() else { + continue; + }; + python_objects.push(full_path.to_string()); + } + } + + python_objects } fn get_node_objects() -> Vec { From 5686ef8d01e7b413102f24e68d1fd6b00be2c8d8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 14:07:43 +0200 Subject: [PATCH 02/11] fixup: remove unneeded python ignore check --- src/run/runner/wall_time/perf/mod.rs | 24 ----------------------- src/run/runner/wall_time/perf/perf_map.rs | 4 ---- 2 files changed, 28 deletions(-) diff --git a/src/run/runner/wall_time/perf/mod.rs b/src/run/runner/wall_time/perf/mod.rs index e0db17e1..d04109c4 100644 --- a/src/run/runner/wall_time/perf/mod.rs +++ b/src/run/runner/wall_time/perf/mod.rs @@ -445,30 +445,6 @@ impl BenchmarkData { } } - // When python is statically linked, we'll not find it in the ignored modules. Add it manually: - let python_modules = self.symbols_by_pid.values().filter_map(|proc| { - proc.loaded_modules().find(|path| { - path.file_name() - .map(|name| name.to_string_lossy().starts_with("python")) - .unwrap_or(false) - }) - }); - for path in python_modules { - if let Some(mapping) = self - .symbols_by_pid - .values() - .find_map(|proc| proc.module_mapping(path)) - { - let (Some((base_addr, _)), Some((_, end_addr))) = ( - mapping.iter().min_by_key(|(base_addr, _)| base_addr), - mapping.iter().max_by_key(|(_, end_addr)| end_addr), - ) else { - continue; - }; - to_ignore.push((path.to_string_lossy().into(), *base_addr, *end_addr)); - } - } - to_ignore }, }; diff --git a/src/run/runner/wall_time/perf/perf_map.rs b/src/run/runner/wall_time/perf/perf_map.rs index 9dcc0cec..34bf0785 100644 --- a/src/run/runner/wall_time/perf/perf_map.rs +++ b/src/run/runner/wall_time/perf/perf_map.rs @@ -213,10 +213,6 @@ impl ProcessSymbols { .push((start_addr, end_addr)); } - pub fn loaded_modules(&self) -> impl Iterator { - self.modules.keys() - } - pub fn module_mapping>( &self, module_path: P, From 6b4da5f53e2b7c12ece24573636dab3fa2ada67a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 14:36:21 +0200 Subject: [PATCH 03/11] chore: also ignore python (which could be statically linked) --- src/run/runner/valgrind/helpers/ignored_objects_path.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index bf9ff322..a70f377c 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -132,6 +132,7 @@ fn normalize_object_paths(objects_path_to_ignore: &mut [String]) { pub fn get_objects_path_to_ignore() -> Vec { let mut objects_path_to_ignore = vec![]; objects_path_to_ignore.extend(get_python_objects()); + objects_path_to_ignore.extend(find_python_paths().unwrap_or_default()); objects_path_to_ignore.extend(get_node_objects()); debug!("objects_path_to_ignore before normalization: {objects_path_to_ignore:?}"); normalize_object_paths(&mut objects_path_to_ignore); From b4577bac939975ed4954d8188dced0ad8762810b Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 16:27:35 +0200 Subject: [PATCH 04/11] fixup: minor changes --- src/run/runner/valgrind/helpers/ignored_objects_path.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index a70f377c..fdcb3e77 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -80,8 +80,7 @@ fn get_python_objects() -> Vec { let file_name = entry.file_name(); let file_name_str = file_name.to_string_lossy(); - // Check if filename matches libpython*.so pattern - if !file_name_str.starts_with("libpython") || !file_name_str.ends_with(".so") { + if !file_name_str.starts_with("libpython") { continue; } @@ -137,5 +136,9 @@ pub fn get_objects_path_to_ignore() -> Vec { debug!("objects_path_to_ignore before normalization: {objects_path_to_ignore:?}"); normalize_object_paths(&mut objects_path_to_ignore); debug!("objects_path_to_ignore after normalization: {objects_path_to_ignore:?}"); + + objects_path_to_ignore.sort(); + objects_path_to_ignore.dedup(); + objects_path_to_ignore } From 0a086f7dd57b6c0d15c0d78245d01aa3516a5e75 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 16:36:40 +0200 Subject: [PATCH 05/11] chore: do not normalize python paths --- src/run/runner/valgrind/helpers/ignored_objects_path.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index fdcb3e77..c2e1aaeb 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -130,13 +130,14 @@ fn normalize_object_paths(objects_path_to_ignore: &mut [String]) { pub fn get_objects_path_to_ignore() -> Vec { let mut objects_path_to_ignore = vec![]; - objects_path_to_ignore.extend(get_python_objects()); - objects_path_to_ignore.extend(find_python_paths().unwrap_or_default()); objects_path_to_ignore.extend(get_node_objects()); debug!("objects_path_to_ignore before normalization: {objects_path_to_ignore:?}"); normalize_object_paths(&mut objects_path_to_ignore); debug!("objects_path_to_ignore after normalization: {objects_path_to_ignore:?}"); + objects_path_to_ignore.extend(get_python_objects()); + objects_path_to_ignore.extend(find_python_paths().unwrap_or_default()); + objects_path_to_ignore.sort(); objects_path_to_ignore.dedup(); From 9d77cd4506546cd45d9d634b539d8f6d3cf1ecdf Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 16:52:12 +0200 Subject: [PATCH 06/11] fix: use cur dir --- src/run/runner/valgrind/helpers/ignored_objects_path.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index c2e1aaeb..3a4efd6c 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -10,6 +10,9 @@ fn find_uv_python_paths() -> anyhow::Result> { "--output-format", "json", ]) + // IMPORTANT: Set to the cwd, so that we also find python + // installations in virtual environments. + .current_dir(std::env::current_dir()?) .output()?; if !output.status.success() { bail!( From 14e8ae70931d148c3a62ce59bdc361d8d6e56eae Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 17:18:36 +0200 Subject: [PATCH 07/11] chore: detect venv python --- .../valgrind/helpers/ignored_objects_path.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index 3a4efd6c..246f37b8 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -1,6 +1,40 @@ use crate::prelude::*; use std::{path::PathBuf, process::Command}; +fn find_venv_python_paths() -> anyhow::Result> { + let venv_path = std::env::var("VIRTUAL_ENV").ok(); + if venv_path.is_none() { + return Ok(vec![]); + } + let venv_path = venv_path.unwrap(); + let python_path = PathBuf::from(venv_path).join("bin").join("python"); + if !python_path.exists() { + return Ok(vec![]); + } + debug!("Found venv python path: {}", python_path.to_string_lossy()); + + // 'uv python find' + let output = Command::new("uv") + .args(["python", "find", &python_path.to_string_lossy()]) + .output()?; + if !output.status.success() { + bail!( + "Failed to get venv python path: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + let python_path = PathBuf::from(String::from_utf8_lossy(&output.stdout).trim()); + if !python_path.exists() { + return Ok(vec![]); + } + debug!( + "Resolved venv python path: {}", + python_path.to_string_lossy() + ); + + Ok(vec![python_path.to_string_lossy().to_string()]) +} + fn find_uv_python_paths() -> anyhow::Result> { let output = Command::new("uv") .args([ @@ -53,10 +87,15 @@ fn find_system_python_paths() -> anyhow::Result> { fn find_python_paths() -> anyhow::Result> { let uv_paths = find_uv_python_paths().unwrap_or_default(); + debug!("uv python paths: {uv_paths:?}"); let system_paths = find_system_python_paths().unwrap_or_default(); + debug!("system python paths: {system_paths:?}"); + let venv_paths = find_venv_python_paths().unwrap_or_default(); + debug!("venv python paths: {venv_paths:?}"); let mut paths = uv_paths; paths.extend(system_paths); + paths.extend(venv_paths); paths.sort(); paths.dedup(); Ok(paths) @@ -143,6 +182,7 @@ pub fn get_objects_path_to_ignore() -> Vec { objects_path_to_ignore.sort(); objects_path_to_ignore.dedup(); + debug!("Final objects_path_to_ignore: {objects_path_to_ignore:#?}"); objects_path_to_ignore } From 2bcf7bffb57ef9caa46e08fe16d46aaf45d3f4a1 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 17:20:00 +0200 Subject: [PATCH 08/11] fixup: normalize all paths --- src/run/runner/valgrind/helpers/ignored_objects_path.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index 246f37b8..a6dfc074 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -173,16 +173,15 @@ fn normalize_object_paths(objects_path_to_ignore: &mut [String]) { pub fn get_objects_path_to_ignore() -> Vec { let mut objects_path_to_ignore = vec![]; objects_path_to_ignore.extend(get_node_objects()); - debug!("objects_path_to_ignore before normalization: {objects_path_to_ignore:?}"); - normalize_object_paths(&mut objects_path_to_ignore); - debug!("objects_path_to_ignore after normalization: {objects_path_to_ignore:?}"); - objects_path_to_ignore.extend(get_python_objects()); objects_path_to_ignore.extend(find_python_paths().unwrap_or_default()); objects_path_to_ignore.sort(); objects_path_to_ignore.dedup(); - debug!("Final objects_path_to_ignore: {objects_path_to_ignore:#?}"); + + debug!("objects_path_to_ignore before normalization: {objects_path_to_ignore:#?}"); + normalize_object_paths(&mut objects_path_to_ignore); + debug!("objects_path_to_ignore after normalization: {objects_path_to_ignore:#?}"); objects_path_to_ignore } From 79d3a33d43e5be856cdf373f5d64a6e3b910614c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 17:42:57 +0200 Subject: [PATCH 09/11] fix: venv check --- .../valgrind/helpers/ignored_objects_path.rs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index a6dfc074..5fa93434 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -2,20 +2,9 @@ use crate::prelude::*; use std::{path::PathBuf, process::Command}; fn find_venv_python_paths() -> anyhow::Result> { - let venv_path = std::env::var("VIRTUAL_ENV").ok(); - if venv_path.is_none() { - return Ok(vec![]); - } - let venv_path = venv_path.unwrap(); - let python_path = PathBuf::from(venv_path).join("bin").join("python"); - if !python_path.exists() { - return Ok(vec![]); - } - debug!("Found venv python path: {}", python_path.to_string_lossy()); - - // 'uv python find' let output = Command::new("uv") - .args(["python", "find", &python_path.to_string_lossy()]) + .args(["python", "find"]) + .current_dir(std::env::current_dir()?) .output()?; if !output.status.success() { bail!( @@ -93,6 +82,8 @@ fn find_python_paths() -> anyhow::Result> { let venv_paths = find_venv_python_paths().unwrap_or_default(); debug!("venv python paths: {venv_paths:?}"); + // For each python path, look at the folder to possibly identify more python versions + let mut paths = uv_paths; paths.extend(system_paths); paths.extend(venv_paths); From 7d102ed6ead5356443a1070110ef96fe1153631a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 18:09:51 +0200 Subject: [PATCH 10/11] fix: venv compat script --- src/run/runner/valgrind/helpers/venv_compat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/run/runner/valgrind/helpers/venv_compat.sh b/src/run/runner/valgrind/helpers/venv_compat.sh index fb5ac21e..533a17e7 100755 --- a/src/run/runner/valgrind/helpers/venv_compat.sh +++ b/src/run/runner/valgrind/helpers/venv_compat.sh @@ -14,7 +14,7 @@ function add_symlink() { echo "Python installation (system): $system_path" echo "Python installation (venv): $venv_path" - libpython_name="$(ldd "$venv_python" 2>/dev/null | grep -o -m1 'libpython[^[:space:]]*' || true)" + libpython_name="$(ldd "$venv_python" 2>/dev/null | grep -o 'libpython[^[:space:]]*' | head -1 | tr -d '\n' || true)" if [ -z "$libpython_name" ]; then echo "Error: exact libpython name not found in $(ldd $venv_python)" >&2 return 0 From 0e1f6511c84a8bb63157d0849826833ec2e0b7a7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 15 Sep 2025 19:38:06 +0200 Subject: [PATCH 11/11] fixup: dedup normalized objects --- src/run/runner/valgrind/helpers/ignored_objects_path.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/run/runner/valgrind/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs index 5fa93434..f18e4a40 100644 --- a/src/run/runner/valgrind/helpers/ignored_objects_path.rs +++ b/src/run/runner/valgrind/helpers/ignored_objects_path.rs @@ -167,11 +167,10 @@ pub fn get_objects_path_to_ignore() -> Vec { objects_path_to_ignore.extend(get_python_objects()); objects_path_to_ignore.extend(find_python_paths().unwrap_or_default()); - objects_path_to_ignore.sort(); - objects_path_to_ignore.dedup(); - debug!("objects_path_to_ignore before normalization: {objects_path_to_ignore:#?}"); normalize_object_paths(&mut objects_path_to_ignore); + objects_path_to_ignore.sort(); + objects_path_to_ignore.dedup(); debug!("objects_path_to_ignore after normalization: {objects_path_to_ignore:#?}"); objects_path_to_ignore