From 9b513233c70d502f3f73881d34b0a5876fba66f0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 2 Jul 2025 08:34:16 +0000 Subject: [PATCH 1/4] Initial plan From 45df41a58040f863c2ad77da75690a6ffdbee302 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 2 Jul 2025 08:45:27 +0000 Subject: [PATCH 2/4] Add force_mode_constant parameter to bypass psutil overhead - Make psutil imports graceful in cpu.py and util.py - Add force_mode_constant parameter to EmissionsTracker - Update CPU tracking logic to prioritize force_mode_constant - Add comprehensive test suite for new functionality - Update documentation with new parameter details Co-authored-by: benoit-cty <6603048+benoit-cty@users.noreply.github.com> --- codecarbon/core/cpu.py | 11 ++- codecarbon/core/resource_tracker.py | 19 ++++ codecarbon/core/util.py | 14 ++- codecarbon/emissions_tracker.py | 3 + docs/_sources/parameters.rst.txt | 9 ++ docs/edit/parameters.rst | 9 ++ tests/test_force_constant_mode.py | 147 ++++++++++++++++++++++++++++ 7 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 tests/test_force_constant_mode.py diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index 4217e2919..84a0d64b6 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -12,9 +12,15 @@ from typing import Dict, Optional, Tuple import pandas as pd -import psutil from rapidfuzz import fuzz, process, utils +try: + import psutil + PSUTIL_AVAILABLE = True +except ImportError: + PSUTIL_AVAILABLE = False + psutil = None + from codecarbon.core.rapl import RAPLFile from codecarbon.core.units import Time from codecarbon.core.util import detect_cpu_model @@ -64,6 +70,9 @@ def is_rapl_available() -> bool: def is_psutil_available(): + if not PSUTIL_AVAILABLE: + logger.debug("psutil module is not available.") + return False try: nice = psutil.cpu_times().nice if nice > 0.0001: diff --git a/codecarbon/core/resource_tracker.py b/codecarbon/core/resource_tracker.py index 3ccbf6c44..d59253277 100644 --- a/codecarbon/core/resource_tracker.py +++ b/codecarbon/core/resource_tracker.py @@ -45,6 +45,25 @@ def set_CPU_tracking(self): max_power = self.tracker._force_cpu_power else: max_power = tdp.tdp * cpu_number if tdp.tdp is not None else None + + # Check for forced constant mode first + if self.tracker._conf.get("force_mode_constant", False): + logger.info("Force constant mode requested - bypassing psutil and using constant CPU power") + model = tdp.model + if max_power is None and self.tracker._force_cpu_power: + max_power = self.tracker._force_cpu_power + logger.debug(f"Using user input TDP for constant mode: {max_power} W") + self.cpu_tracker = "User Input TDP constant" + else: + self.cpu_tracker = "TDP constant" + logger.info(f"CPU Model on forced constant consumption mode: {model}") + self.tracker._conf["cpu_model"] = model + hardware_cpu = CPU.from_utils( + self.tracker._output_dir, "constant", model, max_power + ) + self.tracker._hardware.append(hardware_cpu) + return + if self.tracker._conf.get("force_mode_cpu_load", False) and ( tdp.tdp is not None or self.tracker._force_cpu_power is not None ): diff --git a/codecarbon/core/util.py b/codecarbon/core/util.py index ad97d6a08..237a493f4 100644 --- a/codecarbon/core/util.py +++ b/codecarbon/core/util.py @@ -8,7 +8,13 @@ from typing import Optional, Union import cpuinfo -import psutil + +try: + import psutil + PSUTIL_AVAILABLE = True +except ImportError: + PSUTIL_AVAILABLE = False + psutil = None from codecarbon.external.logger import logger @@ -118,6 +124,12 @@ def count_physical_cpus(): def count_cpus() -> int: + if not PSUTIL_AVAILABLE: + logger.warning("psutil not available, using fallback CPU count detection") + # Fallback to using os.cpu_count() or physical CPU count + cpu_count = os.cpu_count() + return cpu_count if cpu_count is not None else 1 + if SLURM_JOB_ID is None: return psutil.cpu_count() diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index a5e502bc5..8396bc86d 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -180,6 +180,7 @@ def __init__( force_ram_power: Optional[int] = _sentinel, pue: Optional[int] = _sentinel, force_mode_cpu_load: Optional[bool] = _sentinel, + force_mode_constant: Optional[bool] = _sentinel, allow_multiple_runs: Optional[bool] = _sentinel, ): """ @@ -237,6 +238,7 @@ def __init__( :param force_ram_power: ram power to be used instead of automatic detection. :param pue: PUE (Power Usage Effectiveness) of the datacenter. :param force_mode_cpu_load: Force the addition of a CPU in MODE_CPU_LOAD + :param force_mode_constant: Force the addition of a CPU in constant mode, bypassing psutil :param allow_multiple_runs: Allow multiple instances of codecarbon running in parallel. Defaults to False. """ @@ -288,6 +290,7 @@ def __init__( self._set_from_conf(force_ram_power, "force_ram_power", None, float) self._set_from_conf(pue, "pue", 1.0, float) self._set_from_conf(force_mode_cpu_load, "force_mode_cpu_load", False, bool) + self._set_from_conf(force_mode_constant, "force_mode_constant", False, bool) self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) diff --git a/docs/_sources/parameters.rst.txt b/docs/_sources/parameters.rst.txt index 608df66a5..e85cb695b 100644 --- a/docs/_sources/parameters.rst.txt +++ b/docs/_sources/parameters.rst.txt @@ -46,6 +46,15 @@ Input Parameters | Estimate it with ``sudo lshw -C memory -short | grep DIMM`` | to get the number of RAM slots used, then do | *RAM power in W = Number of RAM Slots * 5 Watts* + * - force_mode_cpu_load + - | Force the use of CPU load mode for measuring CPU power consumption, + | defaults to ``False``. When enabled, uses psutil to monitor CPU load + | and estimates power consumption based on TDP and current CPU usage. + * - force_mode_constant + - | Force the use of constant mode for CPU power consumption measurement, + | defaults to ``False``. When enabled, bypasses psutil completely and + | uses a constant power consumption based on CPU TDP. Useful when + | psutil overhead is significant or psutil is unavailable. * - allow_multiple_runs - | Boolean variable indicating if multiple instance of CodeCarbon | on the same machine is allowed, diff --git a/docs/edit/parameters.rst b/docs/edit/parameters.rst index 608df66a5..e85cb695b 100644 --- a/docs/edit/parameters.rst +++ b/docs/edit/parameters.rst @@ -46,6 +46,15 @@ Input Parameters | Estimate it with ``sudo lshw -C memory -short | grep DIMM`` | to get the number of RAM slots used, then do | *RAM power in W = Number of RAM Slots * 5 Watts* + * - force_mode_cpu_load + - | Force the use of CPU load mode for measuring CPU power consumption, + | defaults to ``False``. When enabled, uses psutil to monitor CPU load + | and estimates power consumption based on TDP and current CPU usage. + * - force_mode_constant + - | Force the use of constant mode for CPU power consumption measurement, + | defaults to ``False``. When enabled, bypasses psutil completely and + | uses a constant power consumption based on CPU TDP. Useful when + | psutil overhead is significant or psutil is unavailable. * - allow_multiple_runs - | Boolean variable indicating if multiple instance of CodeCarbon | on the same machine is allowed, diff --git a/tests/test_force_constant_mode.py b/tests/test_force_constant_mode.py new file mode 100644 index 000000000..ef46fecbd --- /dev/null +++ b/tests/test_force_constant_mode.py @@ -0,0 +1,147 @@ +import os +import tempfile +import time +import unittest +from unittest import mock + +import pandas as pd + +from codecarbon.core import cpu +from codecarbon.emissions_tracker import ( + EmissionsTracker, + OfflineEmissionsTracker, +) + + +def light_computation(run_time_secs: int = 1): + end_time: float = ( + time.perf_counter() + run_time_secs + ) # Run for `run_time_secs` seconds + while time.perf_counter() < end_time: + pass + + +class TestForceConstantMode(unittest.TestCase): + def setUp(self) -> None: + self.project_name = "project_TestForceConstantMode" + self.emissions_file = "emissions-test-TestForceConstantMode.csv" + self.emissions_path = tempfile.gettempdir() + self.emissions_file_path = os.path.join( + self.emissions_path, self.emissions_file + ) + if os.path.isfile(self.emissions_file_path): + os.remove(self.emissions_file_path) + + def tearDown(self) -> None: + if os.path.isfile(self.emissions_file_path): + os.remove(self.emissions_file_path) + + def test_force_constant_mode_online(self): + """Test force_mode_constant parameter with online tracker""" + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + # Check that emissions were calculated + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + + # Verify output file was created + self.verify_output_file(self.emissions_file_path) + + # Check CSV content shows constant mode + df = pd.read_csv(self.emissions_file_path) + # The cpu_power should be a constant value (not varying like in load mode) + self.assertGreater(df["cpu_power"].iloc[0], 0) + + def test_force_constant_mode_offline(self): + """Test force_mode_constant parameter with offline tracker""" + tracker = OfflineEmissionsTracker( + country_iso_code="USA", + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + self.verify_output_file(self.emissions_file_path) + + def test_force_constant_mode_with_custom_cpu_power(self): + """Test force_mode_constant with custom CPU power""" + custom_cpu_power = 200 # 200W + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True, + force_cpu_power=custom_cpu_power + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + + # Check that the custom CPU power was used + df = pd.read_csv(self.emissions_file_path) + # CPU power should be 50% of the TDP (constant mode assumption) + expected_cpu_power = custom_cpu_power / 2 + self.assertEqual(df["cpu_power"].iloc[0], expected_cpu_power) + + @mock.patch("codecarbon.core.cpu.PSUTIL_AVAILABLE", False) + @mock.patch("codecarbon.core.util.PSUTIL_AVAILABLE", False) + def test_force_constant_mode_without_psutil(self): + """Test that force_mode_constant works when psutil is not available""" + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + self.verify_output_file(self.emissions_file_path) + + def test_force_constant_mode_takes_precedence_over_cpu_load(self): + """Test that force_mode_constant takes precedence over force_mode_cpu_load""" + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True, + force_mode_cpu_load=True # This should be ignored + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + self.verify_output_file(self.emissions_file_path) + + def verify_output_file(self, file_path: str) -> None: + """Verify that the output CSV file exists and has expected structure""" + with open(file_path, "r") as f: + lines = [line.rstrip() for line in f] + assert len(lines) == 2 # Header + 1 data row + + # Check that it's a valid CSV with expected columns + df = pd.read_csv(file_path) + expected_columns = ["emissions", "cpu_power", "cpu_energy"] + for col in expected_columns: + self.assertIn(col, df.columns) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 2b49f4cbd33962ea0a7c2e57f04015e622c69437 Mon Sep 17 00:00:00 2001 From: benoit-cty <4-benoit-cty@users.noreply.git.leximpact.dev> Date: Wed, 24 Sep 2025 19:43:24 +0200 Subject: [PATCH 3/4] Lint --- tests/test_force_constant_mode.py | 33 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/test_force_constant_mode.py b/tests/test_force_constant_mode.py index ef46fecbd..4b1ca8429 100644 --- a/tests/test_force_constant_mode.py +++ b/tests/test_force_constant_mode.py @@ -6,7 +6,6 @@ import pandas as pd -from codecarbon.core import cpu from codecarbon.emissions_tracker import ( EmissionsTracker, OfflineEmissionsTracker, @@ -39,21 +38,21 @@ def tearDown(self) -> None: def test_force_constant_mode_online(self): """Test force_mode_constant parameter with online tracker""" tracker = EmissionsTracker( - output_dir=self.emissions_path, + output_dir=self.emissions_path, output_file=self.emissions_file, - force_mode_constant=True + force_mode_constant=True, ) tracker.start() light_computation(run_time_secs=1) emissions = tracker.stop() - + # Check that emissions were calculated assert isinstance(emissions, float) self.assertNotEqual(emissions, 0.0) - + # Verify output file was created self.verify_output_file(self.emissions_file_path) - + # Check CSV content shows constant mode df = pd.read_csv(self.emissions_file_path) # The cpu_power should be a constant value (not varying like in load mode) @@ -65,12 +64,12 @@ def test_force_constant_mode_offline(self): country_iso_code="USA", output_dir=self.emissions_path, output_file=self.emissions_file, - force_mode_constant=True + force_mode_constant=True, ) tracker.start() light_computation(run_time_secs=1) emissions = tracker.stop() - + assert isinstance(emissions, float) self.assertNotEqual(emissions, 0.0) self.verify_output_file(self.emissions_file_path) @@ -82,15 +81,15 @@ def test_force_constant_mode_with_custom_cpu_power(self): output_dir=self.emissions_path, output_file=self.emissions_file, force_mode_constant=True, - force_cpu_power=custom_cpu_power + force_cpu_power=custom_cpu_power, ) tracker.start() light_computation(run_time_secs=1) emissions = tracker.stop() - + assert isinstance(emissions, float) self.assertNotEqual(emissions, 0.0) - + # Check that the custom CPU power was used df = pd.read_csv(self.emissions_file_path) # CPU power should be 50% of the TDP (constant mode assumption) @@ -104,12 +103,12 @@ def test_force_constant_mode_without_psutil(self): tracker = EmissionsTracker( output_dir=self.emissions_path, output_file=self.emissions_file, - force_mode_constant=True + force_mode_constant=True, ) tracker.start() light_computation(run_time_secs=1) emissions = tracker.stop() - + assert isinstance(emissions, float) self.assertNotEqual(emissions, 0.0) self.verify_output_file(self.emissions_file_path) @@ -120,12 +119,12 @@ def test_force_constant_mode_takes_precedence_over_cpu_load(self): output_dir=self.emissions_path, output_file=self.emissions_file, force_mode_constant=True, - force_mode_cpu_load=True # This should be ignored + force_mode_cpu_load=True, # This should be ignored ) tracker.start() light_computation(run_time_secs=1) emissions = tracker.stop() - + assert isinstance(emissions, float) self.assertNotEqual(emissions, 0.0) self.verify_output_file(self.emissions_file_path) @@ -135,7 +134,7 @@ def verify_output_file(self, file_path: str) -> None: with open(file_path, "r") as f: lines = [line.rstrip() for line in f] assert len(lines) == 2 # Header + 1 data row - + # Check that it's a valid CSV with expected columns df = pd.read_csv(file_path) expected_columns = ["emissions", "cpu_power", "cpu_energy"] @@ -144,4 +143,4 @@ def verify_output_file(self, file_path: str) -> None: if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 470062ef4f264cd72d7637bb5648fe35c96c2189 Mon Sep 17 00:00:00 2001 From: benoit-cty <4-benoit-cty@users.noreply.git.leximpact.dev> Date: Wed, 24 Sep 2025 19:45:53 +0200 Subject: [PATCH 4/4] Lint --- codecarbon/core/cpu.py | 1 + codecarbon/core/resource_tracker.py | 8 +++++--- codecarbon/core/util.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index 84a0d64b6..b1c99396c 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -16,6 +16,7 @@ try: import psutil + PSUTIL_AVAILABLE = True except ImportError: PSUTIL_AVAILABLE = False diff --git a/codecarbon/core/resource_tracker.py b/codecarbon/core/resource_tracker.py index d59253277..2dc560833 100644 --- a/codecarbon/core/resource_tracker.py +++ b/codecarbon/core/resource_tracker.py @@ -45,10 +45,12 @@ def set_CPU_tracking(self): max_power = self.tracker._force_cpu_power else: max_power = tdp.tdp * cpu_number if tdp.tdp is not None else None - + # Check for forced constant mode first if self.tracker._conf.get("force_mode_constant", False): - logger.info("Force constant mode requested - bypassing psutil and using constant CPU power") + logger.info( + "Force constant mode requested - bypassing psutil and using constant CPU power" + ) model = tdp.model if max_power is None and self.tracker._force_cpu_power: max_power = self.tracker._force_cpu_power @@ -63,7 +65,7 @@ def set_CPU_tracking(self): ) self.tracker._hardware.append(hardware_cpu) return - + if self.tracker._conf.get("force_mode_cpu_load", False) and ( tdp.tdp is not None or self.tracker._force_cpu_power is not None ): diff --git a/codecarbon/core/util.py b/codecarbon/core/util.py index 237a493f4..1a58d719b 100644 --- a/codecarbon/core/util.py +++ b/codecarbon/core/util.py @@ -11,6 +11,7 @@ try: import psutil + PSUTIL_AVAILABLE = True except ImportError: PSUTIL_AVAILABLE = False @@ -129,7 +130,7 @@ def count_cpus() -> int: # Fallback to using os.cpu_count() or physical CPU count cpu_count = os.cpu_count() return cpu_count if cpu_count is not None else 1 - + if SLURM_JOB_ID is None: return psutil.cpu_count()