diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index 4217e2919..b1c99396c 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -12,9 +12,16 @@ from typing import Dict, Optional, Tuple import pandas as pd -import psutil from rapidfuzz import fuzz, process, utils +try: + import psutil + + PSUTIL_AVAILABLE = True +except ImportError: + PSUTIL_AVAILABLE = False + psutil = None + from codecarbon.core.rapl import RAPLFile from codecarbon.core.units import Time from codecarbon.core.util import detect_cpu_model @@ -64,6 +71,9 @@ def is_rapl_available() -> bool: def is_psutil_available(): + if not PSUTIL_AVAILABLE: + logger.debug("psutil module is not available.") + return False try: nice = psutil.cpu_times().nice if nice > 0.0001: diff --git a/codecarbon/core/resource_tracker.py b/codecarbon/core/resource_tracker.py index 3ccbf6c44..2dc560833 100644 --- a/codecarbon/core/resource_tracker.py +++ b/codecarbon/core/resource_tracker.py @@ -45,6 +45,27 @@ def set_CPU_tracking(self): max_power = self.tracker._force_cpu_power else: max_power = tdp.tdp * cpu_number if tdp.tdp is not None else None + + # Check for forced constant mode first + if self.tracker._conf.get("force_mode_constant", False): + logger.info( + "Force constant mode requested - bypassing psutil and using constant CPU power" + ) + model = tdp.model + if max_power is None and self.tracker._force_cpu_power: + max_power = self.tracker._force_cpu_power + logger.debug(f"Using user input TDP for constant mode: {max_power} W") + self.cpu_tracker = "User Input TDP constant" + else: + self.cpu_tracker = "TDP constant" + logger.info(f"CPU Model on forced constant consumption mode: {model}") + self.tracker._conf["cpu_model"] = model + hardware_cpu = CPU.from_utils( + self.tracker._output_dir, "constant", model, max_power + ) + self.tracker._hardware.append(hardware_cpu) + return + if self.tracker._conf.get("force_mode_cpu_load", False) and ( tdp.tdp is not None or self.tracker._force_cpu_power is not None ): diff --git a/codecarbon/core/util.py b/codecarbon/core/util.py index ad97d6a08..1a58d719b 100644 --- a/codecarbon/core/util.py +++ b/codecarbon/core/util.py @@ -8,7 +8,14 @@ from typing import Optional, Union import cpuinfo -import psutil + +try: + import psutil + + PSUTIL_AVAILABLE = True +except ImportError: + PSUTIL_AVAILABLE = False + psutil = None from codecarbon.external.logger import logger @@ -118,6 +125,12 @@ def count_physical_cpus(): def count_cpus() -> int: + if not PSUTIL_AVAILABLE: + logger.warning("psutil not available, using fallback CPU count detection") + # Fallback to using os.cpu_count() or physical CPU count + cpu_count = os.cpu_count() + return cpu_count if cpu_count is not None else 1 + if SLURM_JOB_ID is None: return psutil.cpu_count() diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index a5e502bc5..8396bc86d 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -180,6 +180,7 @@ def __init__( force_ram_power: Optional[int] = _sentinel, pue: Optional[int] = _sentinel, force_mode_cpu_load: Optional[bool] = _sentinel, + force_mode_constant: Optional[bool] = _sentinel, allow_multiple_runs: Optional[bool] = _sentinel, ): """ @@ -237,6 +238,7 @@ def __init__( :param force_ram_power: ram power to be used instead of automatic detection. :param pue: PUE (Power Usage Effectiveness) of the datacenter. :param force_mode_cpu_load: Force the addition of a CPU in MODE_CPU_LOAD + :param force_mode_constant: Force the addition of a CPU in constant mode, bypassing psutil :param allow_multiple_runs: Allow multiple instances of codecarbon running in parallel. Defaults to False. """ @@ -288,6 +290,7 @@ def __init__( self._set_from_conf(force_ram_power, "force_ram_power", None, float) self._set_from_conf(pue, "pue", 1.0, float) self._set_from_conf(force_mode_cpu_load, "force_mode_cpu_load", False, bool) + self._set_from_conf(force_mode_constant, "force_mode_constant", False, bool) self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) diff --git a/docs/_sources/parameters.rst.txt b/docs/_sources/parameters.rst.txt index 608df66a5..e85cb695b 100644 --- a/docs/_sources/parameters.rst.txt +++ b/docs/_sources/parameters.rst.txt @@ -46,6 +46,15 @@ Input Parameters | Estimate it with ``sudo lshw -C memory -short | grep DIMM`` | to get the number of RAM slots used, then do | *RAM power in W = Number of RAM Slots * 5 Watts* + * - force_mode_cpu_load + - | Force the use of CPU load mode for measuring CPU power consumption, + | defaults to ``False``. When enabled, uses psutil to monitor CPU load + | and estimates power consumption based on TDP and current CPU usage. + * - force_mode_constant + - | Force the use of constant mode for CPU power consumption measurement, + | defaults to ``False``. When enabled, bypasses psutil completely and + | uses a constant power consumption based on CPU TDP. Useful when + | psutil overhead is significant or psutil is unavailable. * - allow_multiple_runs - | Boolean variable indicating if multiple instance of CodeCarbon | on the same machine is allowed, diff --git a/docs/edit/parameters.rst b/docs/edit/parameters.rst index 608df66a5..e85cb695b 100644 --- a/docs/edit/parameters.rst +++ b/docs/edit/parameters.rst @@ -46,6 +46,15 @@ Input Parameters | Estimate it with ``sudo lshw -C memory -short | grep DIMM`` | to get the number of RAM slots used, then do | *RAM power in W = Number of RAM Slots * 5 Watts* + * - force_mode_cpu_load + - | Force the use of CPU load mode for measuring CPU power consumption, + | defaults to ``False``. When enabled, uses psutil to monitor CPU load + | and estimates power consumption based on TDP and current CPU usage. + * - force_mode_constant + - | Force the use of constant mode for CPU power consumption measurement, + | defaults to ``False``. When enabled, bypasses psutil completely and + | uses a constant power consumption based on CPU TDP. Useful when + | psutil overhead is significant or psutil is unavailable. * - allow_multiple_runs - | Boolean variable indicating if multiple instance of CodeCarbon | on the same machine is allowed, diff --git a/tests/test_force_constant_mode.py b/tests/test_force_constant_mode.py new file mode 100644 index 000000000..4b1ca8429 --- /dev/null +++ b/tests/test_force_constant_mode.py @@ -0,0 +1,146 @@ +import os +import tempfile +import time +import unittest +from unittest import mock + +import pandas as pd + +from codecarbon.emissions_tracker import ( + EmissionsTracker, + OfflineEmissionsTracker, +) + + +def light_computation(run_time_secs: int = 1): + end_time: float = ( + time.perf_counter() + run_time_secs + ) # Run for `run_time_secs` seconds + while time.perf_counter() < end_time: + pass + + +class TestForceConstantMode(unittest.TestCase): + def setUp(self) -> None: + self.project_name = "project_TestForceConstantMode" + self.emissions_file = "emissions-test-TestForceConstantMode.csv" + self.emissions_path = tempfile.gettempdir() + self.emissions_file_path = os.path.join( + self.emissions_path, self.emissions_file + ) + if os.path.isfile(self.emissions_file_path): + os.remove(self.emissions_file_path) + + def tearDown(self) -> None: + if os.path.isfile(self.emissions_file_path): + os.remove(self.emissions_file_path) + + def test_force_constant_mode_online(self): + """Test force_mode_constant parameter with online tracker""" + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True, + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + # Check that emissions were calculated + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + + # Verify output file was created + self.verify_output_file(self.emissions_file_path) + + # Check CSV content shows constant mode + df = pd.read_csv(self.emissions_file_path) + # The cpu_power should be a constant value (not varying like in load mode) + self.assertGreater(df["cpu_power"].iloc[0], 0) + + def test_force_constant_mode_offline(self): + """Test force_mode_constant parameter with offline tracker""" + tracker = OfflineEmissionsTracker( + country_iso_code="USA", + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True, + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + self.verify_output_file(self.emissions_file_path) + + def test_force_constant_mode_with_custom_cpu_power(self): + """Test force_mode_constant with custom CPU power""" + custom_cpu_power = 200 # 200W + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True, + force_cpu_power=custom_cpu_power, + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + + # Check that the custom CPU power was used + df = pd.read_csv(self.emissions_file_path) + # CPU power should be 50% of the TDP (constant mode assumption) + expected_cpu_power = custom_cpu_power / 2 + self.assertEqual(df["cpu_power"].iloc[0], expected_cpu_power) + + @mock.patch("codecarbon.core.cpu.PSUTIL_AVAILABLE", False) + @mock.patch("codecarbon.core.util.PSUTIL_AVAILABLE", False) + def test_force_constant_mode_without_psutil(self): + """Test that force_mode_constant works when psutil is not available""" + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True, + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + self.verify_output_file(self.emissions_file_path) + + def test_force_constant_mode_takes_precedence_over_cpu_load(self): + """Test that force_mode_constant takes precedence over force_mode_cpu_load""" + tracker = EmissionsTracker( + output_dir=self.emissions_path, + output_file=self.emissions_file, + force_mode_constant=True, + force_mode_cpu_load=True, # This should be ignored + ) + tracker.start() + light_computation(run_time_secs=1) + emissions = tracker.stop() + + assert isinstance(emissions, float) + self.assertNotEqual(emissions, 0.0) + self.verify_output_file(self.emissions_file_path) + + def verify_output_file(self, file_path: str) -> None: + """Verify that the output CSV file exists and has expected structure""" + with open(file_path, "r") as f: + lines = [line.rstrip() for line in f] + assert len(lines) == 2 # Header + 1 data row + + # Check that it's a valid CSV with expected columns + df = pd.read_csv(file_path) + expected_columns = ["emissions", "cpu_power", "cpu_energy"] + for col in expected_columns: + self.assertIn(col, df.columns) + + +if __name__ == "__main__": + unittest.main()