From 4d124c2ea39ab12dd4b1d0ac903c00a2739d6455 Mon Sep 17 00:00:00 2001 From: Pavel Abramov Date: Tue, 27 Jan 2026 12:32:42 +0100 Subject: [PATCH 1/2] Add BIOS scraping from Redfish Signed-off-by: Pavel Abramov --- conf/config.yaml | 16 +++ main.py | 6 + src/bios_settings.py | 260 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 src/bios_settings.py diff --git a/conf/config.yaml b/conf/config.yaml index 3669971..4c167ba 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -1,6 +1,22 @@ benchmark_output_path: "${hydra:run.dir}/output.csv" sysinfo_collector_file: "${hydra:run.dir}/sysinfo.json" +# --- Configuration for getting BIOS data from redfish + +bios: + enable: true + redfish: + host: "0.0.0.0" + username: "root" + password: "SECRET_PASSWORD" + verify_ssl: false + timeout: 15 + + output: + format: "json" + file: "${hydra:run.dir}/bios.json" + pretty: true + # --- Intel CAT specific configuration --- pqos: diff --git a/main.py b/main.py index 41f2b61..485f0ac 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,8 @@ from omegaconf import DictConfig, OmegaConf +from src.bios_settings import process_bios_settings + from src.metrics import ( CPUmonitor, InterruptMonitor, @@ -91,6 +93,10 @@ def main(cfg: DictConfig): collector.gather_all(cfg) collector.dump_to_file(cfg.sysinfo_collector_file) + # Collect BIOS settings via redfish + if cfg.bios.enable: + process_bios_settings(cfg.bios) + runner = DockerTestRunner(cfg) if cfg.run.command == "build": diff --git a/src/bios_settings.py b/src/bios_settings.py new file mode 100644 index 0000000..7753cf2 --- /dev/null +++ b/src/bios_settings.py @@ -0,0 +1,260 @@ +import argparse +import urllib3 +import json +import re +import sys + +from omegaconf import DictConfig +from pathlib import Path +from typing import Dict, Optional, Tuple + + +try: + import requests + from requests.auth import HTTPBasicAuth + + REQUESTS_AVAILABLE = True +except ImportError: + print( + "ERROR: requests library not found. Install with: pip install requests", + file=sys.stderr, + ) + sys.exit(1) + +try: + import yaml + + YAML_AVAILABLE = True +except ImportError: + YAML_AVAILABLE = False + +# Disable SSL warnings for self-signed iDRAC certificates +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +import urllib3 + + +def process_bios_settings(cfg: DictConfig): + """ + Fetches BIOS settings using parameters from a Hydra config object. + + Expected Config Structure (example): + redfish: + enabled: true + host: "1.2.3.4" + username: "root" + password: "password" + verify_ssl: false + timeout: 10 + output: + format: "text" # json, yaml, text + file: "bios.json" # or null for stdout + pretty: true + """ + + # 1. Validation & Setup + # Access keys safely. Using .get() allows for defaults if keys are missing in YAML. + # We assume 'redfish' and 'output' are root keys in the passed cfg, + # or you can pass cfg.bios_task to this function. + rf_cfg = cfg.get("redfish", {}) + out_cfg = cfg.get("output", {}) + + if not rf_cfg.get("enabled", True): + print("WARNING: Redfish task is disabled in config", file=sys.stderr) + return + + host = rf_cfg.get("host") + username = rf_cfg.get("username", "root") + password = rf_cfg.get("password") + verify_ssl = rf_cfg.get("verify_ssl", False) + timeout = rf_cfg.get("timeout", 10) + + if not host: + print("ERROR: No host specified in config (redfish.host)", file=sys.stderr) + raise ValueError("Host Missing") + + if not password: + raise ValueError("ERROR: No password specified in config (redfish.password)") + + session, host_url = connect_redfish(host, username, password, verify_ssl, timeout) + if not session: + raise ValueError("Session missing") + + attributes = get_bios_attributes(session, host_url, timeout) + if not attributes: + raise ValueError("Attributes missing") + + output_fmt = out_cfg.get("format", "text") + is_pretty = out_cfg.get("pretty", True) + + if output_fmt == "json": + output_data = format_json(attributes, pretty=is_pretty) + elif output_fmt == "yaml": + output_data = format_yaml(attributes) + else: + output_data = format_text(attributes) + + output_file = out_cfg.get("file") + + if output_file: + try: + with open(output_file, "w") as f: + f.write(output_data) + print(f"✓ Saved to {output_file}", file=sys.stderr) + except Exception as e: + raise ValueError(f"ERROR: Failed to write to {output_file}: {e}") + else: + # Print to stdout + print(output_data) + + +def connect_redfish( + host: str, username: str, password: str, verify_ssl: bool = False, timeout: int = 10 +) -> Tuple[Optional[requests.Session], str]: + """ + Create authenticated Redfish session to iDRAC. + + Returns: + Tuple of (authenticated session or None, normalized host URL) + """ + # Ensure host has https:// prefix + if not host.startswith("http://") and not host.startswith("https://"): + host = f"https://{host}" + + session = requests.Session() + session.auth = HTTPBasicAuth(username, password) + session.verify = verify_ssl + session.headers.update( + {"Content-Type": "application/json", "Accept": "application/json"} + ) + + # Test connection with root endpoint + try: + print(f"Connecting to {host}...", file=sys.stderr) + response = session.get(f"{host}/redfish/v1/", timeout=timeout) + response.raise_for_status() + print(f"✓ Connected successfully", file=sys.stderr) + return session, host + except requests.exceptions.Timeout: + print(f"ERROR: Connection timeout to {host}", file=sys.stderr) + return None, host + except requests.exceptions.ConnectionError as e: + print(f"ERROR: Connection failed to {host}: {e}", file=sys.stderr) + return None, host + except requests.exceptions.HTTPError as e: + print(f"ERROR: HTTP error from {host}: {e}", file=sys.stderr) + return None, host + except Exception as e: + print(f"ERROR: Unexpected error connecting to {host}: {e}", file=sys.stderr) + return None, host + + +def get_bios_attributes( + session: requests.Session, host: str, timeout: int = 10 +) -> Optional[Dict[str, any]]: + """ + Retrieve BIOS attributes from Dell iDRAC. + + Returns: + Dictionary of BIOS attributes or None on failure + """ + try: + # Dell iDRAC standard endpoint for BIOS settings + url = f"{host}/redfish/v1/Systems/System.Embedded.1/Bios" + print(f"Fetching BIOS attributes from {url}...", file=sys.stderr) + + response = session.get(url, timeout=timeout) + response.raise_for_status() + data = response.json() + + # BIOS attributes are in the "Attributes" key + attributes = data.get("Attributes", {}) + print(f"✓ Retrieved {len(attributes)} BIOS attributes", file=sys.stderr) + return attributes + except requests.exceptions.HTTPError as e: + print(f"ERROR: Failed to fetch BIOS attributes: {e}", file=sys.stderr) + print( + f" Response: {e.response.text if e.response else 'No response'}", + file=sys.stderr, + ) + return None + except Exception as e: + print(f"ERROR: Unexpected error fetching BIOS: {e}", file=sys.stderr) + return None + + +def format_text(attributes: Dict[str, any]) -> str: + """ + Format BIOS attributes in human-readable text format. + Groups attributes by prefix for better organization. + """ + if not attributes: + return "No BIOS attributes available" + + lines = [] + lines.append("=" * 80) + lines.append(f"BIOS SETTINGS ({len(attributes)} total attributes)") + lines.append("=" * 80) + lines.append("") + + # Group by prefix (first word before capital letter) + groups = {} + ungrouped = [] + + for key in sorted(attributes.keys()): + value = attributes[key] + # Try to extract prefix (e.g., "Proc" from "ProcTurboMode") + match = re.match(r"^([A-Z][a-z]+)", key) + if match: + prefix = match.group(1) + if prefix not in groups: + groups[prefix] = [] + groups[prefix].append((key, value)) + else: + ungrouped.append((key, value)) + + # Output grouped attributes + for prefix in sorted(groups.keys()): + lines.append(f"[{prefix}*] Settings ({len(groups[prefix])} attributes)") + lines.append("-" * 80) + for key, value in groups[prefix]: + # Format value nicely + if isinstance(value, bool): + value_str = "Enabled" if value else "Disabled" + elif isinstance(value, str) and len(value) > 60: + value_str = value[:57] + "..." + else: + value_str = str(value) + lines.append(f" {key:<40} = {value_str}") + lines.append("") + + # Output ungrouped attributes + if ungrouped: + lines.append(f"[Other] Settings ({len(ungrouped)} attributes)") + lines.append("-" * 80) + for key, value in ungrouped: + if isinstance(value, bool): + value_str = "Enabled" if value else "Disabled" + elif isinstance(value, str) and len(value) > 60: + value_str = value[:57] + "..." + else: + value_str = str(value) + lines.append(f" {key:<40} = {value_str}") + lines.append("") + + return "\n".join(lines) + + +def format_json(attributes: Dict[str, any], pretty: bool = True) -> str: + """Format BIOS attributes as JSON.""" + if pretty: + return json.dumps(attributes, indent=2, sort_keys=True) + else: + return json.dumps(attributes, sort_keys=True) + + +def format_yaml(attributes: Dict[str, any]) -> str: + """Format BIOS attributes as YAML.""" + if not YAML_AVAILABLE: + raise ValueError("ERROR: PyYAML not installed. Cannot output YAML format.") + return yaml.dump(attributes, default_flow_style=False, sort_keys=True) From 8424b6e6e8c0ca513947d877b0953037b6f6d65e Mon Sep 17 00:00:00 2001 From: Pavel Abramov Date: Tue, 27 Jan 2026 13:09:01 +0100 Subject: [PATCH 2/2] Update README Signed-off-by: Pavel Abramov --- README.md | 209 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 191 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 245f89d..0384289 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,14 @@ # rtos_bench: benchmarking suite to analyse real-time (RT) performance of an operating system -A lightweight, configurable Python framework for running system and application benchmarks using Hydra for flexible experiment management. -This repository provides a single entry-point script to run various performance tests with reproducible configurations defined in conf/config.yaml. +A comprehensive Python framework for benchmarking, analyzing and validating real-time (RT) performance of operating systems. Combines Docker-containerized benchmarks with statistical analysis tools based on Extreme Value Theory (EVT) to determine if a system meets real-time requirements. + +## Key Features + +- **Containerized Benchmarks**: Run reproducible RT benchmarks (Caterpillar, Cyclictest, iperf3, CODESYS) in Docker or on your host system +- **Intel RDT Integration**: Full support for Cache Allocation Technology (CAT) and Memory Bandwidth Allocation (MBA) +- **Statistical RT Validation**: EVT-based analysis with Region of Acceptance (RoA) for probabilistic WCET estimation +- **BIOS Collection via Redfish**: Automatically capture BIOS settings from BMC/iDRAC before benchmarks +- **Jupyter Analysis Notebooks**: Interactive reports for analyzing benchmark results and RT readiness ## Prerequisites: @@ -19,12 +26,32 @@ curl -LsSf https://astral.sh/uv/install.sh | sh uv sync ``` -## How to run benchmark +4. Additional system requirements: + - Docker (for containerized execution) + - `intel-cmt-cat` package (for pqos/Intel RDT support) + - Root access (required for pqos, IRQ affinity, and some metrics) + + + +## Quick Start ```bash -uv run main.py + +# Install dependencies and virtual environment (venv) +uv sync + +# Build all Docker images first +sudo .venv/bin/python3 main.py run.command=build + +# Run a benchmark (e.g. caterpillar) +sudo .venv/bin/python3 main.py run.command=caterpillar + +# Analyze results in Jupyter +uv run jupyter-lab ``` +> **Note**: Benchmarks require root access for pqos, IRQ affinity configuration, and hardware monitoring. Use `sudo .venv/bin/python3/main.py` instead of `uv run main.py` + ## How to run jupyter notebook (analysis software) ``` @@ -56,7 +83,6 @@ After that you can open any report and run it, just double-click on it like here ├── iperf3/ ├── mega-benchmark/ ├── codesys-jitter-benchmark/ -├── data/ # Store experiments here ├── outputs/ # Where we run experiment bundles ├── notebooks/ # Jupyter notebooks to analyse data ├── src/ # libraries @@ -73,24 +99,171 @@ All experiment parameters are controlled via Hydra’s configuration file at: conf/config.yaml ``` -## Example configuration +You can override any configuration parameter from the command line: +```bash +sudo .venv/bin/python3 main.py run.command=cyclictest run.t_core="3,5" ``` + +## Run Configuration + +```yaml run: - command: "caterpillar" - llc_cache_mask: "0x000f" - t_core: "3" - stressor: true - tests_path: "tests" + command: "caterpillar" # Benchmark to run + t_core: "9,11" # Target CPU cores + numa_node: "1" # NUMA node for cpuset-mems (should be same as NUMA node for t_core) + stressor: true # Enable stress workload + metrics: true # Enable metrics monitoring + docker: true # Run inside Docker container + cat_clos_pinning: + enable: true # Pin test PID to CLOS + clos: 1 # CLOS ID to use +``` + +| Parameter | Type | Description | +| ---------------------------- | ------- | -------------------------------------------------------------------------------------------------------------- | +| `run.command` | str | Benchmark to run: `caterpillar`, `cyclictest`, `iperf3`, `mega-benchmark`, `codesys-jitter-benchmark`, `codesys-opcua-pubsub`, or `build`. | +| `run.t_core` | str | Target CPU cores for running the benchmark (e.g., `"3,5,7,9"` or `"9,11"`) | +| `run.numa_node` | str | NUMA node for cpuset-mems (should be same as NUMA node for t_core) | +| `run.stressor` | bool | Enables additional stress workload during the benchmark | +| `run.metrics` | bool | Enable real-time metrics monitoring (CPU temp, IRQs, memory, etc.) | +| `run.docker` | bool | Run benchmark inside Docker container (if `false`, runs on host) | +| `run.cat_clos_pinning.enable`| bool | Enable pinning test PID to specified CLOS (caterpillar/cyclictest only) | +| `run.cat_clos_pinning.clos` | int | CLOS ID to pin the test process to | + +## Intel RDT/CAT Configuration (pqos) + +Configure Intel Resource Director Technology (Cache Allocation Technology, Memory Bandwidth Allocation): + +```yaml +pqos: + interface: "os" # 'os' for resctrl (recommended), 'msr' for direct access + reset_before_apply: true # Reset all allocations before applying new ones + + classes: + - id: 1 + description: "real-time workload" + l3_mask: "0x00ff" # L3 cache mask (8 cache ways) + l2_mask: "0x00ff" # L2 cache mask + mba: 100 # Memory Bandwidth Allocation (%) + pids: [] # PIDs to assign to this class + cores: [] # CPU cores to assign to this class + - id: 0 + description: "background worker" + l3_mask: "0x7f00" # Different cache ways for isolation + l2_mask: "0xff00" + mba: 10 + cores: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] + pids: [115, 118] ``` -| Parameter | Type | Description | -| ---------------- | ------- | -------------------------------------------------------------------------------------------------------------- | -| `command` | str | Benchmark to run. One of: `caterpillar`, `cyclictest`, `iperf3`, `mega-benchmark`, `codesys-jitter-benchmark`. | -| `llc_cache_mask` | str | Hexadecimal mask for Last-Level Cache (LLC) configuration. | -| `t_core` | str/int | Target CPU core for running the benchmark (i.e. '3,5,7,9') | -| `stressor` | bool | Enables additional stress workload during the benchmark. | -| `tests_path` | str | Path to the directory containing benchmark implementations. | +| Parameter | Type | Description | +| -------------------------- | ------ | ------------------------------------------------------------------ | +| `pqos.interface` | str | Interface mode: `os` (resctrl, required for PIDs) or `msr` (direct)| +| `pqos.reset_before_apply` | bool | Reset all allocations before applying new configuration | +| `pqos.classes[].id` | int | Class of Service (CLOS) ID | +| `pqos.classes[].l3_mask` | str | Hexadecimal L3 cache way mask | +| `pqos.classes[].l2_mask` | str | Hexadecimal L2 cache way mask | +| `pqos.classes[].mba` | int | Memory Bandwidth Allocation percentage (10-100) | +| `pqos.classes[].cores` | list | CPU cores to assign to this CLOS leave empty if not used | +| `pqos.classes[].pids` | list | Process IDs to assign to this CLOS leave empty if not used | + + +## IRQ Affinity Configuration + +Configure IRQ and RCU task affinity to isolate real-time cores: + +```yaml +irq_affinity: + enabled: true + housekeeping_cores: "0-1" # Cores for handling IRQs and RCU +``` + +## BIOS Settings Collection via Redfish + +Automatically collect BIOS settings from servers with Redfish-enabled BMC (e.g., Dell iDRAC) before running benchmarks: + +```yaml +bios: + enable: true + redfish: + host: "192.168.1.100" # BMC/iDRAC IP address + username: "root" + password: "YOUR_PASSWORD" + verify_ssl: false # Set to true for valid SSL certificates + timeout: 15 + + output: + format: "json" # Output format: json, yaml, or text + file: "${hydra:run.dir}/bios.json" + pretty: true +``` + +| Parameter | Type | Description | +| ------------------------- | ------ | ------------------------------------------------------------------ | +| `bios.enable` | bool | Enable/disable BIOS settings collection | +| `bios.redfish.host` | str | BMC/iDRAC hostname or IP address | +| `bios.redfish.username` | str | Username for Redfish API authentication | +| `bios.redfish.password` | str | Password for Redfish API authentication | +| `bios.redfish.verify_ssl` | bool | Verify SSL certificates (set `false` for self-signed certs) | +| `bios.redfish.timeout` | int | Connection timeout in seconds | +| `bios.output.format` | str | Output format: `json`, `yaml`, or `text` | +| `bios.output.file` | str | Path to save BIOS settings (supports Hydra interpolation) | +| `bios.output.pretty` | bool | Enable pretty-printing for JSON output | + +## Test-Specific Configuration + +### Caterpillar +```yaml +caterpillar: + n_cycles: 7200 # Number of measurement cycles +``` + +### Cyclictest +```yaml +cyclictest: + loops: 100000 # Number of test loops +``` + +## Metrics Monitoring + +When `run.metrics: true`, the following monitors collect data during benchmark execution: + +| Monitor | Output File | Description | +| ---------------- | ------------------------------ | ---------------------------------------- | +| CPU Monitor | `cpu_monitor.csv` | Per-core CPU temperatures | +| IRQ Monitor | `irq_monitor.csv` | Interrupt counts per CPU | +| MemInfo Monitor | `meminfo_monitor.csv` | Memory statistics from `/proc/meminfo` | +| SoftIRQ Monitor | `softirq_monitor.csv` | Software interrupt statistics | +| CPUStat Monitor | `cpustat_monitor.csv` | CPU usage statistics | +| PQOS Monitor | `pqos_monitor.csv` | Intel RDT monitoring data | + +Configure monitoring intervals in the config: + +```yaml +cpu_monitor: + path: "${hydra:run.dir}/cpu_monitor.csv" + interval: 1.0 +``` + +## Output Files + +Each benchmark run creates a timestamped directory in `outputs/` containing: + +- `output.csv` - Benchmark results +- `sysinfo.json` - System information snapshot (includes Hydra configuration) +- `bios.json` - BIOS settings (if enabled) +- `*_monitor.csv` - Various metrics logs (if enabled) +- `.hydra/` - Hydra configuration logs + +## Security Note +⚠️ **Important**: The Redfish password is stored in the configuration file. Consider: +- Using environment variables for sensitive credentials +- Restricting file permissions on `config.yaml` +- Not committing passwords to version control +## References +- [Dealing with Uncertainty in pWCET Estimations](https://dl.acm.org/doi/abs/10.1145/3396234) - Region of Acceptance methodology +- [Probabilistic-WCET Reliability](https://dl.acm.org/doi/10.1145/3126495) - EVT hypothesis validation