diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt index 2a7b451f4..c27014566 100644 --- a/.cspell/custom-dictionary-workspace.txt +++ b/.cspell/custom-dictionary-workspace.txt @@ -7,6 +7,7 @@ AIO AIO's aiohttp Alertfeed +allclose Anson apexcharts appdaemon @@ -19,6 +20,11 @@ autoflake automations autopep autoupdate +axvline +axvspan +backprop +Backpropagate +backpropagation Basepath Batpred battemperature @@ -62,6 +68,7 @@ dayname daynumber daysymbol dend +denorm devcontainer devcontainers dexport @@ -91,6 +98,7 @@ energythroughput epod euids evse +exog exportlimit fdpwr fdsoc @@ -163,12 +171,17 @@ kvar kvarh kwargs kwhb +labelcolor linebreak +linestyle +loadml +loadmlpower loadspower localfolder lockstep logdata loglines +lookback luxpower markdownlint matplotlib @@ -254,6 +267,7 @@ pylint pyproject pytest pytz +randn rarr recp Redownload @@ -271,6 +285,7 @@ rstart rtype ruamel saverestore +savez scalarstring searr securetoken @@ -327,11 +342,13 @@ timekey timelapse timenow timeobj +timestep timestr timezone tojson Trefor treforsiphone +twinx unsmoothed unstaged useid @@ -349,6 +366,7 @@ wrongsha xaxis xaxistooltip xlabel +xlim xload xticks yaxis diff --git a/apps/predbat/components.py b/apps/predbat/components.py index 749b754da..110e30d52 100644 --- a/apps/predbat/components.py +++ b/apps/predbat/components.py @@ -14,6 +14,7 @@ from ohme import OhmeAPI from octopus import OctopusAPI from carbon import CarbonAPI +from temperature import TemperatureAPI from axle import AxleAPI from solax import SolaxAPI from solis import SolisAPI @@ -23,6 +24,7 @@ from db_manager import DatabaseManager from fox import FoxAPI from web_mcp import PredbatMCPServer +from load_ml_component import LoadMLComponent from datetime import datetime, timezone, timedelta import asyncio import os @@ -220,6 +222,17 @@ }, "phase": 1, }, + "temperature": { + "class": TemperatureAPI, + "name": "External Temperature API", + "args": { + "temperature_enable": {"required_true": True, "config": "temperature_enable", "default": False}, + "temperature_latitude": {"required": False, "config": "temperature_latitude", "default": None}, + "temperature_longitude": {"required": False, "config": "temperature_longitude", "default": None}, + "temperature_url": {"required": False, "config": "temperature_url", "default": "https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m&past_days=7"}, + }, + "phase": 1, + }, "axle": { "class": AxleAPI, "name": "Axle Energy", @@ -265,6 +278,17 @@ "phase": 1, "can_restart": True, }, + "load_ml": { + "class": LoadMLComponent, + "name": "ML Load Forecaster", + "event_filter": "predbat_load_ml_", + "args": { + "load_ml_enable": {"required_true": True, "config": "load_ml_enable", "default": False}, + "load_ml_source": {"required": False, "config": "load_ml_source", "default": False}, + }, + "phase": 1, + "can_restart": True, + }, } diff --git a/apps/predbat/config.py b/apps/predbat/config.py index d1bf4c121..2d2bf4fd9 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -2101,4 +2101,5 @@ "forecast_solar_max_age": {"type": "float"}, "enable_coarse_fine_levels": {"type": "boolean"}, "load_power_fill_enable": {"type": "boolean"}, + "load_ml_enable": {"type": "boolean"}, } diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 1a3bf8b8f..86517286a 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -507,7 +507,7 @@ def get_from_incrementing(self, data, index, backwards=True): else: return max(data.get(index + 1, 0) - data.get(index, 0), 0) - def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True): + def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True): """ Download one or more entities for import/export data """ @@ -529,7 +529,7 @@ def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None, continue try: - history = self.get_history_wrapper(entity_id=entity_id, days=self.max_days_previous) + history = self.get_history_wrapper(entity_id=entity_id, days=max_days_previous) except (ValueError, TypeError) as exc: self.log("Warn: No history data found for {} : {}".format(entity_id, exc)) history = [] @@ -537,7 +537,7 @@ def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None, if history and len(history) > 0: import_today, _ = minute_data( history[0], - self.max_days_previous, + max_days_previous, now_utc, "state", "last_updated", @@ -674,8 +674,15 @@ def fetch_sensor_data(self, save=True): self.iboost_today = dp2(abs(self.iboost_energy_today[0] - self.iboost_energy_today[self.minutes_now])) self.log("iBoost energy today from sensor reads {} kWh".format(self.iboost_today)) + # Fetch ML forecast if enabled + load_ml_forecast = {} + if self.get_arg("load_ml_enable", False) and self.get_arg("load_ml_source", False): + load_ml_forecast = self.fetch_ml_load_forecast(self.now_utc) + if load_ml_forecast: + self.load_forecast_only = True # Use only ML forecast for load if enabled and we have data + # Fetch extra load forecast - self.load_forecast, self.load_forecast_array = self.fetch_extra_load_forecast(self.now_utc) + self.load_forecast, self.load_forecast_array = self.fetch_extra_load_forecast(self.now_utc, load_ml_forecast) # Load previous load data if self.get_arg("ge_cloud_data", False): @@ -712,28 +719,28 @@ def fetch_sensor_data(self, save=True): # Load import today data if "import_today" in self.args: - self.import_today = self.minute_data_import_export(self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh") + self.import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh") self.import_today_now = get_now_from_cumulative(self.import_today, self.minutes_now, backwards=True) else: self.log("Warn: You have not set import_today in apps.yaml, you will have no previous import data") # Load export today data if "export_today" in self.args: - self.export_today = self.minute_data_import_export(self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh") + self.export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh") self.export_today_now = get_now_from_cumulative(self.export_today, self.minutes_now, backwards=True) else: self.log("Warn: You have not set export_today in apps.yaml, you will have no previous export data") # PV today data if "pv_today" in self.args: - self.pv_today = self.minute_data_import_export(self.now_utc, "pv_today", required_unit="kWh") + self.pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh") self.pv_today_now = get_now_from_cumulative(self.pv_today, self.minutes_now, backwards=True) else: self.log("Warn: You have not set pv_today in apps.yaml, you will have no previous PV data") # Battery temperature if "battery_temperature_history" in self.args: - self.battery_temperature_history = self.minute_data_import_export(self.now_utc, "battery_temperature_history", scale=1.0, increment=False, smoothing=False) + self.battery_temperature_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, "battery_temperature_history", scale=1.0, increment=False, smoothing=False) data = [] for minute in range(0, 24 * 60, 5): data.append({minute: self.battery_temperature_history.get(minute, 0)}) @@ -1059,8 +1066,8 @@ def fetch_sensor_data(self, save=True): # Fetch PV forecast if enabled, today must be enabled, other days are optional self.pv_forecast_minute, self.pv_forecast_minute10 = self.fetch_pv_forecast() - # Apply modal filter to historical data if self.load_minutes and not self.load_forecast_only: + # Apply modal filter to historical data self.previous_days_modal_filter(self.load_minutes) self.log("Historical days now {} weight {}".format(self.days_previous, self.days_previous_weight)) @@ -1203,17 +1210,17 @@ def download_ge_data(self, now_utc): self.log("GECloudData load_last_period from immediate sensor is {} kW".format(dp2(self.load_last_period))) if "import_today" in self.args: - import_today = self.minute_data_import_export(self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh") + import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh") self.import_today_now = get_now_from_cumulative(import_today, self.minutes_now, backwards=True) # Load export today data if "export_today" in self.args: - export_today = self.minute_data_import_export(self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh") + export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh") self.export_today_now = get_now_from_cumulative(export_today, self.minutes_now, backwards=True) # PV today data if "pv_today" in self.args: - pv_today = self.minute_data_import_export(self.now_utc, "pv_today", required_unit="kWh") + pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh") self.pv_today_now = get_now_from_cumulative(pv_today, self.minutes_now, backwards=True) self.log("Downloaded {} datapoints from GECloudData going back {} days".format(len(self.load_minutes), self.load_minutes_age)) @@ -1769,13 +1776,51 @@ def get_car_charging_planned(self): ) ) - def fetch_extra_load_forecast(self, now_utc): + def fetch_ml_load_forecast(self, now_utc): + """ + Fetches ML load forecast from sensor + and returns it as a minute_data dictionary + """ + # Use ML Model for load prediction + load_ml_forecast = self.get_state_wrapper("sensor." + self.prefix + "_load_ml_forecast", attribute="results") + if load_ml_forecast: + self.log("Loading ML load forecast from sensor.sensor.{}_load_ml_forecast".format(self.prefix)) + # Convert format from dict to array + if isinstance(load_ml_forecast, dict): + data_array = [] + for key, value in load_ml_forecast.items(): + data_array.append({"energy": value, "last_updated": key}) + + # Load data + load_forecast, _ = minute_data( + data_array, + self.forecast_days + 1, + self.midnight_utc, + "energy", + "last_updated", + backwards=False, + clean_increment=False, + smoothing=True, + divide_by=1.0, + scale=self.load_scaling, + ) + + if load_forecast: + self.log("Loaded the ML load forecast; now {}kWh to midnight {}kwh".format(load_forecast.get(self.minutes_now, 0), load_forecast.get(24 * 60 - PREDICT_STEP, 0))) + return load_forecast + return {} + + def fetch_extra_load_forecast(self, now_utc, ml_forecast=None): """ Fetch extra load forecast, this is future load data """ load_forecast_final = {} load_forecast_array = [] + # Add ML forecast if available + if ml_forecast: + load_forecast_array.append(ml_forecast) + if "load_forecast" in self.args: entity_ids = self.get_arg("load_forecast", indirect=False) if isinstance(entity_ids, str): @@ -1855,7 +1900,7 @@ def fetch_carbon_intensity(self, entity_id): state = self.get_state_wrapper(entity_id=entity_id) if state is not None: try: - carbon_history = self.minute_data_import_export(self.now_utc, entity_id, required_unit="g/kWh", increment=False, smoothing=False) + carbon_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, entity_id, required_unit="g/kWh", increment=False, smoothing=False) except (ValueError, TypeError): self.log("Warn: No carbon intensity history in sensor {}".format(entity_id)) else: @@ -2166,7 +2211,7 @@ def load_car_energy(self, now_utc): """ self.car_charging_energy = {} if "car_charging_energy" in self.args: - self.car_charging_energy = self.minute_data_import_export(now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh") + self.car_charging_energy = self.minute_data_import_export(self.max_days_previous, now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh") else: self.log("Car charging hold {}, threshold {}kWh".format(self.car_charging_hold, self.car_charging_threshold * 60.0)) return self.car_charging_energy diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py new file mode 100644 index 000000000..b3f66d9d9 --- /dev/null +++ b/apps/predbat/load_ml_component.py @@ -0,0 +1,518 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2025 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# ML Load Forecaster Component - ComponentBase wrapper for LoadPredictor +# ----------------------------------------------------------------------------- +# fmt off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +import asyncio +import os +from datetime import datetime, timezone, timedelta +from component_base import ComponentBase +from utils import get_now_from_cumulative, dp2, minute_data +from load_predictor import LoadPredictor, MODEL_VERSION +from const import TIME_FORMAT, PREDICT_STEP +import traceback + +# Training intervals +RETRAIN_INTERVAL_SECONDS = 2 * 60 * 60 # 2 hours between training cycles +PREDICTION_INTERVAL_SECONDS = 30 * 60 # 30 minutes between predictions + + +class LoadMLComponent(ComponentBase): + """ + ML Load Forecaster component that predicts household load for the next 48 hours. + + This component: + - Fetches load history from configured sensor + - Optionally fills gaps using load_power sensor + - Subtracts configured sensors (e.g., car charging) from load + - Trains/fine-tunes an MLP model on historical load data + - Generates predictions in the same format as load_forecast + - Falls back to empty predictions when validation fails or model is stale + """ + + def initialize(self, load_ml_enable, load_ml_source=True): + """ + Initialize the ML load forecaster component. + + Args: + load_ml_enable: Whether ML forecasting is enabled + """ + self.ml_enable = load_ml_enable + self.ml_source = load_ml_source + self.ml_load_sensor = self.get_arg("load_today", default=[], indirect=False) + self.ml_load_power_sensor = self.get_arg("load_power", default=[], indirect=False) + self.ml_pv_sensor = self.get_arg("pv_today", default=[], indirect=False) + self.ml_subtract_sensors = self.get_arg("car_charging_energy", default=[], indirect=False) + self.car_charging_hold = self.get_arg("car_charging_hold", True) + self.car_charging_threshold = float(self.get_arg("car_charging_threshold", 6.0)) / 60.0 + self.car_charging_energy_scale = self.get_arg("car_charging_energy_scale", 1.0) + self.car_charging_rate = float(self.get_arg("car_charging_rate", 7.5)) / 60.0 + + self.ml_learning_rate = 0.001 + self.ml_epochs_initial = 50 + self.ml_epochs_update = 2 + self.ml_min_days = 1 + self.ml_validation_threshold = 2.0 + self.ml_time_decay_days = 7 + self.ml_max_load_kw = 50.0 + self.ml_max_model_age_hours = 48 + + # Data state + self.load_data = None + self.load_data_age_days = 0 + self.pv_data = None + self.temperature_data = None + self.data_ready = False + self.data_lock = asyncio.Lock() + self.last_data_fetch = None + + # Model state + self.predictor = None + self.model_valid = False + self.model_status = "not_initialized" + self.last_train_time = None + self.initial_training_done = False + + # Predictions cache + self.current_predictions = {} + + # Model file path + self.model_filepath = None + + # Validate configuration + if self.ml_enable and not self.ml_load_sensor: + self.log("Error: ML Component: ml_load_sensor must be configured when ml_enable is True") + self.ml_enable = False + + # Initialize predictor + self._init_predictor() + + def _init_predictor(self): + """Initialize or reinitialize the predictor.""" + self.predictor = LoadPredictor(log_func=self.log, learning_rate=self.ml_learning_rate, max_load_kw=self.ml_max_load_kw) + + # Determine model save path + if self.config_root: + self.model_filepath = os.path.join(self.config_root, "predbat_ml_model.npz") + else: + self.model_filepath = None + + # Try to load existing model + if self.model_filepath and os.path.exists(self.model_filepath): + load_success = self.predictor.load(self.model_filepath) + if load_success: + self.log("ML Component: Loaded existing model") + # Check if model is still valid + is_valid, reason = self.predictor.is_valid(validation_threshold=self.ml_validation_threshold, max_age_hours=self.ml_max_model_age_hours) + if is_valid: + self.model_valid = True + self.model_status = "active" + self.initial_training_done = True + else: + self.log("ML Component: Loaded model is invalid ({}), will retrain".format(reason)) + self.model_status = "fallback_" + reason + else: + # Model load failed (version mismatch, architecture change, etc.) + # Reinitialize predictor to ensure clean state + self.log("ML Component: Failed to load model, reinitializing predictor") + self.predictor = LoadPredictor(log_func=self.log, learning_rate=self.ml_learning_rate, max_load_kw=self.ml_max_load_kw) + + async def _fetch_load_data(self): + """ + Fetch and process load data from configured sensors. + + Returns: + Tuple of (load_minutes_dict, age_days, load_minutes_now, pv_data) or (None, 0, 0, None) on failure + """ + if not self.ml_load_sensor: + return None, 0, 0, None, None + + try: + # Determine how many days of history to fetch, up to 7 days back + days_to_fetch = max(7, self.ml_min_days) + + # Fetch load sensor history + self.log("ML Component: Fetching {} days of load history from {}".format(days_to_fetch, self.ml_load_sensor)) + + load_minutes, load_minutes_age = self.base.minute_data_load(self.now_utc, "load_today", days_to_fetch, required_unit="kWh", load_scaling=self.get_arg("load_scaling", 1.0), interpolate=True) + if not load_minutes: + self.log("Warn: ML Component: Failed to convert load history to minute data") + return None, 0, 0, None, None + + if self.get_arg("load_power", default=None, indirect=False): + load_power_data, _ = self.base.minute_data_load(self.now_utc, "load_power", days_to_fetch, required_unit="W", load_scaling=1.0, interpolate=True) + load_minutes = self.base.fill_load_from_power(load_minutes, load_power_data) + + # Get current cumulative load value + load_minutes_now = get_now_from_cumulative(load_minutes, self.minutes_now, backwards=True) + + car_charging_energy = {} + if self.get_arg("car_charging_energy", default=None, indirect=False): + car_charging_energy = self.base.minute_data_import_export(days_to_fetch, self.now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh") + + max_minute = max(load_minutes.keys()) if load_minutes else 0 + max_minute = (max_minute // 5) * 5 # Align to 5-minute intervals + load_minutes_new = {} + + # Subtract configured sensors (e.g., car charging) + total_load_energy = 0 + car_delta = 0.0 + STEP = PREDICT_STEP + for minute in range(max_minute, -STEP, -STEP): + if self.car_charging_hold and car_charging_energy: + car_delta = abs(car_charging_energy.get(minute, 0.0) - car_charging_energy.get(minute - STEP, car_charging_energy.get(minute, 0.0))) + elif self.car_charging_hold: + load_now = abs(load_minutes.get(minute, 0.0) - load_minutes.get(minute - STEP, load_minutes.get(minute, 0.0))) + if load_now >= self.car_charging_threshold * STEP: + car_delta = self.car_charging_rate * STEP + if car_delta > 0: + # When car is enable spread over 5 minutes due to alignment between car and house load data + load_delta = abs(load_minutes.get(minute, 0.0) - load_minutes.get(minute - STEP, load_minutes.get(minute, 0.0))) + load_delta = max(0.0, load_delta - car_delta) + for m in range(minute, minute - STEP, -1): + load_minutes_new[m] = total_load_energy + load_delta / STEP + total_load_energy += load_delta + else: + # Otherwise just copy load data + for m in range(minute, minute - STEP, -1): + load_delta = abs(load_minutes.get(minute, 0.0) - load_minutes.get(minute - 1, load_minutes.get(minute, 0.0))) + load_minutes_new[m] = total_load_energy + total_load_energy += load_delta + + # Calculate age of data + age_days = max_minute / (24 * 60) + + # PV Data + if self.ml_pv_sensor: + pv_data, _ = self.base.minute_data_load(self.now_utc, "pv_today", days_to_fetch, required_unit="kWh", load_scaling=1.0, interpolate=True) + else: + pv_data = {} + + # Temperature predictions + temp_entity = "sensor." + self.prefix + "_temperature" + temperature_info = self.get_state_wrapper(temp_entity, attribute="results") + temperature_data = {} + if isinstance(temperature_info, dict): + data_array = [] + for key, value in temperature_info.items(): + data_array.append({"state": value, "last_updated": key}) + + # Load data from past and future predictions, base backwards around now_utc + # We also get the last 7 days in the past to help the model learn the daily pattern + temperature_data, _ = minute_data( + data_array, + days_to_fetch, + self.now_utc, + "state", + "last_updated", + backwards=True, + clean_increment=False, + smoothing=True, + divide_by=1.0, + scale=1.0, + ) + self.log("ML Temperature data points: {}".format(len(temperature_data))) + + self.log("ML Component: Fetched {} load data points, {:.1f} days of history".format(len(load_minutes_new), age_days)) + # with open("input_train_data.json", "w") as f: + # import json + # json.dump([load_minutes_new, age_days, load_minutes_now, pv_data, temperature_data], f, indent=2) + return load_minutes_new, age_days, load_minutes_now, pv_data, temperature_data + + except Exception as e: + self.log("Error: ML Component: Failed to fetch load data: {}".format(e)) + self.log("Error: ML Component: {}".format(traceback.format_exc())) + return None, 0, 0, None, None + + def get_current_prediction(self): + """ + Returns the current ML load predictions. + + Output format: + Dict of {minute: cumulative_kwh} + """ + return self.current_predictions + + def _get_predictions(self, now_utc, midnight_utc, exog_features=None): + """ + Get current predictions for integration with load_forecast. + + Called from fetch.py to retrieve ML predictions. + + Args: + now_utc: Current UTC timestamp + midnight_utc: Today's midnight UTC timestamp + exog_features: Optional dict with future exogenous data + + Returns: + Dict of {minute: cumulative_kwh} or empty dict on fallback + """ + if not self.ml_enable: + return {} + + if not self.data_ready: + self.log("ML Component: No load data available for prediction") + return {} + + if not self.model_valid: + self.log("ML Component: Model not valid ({}), returning empty predictions".format(self.model_status)) + return {} + + # Generate predictions using current model + try: + predictions = self.predictor.predict(self.load_data, now_utc, midnight_utc, pv_minutes=self.pv_data, temp_minutes=self.temperature_data, exog_features=exog_features) + + if predictions: + self.current_predictions = predictions + self.log("ML Component: Generated {} predictions (total {:.2f} kWh over 48h)".format(len(predictions), max(predictions.values()) if predictions else 0)) + + return predictions + + except Exception as e: + self.log("Error: ML Component: Prediction failed: {}".format(e)) + return {} + + async def run(self, seconds, first): + """ + Main component loop - handles data fetching, training and prediction cycles. + + Args: + seconds: Seconds since component start + first: True if this is the first run + + Returns: + True if successful, False otherwise + """ + if not self.ml_enable: + self.api_started = True + return True + + # Fetch fresh load data periodically (every 15 minutes) + should_fetch = first or ((seconds % PREDICTION_INTERVAL_SECONDS) == 0) + + if should_fetch: + async with self.data_lock: + load_data, age_days, load_minutes_now, pv_data, temperature_data = await self._fetch_load_data() + if load_data: + self.load_data = load_data + self.load_data_age_days = age_days + self.load_minutes_now = load_minutes_now + self.data_ready = True + self.last_data_fetch = self.now_utc + pv_forecast_minute, pv_forecast_minute10 = self.base.fetch_pv_forecast() + # PV Data has the historical PV data (minute is the number of minutes in the past) + # PV forecast has the predicted PV generation for the next 24 hours (minute is the number of minutes from midnight forward + # Combine the two into a new dict where negative minutes are in the future and positive in the past + self.pv_data = pv_data + current_value = pv_data.get(0, 0) + if pv_forecast_minute: + max_minute = max(pv_forecast_minute.keys()) + PREDICT_STEP + for minute in range(self.minutes_now + PREDICT_STEP, max_minute, PREDICT_STEP): + current_value += pv_forecast_minute.get(minute, current_value) + pv_data[-minute + self.minutes_now] = current_value + self.temperature_data = temperature_data + else: + self.log("Warn: ML Component: Failed to fetch load data") + + # Check if we have data + if not self.data_ready: + if first: + self.log("ML Component: Waiting for load data from sensors") + return True # Not an error, just waiting + + # Check if we have enough data + if self.load_data_age_days < self.ml_min_days: + self.model_status = "insufficient_data" + self.model_valid = False + if first: + self.log("ML Component: Insufficient data ({:.1f} days, need {})".format(self.load_data_age_days, self.ml_min_days)) + return True + + # Determine if training is needed + should_train = False + is_initial = False + + if not self.initial_training_done: + # First training + should_train = True + is_initial = True + self.log("ML Component: Starting initial training") + elif seconds % RETRAIN_INTERVAL_SECONDS == 0: + # Periodic fine-tuning every 2 hours + should_train = True + is_initial = False + self.log("ML Component: Starting fine-tune training (2h interval)") + + if should_train: + await self._do_training(is_initial) + + # Update model validity status + self._update_model_status() + + if should_fetch: + self._get_predictions(self.now_utc, self.midnight_utc) + # Publish entity with current state + self._publish_entity() + self.log("ML Component: Prediction cycle completed") + + self.update_success_timestamp() + return True + + async def _do_training(self, is_initial): + """ + Perform model training. + + Args: + is_initial: True for full training, False for fine-tuning + """ + async with self.data_lock: + if not self.load_data: + self.log("Warn: ML Component: No data for training") + return + + # Warn if limited data + if self.load_data_age_days < 3: + self.log("Warn: ML Component: Training with only {} days of data, recommend 3+ days for better accuracy".format(self.load_data_age_days)) + + try: + # Run training in executor to avoid blocking + epochs = self.ml_epochs_initial if is_initial else self.ml_epochs_update + + val_mae = self.predictor.train(self.load_data, self.now_utc, pv_minutes=self.pv_data, temp_minutes=self.temperature_data, is_initial=is_initial, epochs=epochs, time_decay_days=self.ml_time_decay_days) + + if val_mae is not None: + self.last_train_time = datetime.now(timezone.utc) + self.initial_training_done = True + + # Check validation threshold + if val_mae <= self.ml_validation_threshold: + self.model_valid = True + self.model_status = "active" + self.log("ML Component: Training successful, val_mae={:.4f} kWh".format(val_mae)) + else: + self.model_valid = False + self.model_status = "fallback_validation" + self.log("Warn: ML Component: Validation MAE ({:.4f}) exceeds threshold ({:.4f})".format(val_mae, self.ml_validation_threshold)) + + # Save model + if self.model_filepath: + self.predictor.save(self.model_filepath) + else: + self.log("Warn: ML Component: Training failed") + + except Exception as e: + self.log("Error: ML Component: Training exception: {}".format(e)) + self.log("Error: " + traceback.format_exc()) + + def _update_model_status(self): + """Update model validity status based on current state.""" + if not self.predictor or not self.predictor.model_initialized: + self.model_valid = False + self.model_status = "not_initialized" + return + + is_valid, reason = self.predictor.is_valid(validation_threshold=self.ml_validation_threshold, max_age_hours=self.ml_max_model_age_hours) + + if is_valid: + self.model_valid = True + self.model_status = "active" + else: + self.model_valid = False + self.model_status = "fallback_" + reason + + def _publish_entity(self): + """Publish the load_forecast_ml entity with current predictions.""" + # Convert predictions to timestamp format for entity + results = {} + reset_amount = 0 + load_today_h1 = 0 + load_today_h8 = 0 + load_today_now = 0 + power_today_now = 0 + power_today_h1 = 0 + power_today_h8 = 0 + # Future predictions + if self.current_predictions: + prev_value = 0 + for minute, value in self.current_predictions.items(): + timestamp = self.midnight_utc + timedelta(minutes=minute + self.minutes_now) + timestamp_str = timestamp.strftime(TIME_FORMAT) + # Reset at midnight + if minute > 0 and ((minute + self.minutes_now) % (24 * 60) == 0): + reset_amount = value + self.load_minutes_now + output_value = round(value - reset_amount + self.load_minutes_now, 4) + results[timestamp_str] = output_value + delta_value = (value - prev_value) / PREDICT_STEP * 60.0 + if minute == 0: + power_today_now = delta_value + if minute == 60: + load_today_h1 = output_value + power_today_h1 = delta_value + if minute == 60 * 8: + load_today_h8 = output_value + power_today_h8 = delta_value + prev_value = value + + # Get model age + model_age_hours = self.predictor.get_model_age_hours() if self.predictor else None + + # Calculate total predicted load + total_kwh = max(self.current_predictions.values()) if self.current_predictions else 0 + + self.dashboard_item( + "sensor." + self.prefix + "_load_ml_forecast", + state=self.model_status, + attributes={ + "results": results, + "friendly_name": "ML Load Forecast", + "icon": "mdi:chart-line", + }, + app="load_ml", + ) + self.dashboard_item( + "sensor." + self.prefix + "_load_ml_stats", + state=round(total_kwh, 2), + attributes={ + "load_today": dp2(self.load_minutes_now), + "load_today_h1": dp2(load_today_h1), + "load_today_h8": dp2(load_today_h8), + "load_total": dp2(total_kwh), + "power_today_now": dp2(power_today_now), + "power_today_h1": dp2(power_today_h1), + "power_today_h8": dp2(power_today_h8), + "mae_kwh": round(self.predictor.validation_mae, 4) if self.predictor and self.predictor.validation_mae else None, + "last_trained": self.last_train_time.isoformat() if self.last_train_time else None, + "model_age_hours": round(model_age_hours, 1) if model_age_hours else None, + "training_days": self.load_data_age_days, + "status": self.model_status, + "model_version": MODEL_VERSION, + "epochs_trained": self.predictor.epochs_trained if self.predictor else 0, + "friendly_name": "ML Load Stats", + "state_class": "measurement", + "unit_of_measurement": "kWh", + "icon": "mdi:chart-line", + }, + app="load_ml", + ) + + def last_updated_time(self): + """Return last successful update time for component health check.""" + return self.last_success_timestamp + + def is_alive(self): + """Check if component is alive and functioning.""" + if not self.ml_enable: + return True + + if self.last_success_timestamp is None: + return False + + age = datetime.now(timezone.utc) - self.last_success_timestamp + return age < timedelta(minutes=10) diff --git a/apps/predbat/load_predictor.py b/apps/predbat/load_predictor.py new file mode 100644 index 000000000..7ff27014a --- /dev/null +++ b/apps/predbat/load_predictor.py @@ -0,0 +1,1059 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2025 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# Lightweight ML Load Predictor - NumPy-only MLP implementation +# ----------------------------------------------------------------------------- +# fmt off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +import numpy as np +import json +import os +from datetime import datetime, timezone, timedelta + +# Architecture constants (not user-configurable) +MODEL_VERSION = 5 # Bumped for temperature feature +LOOKBACK_STEPS = 288 # 24 hours at 5-min intervals +OUTPUT_STEPS = 1 # Single step output (autoregressive) +PREDICT_HORIZON = 576 # 48 hours of predictions (576 * 5 min) +HIDDEN_SIZES = [512, 256, 128, 64] # Deeper network with more capacity +BATCH_SIZE = 128 # Smaller batches for better gradient estimates +FINETUNE_HOURS = 24 # Hours of data for fine-tuning +STEP_MINUTES = 5 # Minutes per step + +# Feature constants +NUM_TIME_FEATURES = 4 # sin/cos minute-of-day, sin/cos day-of-week (for TARGET time) +NUM_LOAD_FEATURES = LOOKBACK_STEPS # Historical load values +NUM_PV_FEATURES = LOOKBACK_STEPS # Historical PV generation values +NUM_TEMP_FEATURES = LOOKBACK_STEPS # Historical temperature values +TOTAL_FEATURES = NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES + NUM_TIME_FEATURES + + +def relu(x): + """ReLU activation function""" + return np.maximum(0, x) + + +def relu_derivative(x): + """Derivative of ReLU""" + return (x > 0).astype(np.float32) + + +def huber_loss(y_true, y_pred, delta=1.0): + """Huber loss - robust to outliers""" + error = y_true - y_pred + abs_error = np.abs(error) + quadratic = np.minimum(abs_error, delta) + linear = abs_error - quadratic + return np.mean(0.5 * quadratic**2 + delta * linear) + + +def huber_loss_derivative(y_true, y_pred, delta=1.0): + """Derivative of Huber loss""" + error = y_pred - y_true + abs_error = np.abs(error) + return np.where(abs_error <= delta, error, delta * np.sign(error)) / y_true.shape[0] + + +def mse_loss(y_true, y_pred): + """Mean Squared Error loss""" + return np.mean((y_true - y_pred) ** 2) + + +def mse_loss_derivative(y_true, y_pred): + """Derivative of MSE loss""" + return 2 * (y_pred - y_true) / y_true.shape[0] + + +class LoadPredictor: + """ + Lightweight MLP-based load predictor using NumPy only. + + Predicts household electrical load for the next 48 hours using: + - Historical load data (lookback window) + - Cyclical time encodings (hour-of-day, day-of-week) + - Placeholder for future exogenous features (temperature, solar) + """ + + def __init__(self, log_func=None, learning_rate=0.001, max_load_kw=23.0): + """ + Initialize the load predictor. + + Args: + log_func: Logging function (defaults to print) + learning_rate: Learning rate for Adam optimizer + max_load_kw: Maximum load in kW for clipping predictions + """ + self.log = log_func if log_func else print + self.learning_rate = learning_rate + self.max_load_kw = max_load_kw + + # Model weights (initialized on first train) + self.weights = None + self.biases = None + + # Adam optimizer state + self.m_weights = None + self.v_weights = None + self.m_biases = None + self.v_biases = None + self.adam_t = 0 + + # Normalization parameters + self.feature_mean = None + self.feature_std = None + self.target_mean = None + self.target_std = None + self.pv_mean = None + self.pv_std = None + + # Training metadata + self.training_timestamp = None + self.validation_mae = None + self.epochs_trained = 0 + self.model_initialized = False + + def _initialize_weights(self): + """Initialize network weights using Xavier initialization""" + np.random.seed(42) # For reproducibility + + layer_sizes = [TOTAL_FEATURES] + HIDDEN_SIZES + [OUTPUT_STEPS] + + self.weights = [] + self.biases = [] + self.m_weights = [] + self.v_weights = [] + self.m_biases = [] + self.v_biases = [] + + for i in range(len(layer_sizes) - 1): + fan_in = layer_sizes[i] + fan_out = layer_sizes[i + 1] + + # Xavier initialization + std = np.sqrt(2.0 / (fan_in + fan_out)) + w = np.random.randn(fan_in, fan_out).astype(np.float32) * std + b = np.zeros(fan_out, dtype=np.float32) + + self.weights.append(w) + self.biases.append(b) + + # Adam optimizer momentum terms + self.m_weights.append(np.zeros_like(w)) + self.v_weights.append(np.zeros_like(w)) + self.m_biases.append(np.zeros_like(b)) + self.v_biases.append(np.zeros_like(b)) + + self.adam_t = 0 + self.model_initialized = True + + def _forward(self, X): + """ + Forward pass through the network. + + Args: + X: Input features (batch_size, TOTAL_FEATURES) + + Returns: + Output predictions and list of layer activations for backprop + """ + activations = [X] + pre_activations = [] + + current = X + for i, (w, b) in enumerate(zip(self.weights, self.biases)): + z = np.dot(current, w) + b + pre_activations.append(z) + + # Apply ReLU for hidden layers, linear for output + if i < len(self.weights) - 1: + current = relu(z) + else: + current = z # Linear output + + activations.append(current) + + return current, activations, pre_activations + + def _backward(self, y_true, activations, pre_activations, sample_weights=None): + """ + Backward pass using backpropagation. + + Args: + y_true: True target values + activations: Layer activations from forward pass + pre_activations: Pre-activation values from forward pass + sample_weights: Optional per-sample weights for weighted loss + + Returns: + Gradients for weights and biases + """ + batch_size = y_true.shape[0] + + # Output layer gradient (MSE loss derivative) + delta = mse_loss_derivative(y_true, activations[-1]) + + # Apply sample weights to gradient if provided + if sample_weights is not None: + delta = delta * sample_weights.reshape(-1, 1) + + weight_grads = [] + bias_grads = [] + + # Backpropagate through layers + for i in range(len(self.weights) - 1, -1, -1): + # Gradient for weights and biases + weight_grads.insert(0, np.dot(activations[i].T, delta)) + bias_grads.insert(0, np.sum(delta, axis=0)) + + if i > 0: + # Propagate gradient to previous layer + delta = np.dot(delta, self.weights[i].T) * relu_derivative(pre_activations[i - 1]) + + return weight_grads, bias_grads + + def _adam_update(self, weight_grads, bias_grads, beta1=0.9, beta2=0.999, epsilon=1e-8): + """ + Update weights using Adam optimizer. + + Args: + weight_grads: Gradients for weights + bias_grads: Gradients for biases + beta1: Exponential decay rate for first moment + beta2: Exponential decay rate for second moment + epsilon: Small constant for numerical stability + """ + self.adam_t += 1 + + for i in range(len(self.weights)): + # Update momentum for weights + self.m_weights[i] = beta1 * self.m_weights[i] + (1 - beta1) * weight_grads[i] + self.v_weights[i] = beta2 * self.v_weights[i] + (1 - beta2) * (weight_grads[i] ** 2) + + # Bias correction + m_hat = self.m_weights[i] / (1 - beta1**self.adam_t) + v_hat = self.v_weights[i] / (1 - beta2**self.adam_t) + + # Update weights + self.weights[i] -= self.learning_rate * m_hat / (np.sqrt(v_hat) + epsilon) + + # Update momentum for biases + self.m_biases[i] = beta1 * self.m_biases[i] + (1 - beta1) * bias_grads[i] + self.v_biases[i] = beta2 * self.v_biases[i] + (1 - beta2) * (bias_grads[i] ** 2) + + # Bias correction + m_hat = self.m_biases[i] / (1 - beta1**self.adam_t) + v_hat = self.v_biases[i] / (1 - beta2**self.adam_t) + + # Update biases + self.biases[i] -= self.learning_rate * m_hat / (np.sqrt(v_hat) + epsilon) + + def _create_time_features(self, minute_of_day, day_of_week): + """ + Create cyclical time features. + + Args: + minute_of_day: Minutes since midnight (0-1439) + day_of_week: Day of week (0-6, Monday=0) + + Returns: + Array of 4 time features: sin/cos minute, sin/cos day + """ + # Cyclical encoding for minute of day + minute_sin = np.sin(2 * np.pi * minute_of_day / 1440) + minute_cos = np.cos(2 * np.pi * minute_of_day / 1440) + + # Cyclical encoding for day of week + day_sin = np.sin(2 * np.pi * day_of_week / 7) + day_cos = np.cos(2 * np.pi * day_of_week / 7) + + return np.array([minute_sin, minute_cos, day_sin, day_cos], dtype=np.float32) + + def _add_exog_features(self, X, exog_dict=None): + """ + Placeholder for adding exogenous features (temperature, solar). + + Args: + X: Current feature array + exog_dict: Dictionary with optional "temperature" and "solar" data + + Returns: + Extended feature array (currently just returns X unchanged) + """ + # Future expansion: add temperature/solar features here + if exog_dict: + pass # Placeholder for future implementation + return X + + def _load_to_energy_per_step(self, load_minutes, step=STEP_MINUTES): + """ + Convert cumulative load_minutes dict to energy per step (kWh per 5 min). + + The load_minutes dict contains cumulative kWh values: + - Positive minutes: going backwards in time (historical data) + - Negative minutes: going forward in time (future forecasts) + Energy consumption for a period is the difference between start and end. + + Args: + load_minutes: Dict of {minute: cumulative_kwh} + step: Step size in minutes + + Returns: + Dict of {minute: energy_kwh_per_step} + """ + energy_per_step = {} + + if not load_minutes: + return energy_per_step + + # Get both positive (historical) and negative (future) minute ranges + all_minutes = list(load_minutes.keys()) + if not all_minutes: + return energy_per_step + + max_minute = max(all_minutes) + min_minute = min(all_minutes) + + # Process historical data (positive minutes, going backwards) + for minute in range(0, max_minute, step): + # Energy = cumulative_now - cumulative_later (going backwards) + val_now = load_minutes.get(minute, 0) + val_next = load_minutes.get(minute + step, 0) + energy = max(val_now - val_next, 0) # Ensure non-negative + energy_per_step[minute] = energy + + # Process future data (negative minutes, going forwards) + if min_minute < 0: + # Need to go from min_minute (-XXX) towards 0 in positive steps + # So we go from min to 0-step in steps of +step + for minute in range(min_minute, -step + 1, step): + # For future: energy = cumulative_now - cumulative_later (cumulative decreases going forward) + val_now = load_minutes.get(minute, 0) + val_next = load_minutes.get(minute + step, 0) + energy = max(val_now - val_next, 0) # Ensure non-negative + energy_per_step[minute] = energy + + return energy_per_step + + def _compute_daily_pattern(self, energy_per_step, smoothing_window=6): + """ + Compute average daily pattern from historical data. + + Groups energy values by minute-of-day and computes rolling average. + Used to blend with predictions to prevent autoregressive drift. + + Args: + energy_per_step: Dict of {minute: energy_kwh} + smoothing_window: Number of adjacent slots to smooth over + + Returns: + Dict of {minute_of_day: avg_energy} for 288 slots in a day + """ + # Collect energy values by minute-of-day (0 to 1435 in 5-min steps) + by_minute = {} + for minute, energy in energy_per_step.items(): + minute_of_day = minute % (24 * 60) # 0-1439 + # Align to 5-minute boundaries + slot = (minute_of_day // STEP_MINUTES) * STEP_MINUTES + if slot not in by_minute: + by_minute[slot] = [] + by_minute[slot].append(energy) + + # Compute mean for each slot + pattern = {} + for slot in range(0, 24 * 60, STEP_MINUTES): + if slot in by_minute and len(by_minute[slot]) > 0: + pattern[slot] = float(np.mean(by_minute[slot])) + else: + pattern[slot] = 0.05 # Default fallback + + # Apply smoothing to reduce noise + slots = sorted(pattern.keys()) + smoothed = {} + for i, slot in enumerate(slots): + values = [] + for offset in range(-smoothing_window // 2, smoothing_window // 2 + 1): + idx = (i + offset) % len(slots) + values.append(pattern[slots[idx]]) + smoothed[slot] = float(np.mean(values)) + + return smoothed + + def _create_dataset(self, load_minutes, now_utc, pv_minutes=None, temp_minutes=None, is_finetune=False, time_decay_days=7, validation_holdout_hours=24): + """ + Create training dataset from load_minutes dict. + + For autoregressive prediction: each sample uses 24h lookback to predict + the next single 5-minute step. Time features are for the TARGET time. + + Training uses all available data (from most recent to as far back as data goes). + Validation uses the most recent 24h as a subset of training data to check model fit. + + Args: + load_minutes: Dict of {minute: cumulative_kwh} going backwards in time + now_utc: Current UTC timestamp + pv_minutes: Dict of {minute: cumulative_kwh} PV generation (backwards for history, negative for future) + temp_minutes: Dict of {minute: temperature_celsius} Temperature (backwards for history, negative for future) + is_finetune: If True, only use last 24 hours; else use full data with time-decay + time_decay_days: Time constant for exponential decay weighting + validation_holdout_hours: Hours of most recent data to hold out for validation + + Returns: + X_train, y_train, train_weights: Training data + X_val, y_val: Validation data (most recent period) + """ + # Convert to energy per step + energy_per_step = self._load_to_energy_per_step(load_minutes) + pv_energy_per_step = self._load_to_energy_per_step(pv_minutes) if pv_minutes else {} + # Temperature is not cumulative, so just use the raw values (already in correct format) + temp_values = temp_minutes if temp_minutes else {} + + if not energy_per_step: + return None, None, None, None, None + + max_minute = max(energy_per_step.keys()) + + # Determine data range + if is_finetune: + # Only use last 48 hours for fine-tuning (24h train + 24h for lookback) + start_minute = 0 + end_minute = min(48 * 60, max_minute) + validation_holdout_hours = 12 # Smaller holdout for fine-tuning + else: + # Use 7 days of data for initial training + start_minute = 0 + end_minute = min(7 * 24 * 60, max_minute) + + # Need enough history for lookback plus validation holdout + min_required = LOOKBACK_STEPS * STEP_MINUTES + validation_holdout_hours * 60 + STEP_MINUTES + + if end_minute < min_required: + self.log("Warn: Insufficient data for ML training, need {} minutes, have {}".format(min_required, end_minute)) + return None, None, None, None, None + + # Validation uses most recent data (minute 0 to validation_holdout) + # Training uses ALL data (minute 0 to end_minute), including validation period + validation_end = validation_holdout_hours * 60 + + X_train_list = [] + y_train_list = [] + weight_list = [] + X_val_list = [] + y_val_list = [] + + # Create training samples (from all available data, including most recent) + # These samples predict targets in the range [0, end_minute - lookback] + for target_minute in range(0, end_minute - LOOKBACK_STEPS * STEP_MINUTES, STEP_MINUTES): + # Lookback window starts at target_minute + STEP_MINUTES (one step after target) + lookback_start = target_minute + STEP_MINUTES + + # Extract lookback window (24 hours of history before the target) + lookback_values = [] + pv_lookback_values = [] + temp_lookback_values = [] + valid_sample = True + + for lb_offset in range(LOOKBACK_STEPS): + lb_minute = lookback_start + lb_offset * STEP_MINUTES + if lb_minute in energy_per_step: + lookback_values.append(energy_per_step[lb_minute]) + # Add PV generation for the same time period (0 if no PV data) + pv_lookback_values.append(pv_energy_per_step.get(lb_minute, 0.0)) + # Add temperature for the same time period (0 if no temp data) + temp_lookback_values.append(temp_values.get(lb_minute, 0.0)) + else: + valid_sample = False + break + + if not valid_sample or len(lookback_values) != LOOKBACK_STEPS: + continue + + # Target is the single next step we're predicting + if target_minute not in energy_per_step: + continue + target_value = energy_per_step[target_minute] + + # Calculate time features for the TARGET time (what we're predicting) + target_time = now_utc - timedelta(minutes=target_minute) + minute_of_day = target_time.hour * 60 + target_time.minute + day_of_week = target_time.weekday() + time_features = self._create_time_features(minute_of_day, day_of_week) + + # Combine features: [load_lookback..., pv_lookback..., temp_lookback..., time_features...] + features = np.concatenate([np.array(lookback_values, dtype=np.float32), np.array(pv_lookback_values, dtype=np.float32), np.array(temp_lookback_values, dtype=np.float32), time_features]) + + X_train_list.append(features) + y_train_list.append(np.array([target_value], dtype=np.float32)) + + # Time-decay weighting (older samples get lower weight) + age_days = target_minute / (24 * 60) + if is_finetune: + weight = 1.0 # Equal weight for fine-tuning + else: + weight = np.exp(-age_days / time_decay_days) + weight_list.append(weight) + + # Create validation samples (from most recent data, minute 0 to validation_end) + # These samples use lookback from validation_end onwards to predict the holdout period + for target_minute in range(0, validation_end, STEP_MINUTES): + # Lookback window starts at target_minute + STEP_MINUTES + lookback_start = target_minute + STEP_MINUTES + + # Extract lookback window + lookback_values = [] + pv_lookback_values = [] + temp_lookback_values = [] + valid_sample = True + + for lb_offset in range(LOOKBACK_STEPS): + lb_minute = lookback_start + lb_offset * STEP_MINUTES + if lb_minute in energy_per_step: + lookback_values.append(energy_per_step[lb_minute]) + pv_lookback_values.append(pv_energy_per_step.get(lb_minute, 0.0)) + temp_lookback_values.append(temp_values.get(lb_minute, 0.0)) + else: + valid_sample = False + break + + if not valid_sample or len(lookback_values) != LOOKBACK_STEPS: + continue + + # Target value + if target_minute not in energy_per_step: + continue + target_value = energy_per_step[target_minute] + + # Time features for target time + target_time = now_utc - timedelta(minutes=target_minute) + minute_of_day = target_time.hour * 60 + target_time.minute + day_of_week = target_time.weekday() + time_features = self._create_time_features(minute_of_day, day_of_week) + + features = np.concatenate([np.array(lookback_values, dtype=np.float32), np.array(pv_lookback_values, dtype=np.float32), np.array(temp_lookback_values, dtype=np.float32), time_features]) + + X_val_list.append(features) + y_val_list.append(np.array([target_value], dtype=np.float32)) + + if not X_train_list: + return None, None, None, None, None + + X_train = np.array(X_train_list, dtype=np.float32) + y_train = np.array(y_train_list, dtype=np.float32) + train_weights = np.array(weight_list, dtype=np.float32) + + # Normalize weights to sum to number of samples + train_weights = train_weights * len(train_weights) / np.sum(train_weights) + + X_val = np.array(X_val_list, dtype=np.float32) if X_val_list else None + y_val = np.array(y_val_list, dtype=np.float32) if y_val_list else None + + return X_train, y_train, train_weights, X_val, y_val + + def _normalize_features(self, X, fit=False): + """ + Normalize features using z-score normalization. + + Args: + X: Feature array + fit: If True, compute and store normalization parameters + + Returns: + Normalized feature array + """ + if fit: + self.feature_mean = np.mean(X, axis=0) + self.feature_std = np.std(X, axis=0) + # Prevent division by zero + self.feature_std = np.maximum(self.feature_std, 1e-8) + + if self.feature_mean is None or self.feature_std is None: + return X + + return (X - self.feature_mean) / self.feature_std + + def _normalize_targets(self, y, fit=False): + """ + Normalize targets using z-score normalization. + + Args: + y: Target array + fit: If True, compute and store normalization parameters + + Returns: + Normalized target array + """ + if fit: + self.target_mean = np.mean(y) + self.target_std = np.std(y) + self.target_std = max(self.target_std, 1e-8) + + if self.target_mean is None or self.target_std is None: + return y + + return (y - self.target_mean) / self.target_std + + def _denormalize_predictions(self, y_pred): + """ + Denormalize predictions back to original scale. + + Args: + y_pred: Normalized predictions + + Returns: + Denormalized predictions in kWh + """ + if self.target_mean is None or self.target_std is None: + return y_pred + + return y_pred * self.target_std + self.target_mean + + def _clip_predictions(self, predictions, lookback_buffer=None): + """ + Apply physical constraints to predictions. + + Args: + predictions: Raw predictions in kWh per 5 min + lookback_buffer: Optional recent values to compute minimum floor + + Returns: + Clipped predictions + """ + # Convert max kW to kWh per 5 minutes + max_kwh_per_step = self.max_load_kw * STEP_MINUTES / 60.0 + + # Compute minimum floor based on recent data (prevent collapse to zero) + # Use 10% of the recent minimum as a floor, but at least 0.01 kWh (120W average) + if lookback_buffer is not None and len(lookback_buffer) > 0: + recent_min = min(lookback_buffer) + recent_mean = sum(lookback_buffer) / len(lookback_buffer) + # Floor is the smaller of: 20% of recent mean, or recent minimum + min_floor = max(0.01, min(recent_min, recent_mean * 0.2)) + else: + min_floor = 0.01 # ~120W baseline + + # Clip to valid range with minimum floor + predictions = np.clip(predictions, min_floor, max_kwh_per_step) + + return predictions + + def train(self, load_minutes, now_utc, pv_minutes=None, temp_minutes=None, is_initial=True, epochs=50, time_decay_days=7, patience=5): + """ + Train or fine-tune the model. + + Training uses all available data (most recent to as far back as data goes). + Validation uses the most recent 24 hours (subset of training data) to check model fit. + + Args: + load_minutes: Dict of {minute: cumulative_kwh} + now_utc: Current UTC timestamp + pv_minutes: Dict of {minute: cumulative_kwh} PV generation (backwards for history, negative for future) + temp_minutes: Dict of {minute: temperature_celsius} Temperature (backwards for history, negative for future) + is_initial: If True, full training; else fine-tuning on last 24h + epochs: Number of training epochs + time_decay_days: Time constant for sample weighting + patience: Early stopping patience + + Returns: + Validation MAE or None if training failed + """ + self.log("ML Predictor: Starting {} training with {} epochs".format("initial" if is_initial else "fine-tune", epochs)) + + # Create dataset with train/validation split + result = self._create_dataset(load_minutes, now_utc, pv_minutes=pv_minutes, temp_minutes=temp_minutes, is_finetune=not is_initial, time_decay_days=time_decay_days) + + if result[0] is None: + self.log("Warn: ML Predictor: Failed to create dataset") + return None + + X_train, y_train, train_weights, X_val, y_val = result + + if len(X_train) < BATCH_SIZE: + self.log("Warn: ML Predictor: Insufficient training data ({} samples)".format(len(X_train))) + return None + + self.log("ML Predictor: Created {} training samples, {} validation samples".format(len(X_train), len(X_val) if X_val is not None else 0)) + + # Check we have validation data + if X_val is None or len(X_val) == 0: + self.log("Warn: ML Predictor: No validation data available") + return None + + # Normalize features and targets + X_train_norm = self._normalize_features(X_train, fit=is_initial or not self.model_initialized) + X_val_norm = self._normalize_features(X_val, fit=False) + y_train_norm = self._normalize_targets(y_train, fit=is_initial or not self.model_initialized) + y_val_norm = self._normalize_targets(y_val, fit=False) + + # Initialize weights if needed + if not self.model_initialized or (is_initial and self.weights is None): + self._initialize_weights() + + # Training loop + best_val_loss = float("inf") + patience_counter = 0 + + for epoch in range(epochs): + # Shuffle training data + indices = np.random.permutation(len(X_train_norm)) + X_shuffled = X_train_norm[indices] + y_shuffled = y_train_norm[indices] + weights_shuffled = train_weights[indices] + + # Mini-batch training + epoch_loss = 0 + num_batches = 0 + + for batch_start in range(0, len(X_shuffled), BATCH_SIZE): + batch_end = min(batch_start + BATCH_SIZE, len(X_shuffled)) + X_batch = X_shuffled[batch_start:batch_end] + y_batch = y_shuffled[batch_start:batch_end] + batch_weights = weights_shuffled[batch_start:batch_end] + + # Forward pass + y_pred, activations, pre_activations = self._forward(X_batch) + + # Compute unweighted loss for monitoring + batch_loss = mse_loss(y_batch, y_pred) + epoch_loss += batch_loss + num_batches += 1 + + # Backward pass with sample weights applied to gradient + weight_grads, bias_grads = self._backward(y_batch, activations, pre_activations, sample_weights=batch_weights) + + # Adam update + self._adam_update(weight_grads, bias_grads) + + epoch_loss /= num_batches + + # Validation + val_pred, _, _ = self._forward(X_val_norm) + val_pred_denorm = self._denormalize_predictions(val_pred) + val_mae = np.mean(np.abs(y_val - val_pred_denorm)) + + self.log("ML Predictor: Epoch {}/{}: train_loss={:.4f} val_mae={:.4f} kWh".format(epoch + 1, epochs, epoch_loss, val_mae)) + + # Early stopping check + if val_mae < best_val_loss: + best_val_loss = val_mae + patience_counter = 0 + else: + patience_counter += 1 + + if patience_counter >= patience: + self.log("ML Predictor: Early stopping at epoch {}".format(epoch + 1)) + break + + self.training_timestamp = datetime.now(timezone.utc) + self.validation_mae = best_val_loss + self.epochs_trained += epochs + + self.log("ML Predictor: Training complete, final val_mae={:.4f} kWh".format(best_val_loss)) + + return best_val_loss + + def predict(self, load_minutes, now_utc, midnight_utc, pv_minutes=None, temp_minutes=None, exog_features=None): + """ + Generate predictions for the next 48 hours using autoregressive approach. + + Each iteration predicts the next 5-minute step, then feeds that prediction + back into the lookback window for the next iteration. This allows the model + to use target-time features for each prediction. + + To prevent autoregressive drift, predictions are blended with historical + daily patterns (average energy by time of day). + + Args: + load_minutes: Dict of {minute: cumulative_kwh} + now_utc: Current UTC timestamp + midnight_utc: Today's midnight UTC timestamp + pv_minutes: Dict of {minute: cumulative_kwh} PV generation (backwards for history, negative for future) + temp_minutes: Dict of {minute: temperature_celsius} Temperature (backwards for history, negative for future) + exog_features: Optional dict with future exogenous data + + Returns: + Dict of {minute: cumulative_kwh} in incrementing format for future, or empty dict on failure + """ + if not self.model_initialized or self.weights is None: + self.log("Warn: ML Predictor: Model not trained, cannot predict") + return {} + + # Convert to energy per step for extracting lookback + energy_per_step = self._load_to_energy_per_step(load_minutes) + pv_energy_per_step = self._load_to_energy_per_step(pv_minutes) if pv_minutes else {} + # Temperature is not cumulative, so just use the raw values + temp_values = temp_minutes if temp_minutes else {} + + if not energy_per_step: + self.log("Warn: ML Predictor: No load data available for prediction") + return {} + + # Compute historical daily patterns for blending (prevents autoregressive drift) + # Group historical energy by minute-of-day and compute average + historical_pattern = self._compute_daily_pattern(energy_per_step) + + # Build initial lookback window from historical data (most recent 24 hours) + # This will be updated as we make predictions (autoregressive) + lookback_buffer = [] + pv_lookback_buffer = [] + temp_lookback_buffer = [] + for lb_offset in range(LOOKBACK_STEPS): + lb_minute = lb_offset * STEP_MINUTES + if lb_minute in energy_per_step: + lookback_buffer.append(energy_per_step[lb_minute]) + else: + lookback_buffer.append(0) # Fallback to zero + # Add PV generation (0 if no data) + pv_lookback_buffer.append(pv_energy_per_step.get(lb_minute, 0.0)) + # Add temperature (0 if no data) + temp_lookback_buffer.append(temp_values.get(lb_minute, 0.0)) + + # Autoregressive prediction loop: predict one step at a time + predictions_energy = [] + + # Blending parameters: model weight decreases as we go further into future + # At step 0: 100% model, at step PREDICT_HORIZON: blend_floor% model + blend_floor = 0.5 # Minimum model weight at horizon (keep more model influence) + + for step_idx in range(PREDICT_HORIZON): + # Calculate target time for this prediction step + target_time = now_utc + timedelta(minutes=(step_idx + 1) * STEP_MINUTES) + minute_of_day = target_time.hour * 60 + target_time.minute + day_of_week = target_time.weekday() + time_features = self._create_time_features(minute_of_day, day_of_week) + + # Get PV value for the next step from forecast (negative minutes are future) + # For future predictions, use forecast; for past, it's already in pv_energy_per_step + future_minute = -(step_idx + 1) * STEP_MINUTES # Negative = future + next_pv_value = pv_energy_per_step.get(future_minute, 0.0) + # Get temperature value for the next step from forecast (negative minutes are future) + next_temp_value = temp_values.get(future_minute, 0.0) + + # Combine features: [load_lookback..., pv_lookback..., temp_lookback..., time_features...] + features = np.concatenate([np.array(lookback_buffer, dtype=np.float32), np.array(pv_lookback_buffer, dtype=np.float32), np.array(temp_lookback_buffer, dtype=np.float32), time_features]) + features = self._add_exog_features(features, exog_features) + + # Normalize and forward pass + features_norm = self._normalize_features(features.reshape(1, -1), fit=False) + pred_norm, _, _ = self._forward(features_norm) + pred_energy = self._denormalize_predictions(pred_norm[0]) + + # Apply physical constraints + pred_energy = self._clip_predictions(pred_energy) + model_pred = float(pred_energy[0]) # Single output + + # Get historical pattern value for this time of day + slot = (minute_of_day // STEP_MINUTES) * STEP_MINUTES + hist_value = historical_pattern.get(slot, model_pred) + + # Blend model prediction with historical pattern + # Linear decay: model weight goes from 1.0 to blend_floor over horizon + progress = step_idx / PREDICT_HORIZON + model_weight = 1.0 - progress * (1.0 - blend_floor) + energy_value = model_weight * model_pred + (1.0 - model_weight) * hist_value + + # Re-apply constraints after blending + max_kwh_per_step = self.max_load_kw * STEP_MINUTES / 60.0 + energy_value = max(0.01, min(energy_value, max_kwh_per_step)) + + predictions_energy.append(energy_value) + + # Update lookback buffer for next iteration (shift and add new prediction) + # Lookback[0] is most recent, so insert at front and remove from end + lookback_buffer.insert(0, energy_value) + lookback_buffer.pop() # Remove oldest value + + # Update PV lookback buffer with next forecast value + pv_lookback_buffer.insert(0, next_pv_value) + pv_lookback_buffer.pop() # Remove oldest value + + # Update temperature lookback buffer with next forecast value + temp_lookback_buffer.insert(0, next_temp_value) + temp_lookback_buffer.pop() # Remove oldest value + + # Convert to cumulative kWh format (incrementing into future) + # Format matches fetch_extra_load_forecast output + result = {} + cumulative = 0 + + for step_idx in range(PREDICT_HORIZON): + minute = step_idx * STEP_MINUTES + energy = predictions_energy[step_idx] + cumulative += energy + result[minute] = round(cumulative, 4) + + return result + + def save(self, filepath): + """ + Save model to file. + + Args: + filepath: Path to save model (without extension) + """ + if not self.model_initialized: + self.log("Warn: ML Predictor: No model to save") + return False + + try: + # Prepare metadata + metadata = { + "model_version": MODEL_VERSION, + "lookback_steps": LOOKBACK_STEPS, + "output_steps": OUTPUT_STEPS, + "predict_horizon": PREDICT_HORIZON, + "hidden_sizes": HIDDEN_SIZES, + "training_timestamp": self.training_timestamp.isoformat() if self.training_timestamp else None, + "validation_mae": float(self.validation_mae) if self.validation_mae else None, + "epochs_trained": self.epochs_trained, + "learning_rate": self.learning_rate, + "max_load_kw": self.max_load_kw, + "feature_mean": self.feature_mean.tolist() if self.feature_mean is not None else None, + "feature_std": self.feature_std.tolist() if self.feature_std is not None else None, + "target_mean": float(self.target_mean) if self.target_mean is not None else None, + "target_std": float(self.target_std) if self.target_std is not None else None, + "pv_mean": float(self.pv_mean) if self.pv_mean is not None else None, + "pv_std": float(self.pv_std) if self.pv_std is not None else None, + } + + # Save weights and metadata + save_dict = { + "metadata_json": json.dumps(metadata), + } + + for i, (w, b) in enumerate(zip(self.weights, self.biases)): + save_dict[f"weight_{i}"] = w + save_dict[f"bias_{i}"] = b + + # Save Adam optimizer state + for i in range(len(self.weights)): + save_dict[f"m_weight_{i}"] = self.m_weights[i] + save_dict[f"v_weight_{i}"] = self.v_weights[i] + save_dict[f"m_bias_{i}"] = self.m_biases[i] + save_dict[f"v_bias_{i}"] = self.v_biases[i] + + save_dict["adam_t"] = np.array([self.adam_t]) + + np.savez(filepath, **save_dict) + self.log("ML Predictor: Model saved to {}".format(filepath)) + return True + + except Exception as e: + self.log("Error: ML Predictor: Failed to save model: {}".format(e)) + return False + + def load(self, filepath): + """ + Load model from file. + + Args: + filepath: Path to model file + + Returns: + True if successful, False otherwise + """ + try: + if not os.path.exists(filepath): + self.log("ML Predictor: No saved model found at {}".format(filepath)) + return False + + data = np.load(filepath, allow_pickle=True) + + # Load metadata + metadata = json.loads(str(data["metadata_json"])) + + # Check version compatibility + saved_version = metadata.get("model_version", 0) + if saved_version != MODEL_VERSION: + self.log("Warn: ML Predictor: Model version mismatch (saved={}, current={}), retraining from scratch".format(saved_version, MODEL_VERSION)) + return False + + # Check architecture compatibility + if metadata.get("lookback_steps") != LOOKBACK_STEPS or metadata.get("output_steps") != OUTPUT_STEPS or metadata.get("hidden_sizes") != HIDDEN_SIZES: + self.log("Warn: ML Predictor: Architecture mismatch, retraining from scratch") + return False + + # Load weights + self.weights = [] + self.biases = [] + self.m_weights = [] + self.v_weights = [] + self.m_biases = [] + self.v_biases = [] + + layer_count = len(HIDDEN_SIZES) + 1 + for i in range(layer_count): + self.weights.append(data[f"weight_{i}"]) + self.biases.append(data[f"bias_{i}"]) + self.m_weights.append(data[f"m_weight_{i}"]) + self.v_weights.append(data[f"v_weight_{i}"]) + self.m_biases.append(data[f"m_bias_{i}"]) + self.v_biases.append(data[f"v_bias_{i}"]) + + self.adam_t = int(data["adam_t"][0]) + + # Load normalization parameters + if metadata.get("feature_mean"): + self.feature_mean = np.array(metadata["feature_mean"], dtype=np.float32) + if metadata.get("feature_std"): + self.feature_std = np.array(metadata["feature_std"], dtype=np.float32) + if metadata.get("target_mean") is not None: + self.target_mean = metadata["target_mean"] + if metadata.get("target_std") is not None: + self.target_std = metadata["target_std"] + if metadata.get("pv_mean") is not None: + self.pv_mean = metadata["pv_mean"] + if metadata.get("pv_std") is not None: + self.pv_std = metadata["pv_std"] + + # Load training metadata + if metadata.get("training_timestamp"): + self.training_timestamp = datetime.fromisoformat(metadata["training_timestamp"]) + self.validation_mae = metadata.get("validation_mae") + self.epochs_trained = metadata.get("epochs_trained", 0) + + self.model_initialized = True + + self.log("ML Predictor: Model loaded from {} (trained {}, val_mae={:.4f})".format(filepath, self.training_timestamp.strftime("%Y-%m-%d %H:%M") if self.training_timestamp else "unknown", self.validation_mae if self.validation_mae else 0)) + return True + + except Exception as e: + self.log("Error: ML Predictor: Failed to load model: {}".format(e)) + return False + + def get_model_age_hours(self): + """Get the age of the model in hours since last training.""" + if self.training_timestamp is None: + return None + + age = datetime.now(timezone.utc) - self.training_timestamp + return age.total_seconds() / 3600 + + def is_valid(self, validation_threshold=2.0, max_age_hours=48): + """ + Check if model is valid for predictions. + + Args: + validation_threshold: Maximum acceptable validation MAE in kWh + max_age_hours: Maximum model age in hours + + Returns: + Tuple of (is_valid, reason_if_invalid) + """ + if not self.model_initialized: + return False, "not_initialized" + + if self.weights is None: + return False, "no_weights" + + if self.validation_mae is not None and self.validation_mae > validation_threshold: + return False, "validation_threshold" + + age_hours = self.get_model_age_hours() + if age_hours is not None and age_hours > max_age_hours: + return False, "stale" + + return True, None diff --git a/apps/predbat/predbat.py b/apps/predbat/predbat.py index 68c906d10..55816897d 100644 --- a/apps/predbat/predbat.py +++ b/apps/predbat/predbat.py @@ -27,10 +27,10 @@ import requests import asyncio -THIS_VERSION = "v8.32.14" +THIS_VERSION = "v8.33.0" # fmt: off -PREDBAT_FILES = ["predbat.py", "const.py", "hass.py", "config.py", "prediction.py", "gecloud.py", "utils.py", "inverter.py", "ha.py", "download.py", "web.py", "web_helper.py", "predheat.py", "futurerate.py", "octopus.py", "solcast.py", "execute.py", "plan.py", "fetch.py", "output.py", "userinterface.py", "energydataservice.py", "alertfeed.py", "compare.py", "db_manager.py", "db_engine.py", "plugin_system.py", "ohme.py", "components.py", "fox.py", "carbon.py", "web_mcp.py", "component_base.py", "axle.py", "solax.py", "solis.py", "unit_test.py"] +PREDBAT_FILES = ["predbat.py", "const.py", "hass.py", "config.py", "prediction.py", "gecloud.py", "utils.py", "inverter.py", "ha.py", "download.py", "web.py", "web_helper.py", "predheat.py", "futurerate.py", "octopus.py", "solcast.py", "execute.py", "plan.py", "fetch.py", "output.py", "userinterface.py", "energydataservice.py", "alertfeed.py", "compare.py", "db_manager.py", "db_engine.py", "plugin_system.py", "ohme.py", "components.py", "fox.py", "carbon.py", "temperature.py", "web_mcp.py", "component_base.py", "axle.py", "solax.py", "solis.py", "unit_test.py"] # fmt: on from download import predbat_update_move, predbat_update_download, check_install diff --git a/apps/predbat/temperature.py b/apps/predbat/temperature.py new file mode 100644 index 000000000..6e53c2364 --- /dev/null +++ b/apps/predbat/temperature.py @@ -0,0 +1,210 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2025 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt: off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +import aiohttp +import asyncio +from datetime import datetime +from utils import dp1 +from component_base import ComponentBase + + +class TemperatureAPI(ComponentBase): + def initialize(self, temperature_enable, temperature_latitude, temperature_longitude, temperature_url): + """Initialize the Temperature API component""" + self.temperature_enable = temperature_enable + self.temperature_latitude = temperature_latitude + self.temperature_longitude = temperature_longitude + self.temperature_url = temperature_url + self.temperature_cache = {} + self.temperature_data = None + self.last_updated_timestamp = None + self.failures_total = 0 + + async def select_event(self, entity_id, value): + pass + + async def number_event(self, entity_id, value): + pass + + async def switch_event(self, entity_id, service): + pass + + async def run(self, seconds, first): + """ + Main run loop - polls API every hour + """ + try: + if not self.temperature_enable: + return True + if first or (seconds % (60 * 60) == 0): + # Fetch temperature data every hour + temperature_data = await self.fetch_temperature_data() + if temperature_data is not None: + self.temperature_data = temperature_data + self.last_updated_timestamp = datetime.now() + self.publish_temperature_sensor() + if self.temperature_data is not None: + self.update_success_timestamp() + self.publish_temperature_sensor() + except Exception as e: + self.log("Warn: TemperatureAPI: Exception in run loop: {}".format(e)) + # Still return True to keep component alive + if self.temperature_data is not None: + # Keep publishing old data even on error + self.publish_temperature_sensor() + + return True + + def get_coordinates(self): + """ + Get latitude and longitude, with fallback to zone.home + """ + # Try config values first + latitude = self.temperature_latitude + longitude = self.temperature_longitude + + # If latitude and longitude are not provided, use zone.home + if latitude is None: + latitude = self.get_state_wrapper("zone.home", attribute="latitude") + if longitude is None: + longitude = self.get_state_wrapper("zone.home", attribute="longitude") + + if latitude is not None and longitude is not None: + self.log("TemperatureAPI: Using coordinates latitude {}, longitude {}".format(dp1(latitude), dp1(longitude))) + return latitude, longitude + else: + self.log("Warn: TemperatureAPI: No latitude or longitude found, cannot fetch temperature data") + return None, None + + def build_api_url(self, latitude, longitude): + """ + Build the API URL with latitude and longitude placeholders replaced + """ + url = self.temperature_url.replace("LATITUDE", str(latitude)).replace("LONGITUDE", str(longitude)) + return url + + def convert_timezone_offset(self, utc_offset_seconds): + """ + Convert UTC offset in seconds to ±HH:MM format + Handles negative offsets correctly + """ + if utc_offset_seconds >= 0: + sign = "+" + else: + sign = "-" + utc_offset_seconds = abs(utc_offset_seconds) + + offset_hours = utc_offset_seconds // 3600 + offset_minutes = (utc_offset_seconds % 3600) // 60 + + return "{}{:02d}:{:02d}".format(sign, offset_hours, offset_minutes) + + async def fetch_temperature_data(self): + """ + Fetch temperature data from Open-Meteo API with retry logic + """ + latitude, longitude = self.get_coordinates() + if latitude is None or longitude is None: + return None + + url = self.build_api_url(latitude, longitude) + + # Try up to 3 times with exponential backoff + max_retries = 3 + for attempt in range(max_retries): + try: + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(url) as response: + if response.status == 200: + data = await response.json() + self.log("TemperatureAPI: Successfully fetched temperature data from Open-Meteo API") + self.update_success_timestamp() + return data + else: + self.log("Warn: TemperatureAPI: Failed to fetch data, status code {}".format(response.status)) + if attempt < max_retries - 1: + sleep_time = 2 ** attempt + self.log("Warn: TemperatureAPI: Retrying in {} seconds...".format(sleep_time)) + await asyncio.sleep(sleep_time) + else: + self.failures_total += 1 + return None + except (aiohttp.ClientError, asyncio.TimeoutError) as e: + if attempt < max_retries - 1: + sleep_time = 2 ** attempt + self.log("Warn: TemperatureAPI: Request attempt {} failed: {}. Retrying in {}s...".format(attempt + 1, e, sleep_time)) + await asyncio.sleep(sleep_time) + else: + self.log("Warn: TemperatureAPI: Request failed after {} attempts: {}".format(max_retries, e)) + self.failures_total += 1 + return None + except Exception as e: + self.log("Warn: TemperatureAPI: Unexpected error fetching temperature data: {}".format(e)) + self.failures_total += 1 + return None + + return None + + def publish_temperature_sensor(self): + """ + Publish temperature sensor to Home Assistant + """ + if self.temperature_data is None: + return + + try: + # Extract current temperature + current = self.temperature_data.get("current", {}) + current_temp = current.get("temperature_2m") + + if current_temp is None: + self.log("Warn: TemperatureAPI: No current temperature in API response") + return + + # Get timezone offset + utc_offset_seconds = self.temperature_data.get("utc_offset_seconds", 0) + timezone_offset = self.convert_timezone_offset(utc_offset_seconds) + + # Build hourly forecast dictionary + hourly = self.temperature_data.get("hourly", {}) + hourly_times = hourly.get("time", []) + hourly_temps = hourly.get("temperature_2m", []) + + forecast = {} + if len(hourly_times) == len(hourly_temps): + for time_str, temp in zip(hourly_times, hourly_temps): + # Convert ISO8601 time to HA format with timezone + # Open-Meteo returns: "2026-02-07T00:00" + # HA format: "2026-02-07T00:00:00+00:00" + ha_timestamp = "{}:00{}".format(time_str, timezone_offset) + forecast[ha_timestamp] = temp + + # Build last_updated string + last_updated_str = str(self.last_updated_timestamp) if self.last_updated_timestamp else "Never" + + # Publish sensor + self.dashboard_item( + "sensor." + self.prefix + "_temperature", + state=current_temp, + attributes={ + "friendly_name": "External Temperature Forecast", + "icon": "mdi:thermometer", + "unit_of_measurement": "°C", + "last_updated": last_updated_str, + "results": forecast, + "timezone_offset": timezone_offset, + "data_points": len(forecast) + }, + app="temperature" + ) + + except Exception as e: + self.log("Warn: TemperatureAPI: Error publishing sensor: {}".format(e)) diff --git a/apps/predbat/tests/test_load_ml.py b/apps/predbat/tests/test_load_ml.py new file mode 100644 index 000000000..1bd633ea3 --- /dev/null +++ b/apps/predbat/tests/test_load_ml.py @@ -0,0 +1,1646 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2025 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt: off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init +# fmt: on + +import numpy as np +from datetime import datetime, timezone, timedelta +import tempfile +import os + +from load_predictor import LoadPredictor, OUTPUT_STEPS, HIDDEN_SIZES, TOTAL_FEATURES, STEP_MINUTES, relu, relu_derivative, huber_loss + + +def test_load_ml(my_predbat=None): + """ + Comprehensive test suite for ML Load Forecaster. + + Tests all major functionality including: + - MLP forward/backward pass correctness + - Dataset creation with cyclical features + - Training convergence on synthetic data + - Model save/load with version check + - Cold-start and fine-tune scenarios + - Validation failure fallback + """ + + # Registry of all sub-tests + sub_tests = [ + ("relu_functions", _test_relu_functions, "ReLU activation and derivative"), + ("huber_loss_functions", _test_huber_loss_functions, "Huber loss computation"), + ("forward_pass", _test_forward_pass, "Forward pass computation"), + ("backward_pass", _test_backward_pass, "Backward pass gradient computation"), + ("cyclical_features", _test_cyclical_features, "Cyclical time feature encoding"), + ("load_to_energy", _test_load_to_energy, "Convert cumulative load to energy per step"), + ("pv_energy_conversion", _test_pv_energy_conversion, "Convert PV data including future forecasts"), + ("dataset_creation", _test_dataset_creation, "Dataset creation from load data"), + ("dataset_with_pv", _test_dataset_with_pv, "Dataset creation with PV features"), + ("dataset_with_temp", _test_dataset_with_temp, "Dataset creation with temperature features"), + ("normalization", _test_normalization, "Z-score normalization correctness"), + ("adam_optimizer", _test_adam_optimizer, "Adam optimizer step"), + ("training_convergence", _test_training_convergence, "Training convergence on synthetic data"), + ("training_with_pv", _test_training_with_pv, "Training with PV input features"), + ("training_with_temp", _test_training_with_temp, "Training with temperature input features"), + ("model_persistence", _test_model_persistence, "Model save/load with version check"), + ("cold_start", _test_cold_start, "Cold start with insufficient data"), + ("fine_tune", _test_fine_tune, "Fine-tune on recent data"), + ("prediction", _test_prediction, "End-to-end prediction"), + ("prediction_with_pv", _test_prediction_with_pv, "Prediction with PV forecast data"), + ("prediction_with_temp", _test_prediction_with_temp, "Prediction with temperature forecast data"), + # ("real_data_training", _test_real_data_training, "Train on real load_minutes_debug.json data with chart"), + ("component_fetch_load_data", _test_component_fetch_load_data, "LoadMLComponent _fetch_load_data method"), + ("component_publish_entity", _test_component_publish_entity, "LoadMLComponent _publish_entity method"), + ] + + failed_tests = [] + passed_count = 0 + + for name, test_func, description in sub_tests: + try: + print(f" Running {name}: {description}...", end=" ") + test_func() + print("PASS") + passed_count += 1 + except Exception as e: + print(f"FAIL: {e}") + import traceback + + traceback.print_exc() + failed_tests.append((name, str(e))) + + print(f"\nML Load Forecaster Tests: {passed_count}/{len(sub_tests)} passed") + if failed_tests: + print("Failed tests:") + for name, error in failed_tests: + print(f" - {name}: {error}") + assert False, f"ML Load Forecaster: {len(failed_tests)} tests failed" + + +def _test_relu_functions(): + """Test ReLU activation and derivative""" + # Test ReLU + x = np.array([-2, -1, 0, 1, 2]) + expected = np.array([0, 0, 0, 1, 2]) + result = relu(x) + assert np.allclose(result, expected), f"ReLU output mismatch: {result} vs {expected}" + + # Test ReLU derivative + expected_deriv = np.array([0, 0, 0, 1, 1]) + result_deriv = relu_derivative(x) + assert np.allclose(result_deriv, expected_deriv), f"ReLU derivative mismatch: {result_deriv} vs {expected_deriv}" + + +def _test_huber_loss_functions(): + """Test Huber loss computation""" + # Test with small error (L2 region) + y_true = np.array([[1.0, 2.0, 3.0]]) + y_pred = np.array([[1.1, 2.1, 3.1]]) # Error = 0.1 + loss = huber_loss(y_true, y_pred, delta=1.0) + # For small errors, Huber is 0.5 * error^2 + expected = 0.5 * (0.1**2) + assert abs(loss - expected) < 0.01, f"Huber loss for small error: expected {expected}, got {loss}" + + # Test with large error (L1 region) + y_pred_large = np.array([[3.0, 4.0, 5.0]]) # Error = 2.0 + loss_large = huber_loss(y_true, y_pred_large, delta=1.0) + # For large errors, Huber is delta * (|error| - 0.5 * delta) + expected_large = 1.0 * (2.0 - 0.5) + assert abs(loss_large - expected_large) < 0.1, f"Huber loss for large error: expected {expected_large}, got {loss_large}" + + +def _test_forward_pass(): + """Test that forward pass produces expected output shape and values""" + predictor = LoadPredictor(learning_rate=0.001) + + # Initialize weights + predictor._initialize_weights() + + # Create test input: batch of 2, with TOTAL_FEATURES features + X = np.random.randn(2, TOTAL_FEATURES).astype(np.float32) + + # Forward pass + output, activations, pre_activations = predictor._forward(X) + + # Check output shape: should be (batch_size, OUTPUT_STEPS) + assert output.shape == (2, OUTPUT_STEPS), f"Expected output shape (2, {OUTPUT_STEPS}), got {output.shape}" + + # Check that output is finite + assert np.all(np.isfinite(output)), "Forward pass produced non-finite values" + + # Check activations structure + assert len(activations) == len(HIDDEN_SIZES) + 2, "Wrong number of activations" + assert len(pre_activations) == len(HIDDEN_SIZES) + 1, "Wrong number of pre-activations" + + +def _test_backward_pass(): + """Test that backward pass produces gradients with correct shapes""" + predictor = LoadPredictor(learning_rate=0.001) + predictor._initialize_weights() + + # Forward pass + np.random.seed(42) + X = np.random.randn(4, TOTAL_FEATURES).astype(np.float32) + y_true = np.random.randn(4, OUTPUT_STEPS).astype(np.float32) + + output, activations, pre_activations = predictor._forward(X) + + # Backward pass + weight_grads, bias_grads = predictor._backward(y_true, activations, pre_activations) + + # Check that gradients exist for all weight layers + assert len(weight_grads) == len(HIDDEN_SIZES) + 1, "Wrong number of weight gradients" + assert len(bias_grads) == len(HIDDEN_SIZES) + 1, "Wrong number of bias gradients" + + # Check gradient shapes match weight shapes + for i, (w_grad, w) in enumerate(zip(weight_grads, predictor.weights)): + assert w_grad.shape == w.shape, f"Weight gradient {i} shape mismatch: {w_grad.shape} vs {w.shape}" + + for i, (b_grad, b) in enumerate(zip(bias_grads, predictor.biases)): + assert b_grad.shape == b.shape, f"Bias gradient {i} shape mismatch: {b_grad.shape} vs {b.shape}" + + +def _test_cyclical_features(): + """Test cyclical time feature encoding""" + predictor = LoadPredictor() + + # Test midnight (minute 0) + features = predictor._create_time_features(0, 0) + assert len(features) == 4, "Should have 4 time features" + assert abs(features[0] - 0.0) < 1e-6, "Midnight sin should be 0" + assert abs(features[1] - 1.0) < 1e-6, "Midnight cos should be 1" + + # Test noon (minute 720) + features = predictor._create_time_features(720, 0) + assert abs(features[0] - 0.0) < 1e-6, "Noon sin should be 0" + assert abs(features[1] - (-1.0)) < 1e-6, "Noon cos should be -1" + + # Test 6 AM (minute 360) - sin should be 1, cos should be 0 + features = predictor._create_time_features(360, 0) + assert abs(features[0] - 1.0) < 1e-6, "6 AM sin should be 1" + assert abs(features[1] - 0.0) < 1e-6, "6 AM cos should be 0" + + # Test Monday (dow 0) vs Thursday (dow 3) + features_mon = predictor._create_time_features(0, 0) + features_thu = predictor._create_time_features(0, 3) + assert features_mon[2] != features_thu[2], "Different days should have different encodings" + + +def _test_load_to_energy(): + """Test conversion of cumulative load to energy per step""" + predictor = LoadPredictor() + + # Create synthetic cumulative load data + # Cumulative: minute 0 = 10, minute 5 = 9, minute 10 = 8, etc. + load_minutes = {0: 10.0, 5: 9.0, 10: 8.0, 15: 7.5, 20: 7.0} + + energy_per_step = predictor._load_to_energy_per_step(load_minutes) + + # Energy from 0-5: 10 - 9 = 1 + assert abs(energy_per_step.get(0, -1) - 1.0) < 1e-6, "Energy 0-5 should be 1.0" + # Energy from 5-10: 9 - 8 = 1 + assert abs(energy_per_step.get(5, -1) - 1.0) < 1e-6, "Energy 5-10 should be 1.0" + # Energy from 10-15: 8 - 7.5 = 0.5 + assert abs(energy_per_step.get(10, -1) - 0.5) < 1e-6, "Energy 10-15 should be 0.5" + # Energy from 15-20: 7.5 - 7 = 0.5 + assert abs(energy_per_step.get(15, -1) - 0.5) < 1e-6, "Energy 15-20 should be 0.5" + + +def _test_pv_energy_conversion(): + """Test conversion of PV data including future forecasts (negative minutes)""" + predictor = LoadPredictor() + + # Create PV data with both historical (positive) and future (negative) minutes + # Historical: minute 0-20 (backwards in time) + # Future: minute -5 to -20 (forward in time) + pv_minutes = { + # Historical (cumulative decreasing as we go back in time) + 0: 10.0, + 5: 9.0, + 10: 8.0, + 15: 7.0, + 20: 6.0, + # Future forecasts (cumulative increasing as we go forward) + -5: 11.0, + -10: 12.5, + -15: 14.0, + -20: 15.0, + } + + pv_energy_per_step = predictor._load_to_energy_per_step(pv_minutes) + + # Historical energy (positive minutes, going backwards) + # Energy from 0-5: 10 - 9 = 1 + assert abs(pv_energy_per_step.get(0, -1) - 1.0) < 1e-6, "PV energy 0-5 should be 1.0" + # Energy from 5-10: 9 - 8 = 1 + assert abs(pv_energy_per_step.get(5, -1) - 1.0) < 1e-6, "PV energy 5-10 should be 1.0" + + # Future energy (negative minutes, going forward) + # Energy from -20 to -15: 15.0 - 14.0 = 1.0 + assert abs(pv_energy_per_step.get(-20, -1) - 1.0) < 1e-6, f"PV future energy -20 to -15 should be 1.0, got {pv_energy_per_step.get(-20, -1)}" + # Energy from -15 to -10: 14.0 - 12.5 = 1.5 + assert abs(pv_energy_per_step.get(-15, -1) - 1.5) < 1e-6, f"PV future energy -15 to -10 should be 1.5, got {pv_energy_per_step.get(-15, -1)}" + # Energy from -10 to -5: 12.5 - 11.0 = 1.5 + assert abs(pv_energy_per_step.get(-10, -1) - 1.5) < 1e-6, f"PV future energy -10 to -5 should be 1.5, got {pv_energy_per_step.get(-10, -1)}" + # Energy from -5 to 0: 11.0 - 10.0 = 1.0 + assert abs(pv_energy_per_step.get(-5, -1) - 1.0) < 1e-6, f"PV future energy -5 to 0 should be 1.0, got {pv_energy_per_step.get(-5, -1)}" + + +def _create_synthetic_pv_data(n_days=7, now_utc=None, forecast_hours=48): + """Create synthetic PV data for testing (historical + forecast)""" + if now_utc is None: + now_utc = datetime.now(timezone.utc) + + pv_minutes = {} + cumulative = 0.0 + + # Historical PV (positive minutes, backwards from now) + n_minutes = n_days * 24 * 60 + # Start from a multiple of STEP_MINUTES and go down to 0 + start_minute = (n_minutes // STEP_MINUTES) * STEP_MINUTES + for minute in range(start_minute, -STEP_MINUTES, -STEP_MINUTES): + dt = now_utc - timedelta(minutes=minute) + hour = dt.hour + + # PV generation pattern: 0 at night, peak at midday + if 6 <= hour < 18: + # Peak around noon (hour 12) + hour_offset = abs(hour - 12) + energy = max(0, 0.5 - hour_offset * 0.08 + 0.05 * np.random.randn()) + else: + energy = 0.0 + + energy = max(0, energy) + cumulative += energy + pv_minutes[minute] = cumulative + + # Future PV forecast (negative minutes, forward from now) + forecast_cumulative = pv_minutes[0] # Start from current cumulative + for step in range(1, (forecast_hours * 60 // STEP_MINUTES) + 1): + minute = -step * STEP_MINUTES + dt = now_utc + timedelta(minutes=step * STEP_MINUTES) + hour = dt.hour + + # Same pattern for forecast + if 6 <= hour < 18: + hour_offset = abs(hour - 12) + energy = max(0, 0.5 - hour_offset * 0.08 + 0.05 * np.random.randn()) + else: + energy = 0.0 + + energy = max(0, energy) + forecast_cumulative += energy + pv_minutes[minute] = forecast_cumulative + + return pv_minutes + + +def _create_synthetic_temp_data(n_days=7, now_utc=None, forecast_hours=48): + """Create synthetic temperature data for testing (historical + forecast)""" + if now_utc is None: + now_utc = datetime.now(timezone.utc) + + temp_minutes = {} + + # Historical temperature (positive minutes, backwards from now) + n_minutes = n_days * 24 * 60 + start_minute = (n_minutes // STEP_MINUTES) * STEP_MINUTES + for minute in range(start_minute, -STEP_MINUTES, -STEP_MINUTES): + dt = now_utc - timedelta(minutes=minute) + hour = dt.hour + dt.minute / 60.0 # Fractional hour for smooth variation + + # Smooth sinusoidal daily temperature pattern + # Temperature peaks around 1pm (hour 13) and minimum around 1am (hour 1) + # Using cosine wave shifted so maximum is at hour 13 + hours_since_peak = (hour - 13.0) % 24.0 + daily_cycle = np.cos(2 * np.pi * hours_since_peak / 24.0) + + # Base temp 6°C, amplitude 4°C, so range is 2°C to 10°C + # Add small multi-day variation (0.5°C amplitude over 3-day cycle) + day_num = minute / (24 * 60) + multi_day_variation = 0.5 * np.sin(2 * np.pi * day_num / 3.0) + + temp = 6.0 + 4.0 * daily_cycle + multi_day_variation + + temp = max(-10.0, min(40.0, temp)) # Reasonable bounds + temp_minutes[minute] = temp + + # Future temperature forecast (negative minutes, forward from now) + for step in range(1, (forecast_hours * 60 // STEP_MINUTES) + 1): + minute = -step * STEP_MINUTES + dt = now_utc + timedelta(minutes=step * STEP_MINUTES) + hour = dt.hour + dt.minute / 60.0 # Fractional hour for smooth variation + + # Same smooth pattern for forecast + hours_since_peak = (hour - 13.0) % 24.0 + daily_cycle = np.cos(2 * np.pi * hours_since_peak / 24.0) + + # Continue the multi-day variation into the future + day_num = -minute / (24 * 60) # Negative minute means future + multi_day_variation = 0.5 * np.sin(2 * np.pi * day_num / 3.0) + + temp = 6.0 + 4.0 * daily_cycle + multi_day_variation + + temp = max(-10.0, min(40.0, temp)) + temp_minutes[minute] = temp + + return temp_minutes + + +def _create_synthetic_load_data(n_days=7, now_utc=None): + """Create synthetic load data for testing""" + if now_utc is None: + now_utc = datetime.now(timezone.utc) + + n_minutes = n_days * 24 * 60 + load_minutes = {} + cumulative = 0.0 + + # Build backwards from now (minute 0 = now) + # Start from a multiple of STEP_MINUTES and go down to 0 + start_minute = (n_minutes // STEP_MINUTES) * STEP_MINUTES + for minute in range(start_minute, -STEP_MINUTES, -STEP_MINUTES): + # Time for this minute + dt = now_utc - timedelta(minutes=minute) + hour = dt.hour + + # Simple daily pattern: higher during day + if 6 <= hour < 22: + energy = 0.2 + 0.1 * np.random.randn() # ~0.2 kWh per 5 min during day + else: + energy = 0.05 + 0.02 * np.random.randn() # ~0.05 kWh at night + + energy = max(0, energy) + cumulative += energy + load_minutes[minute] = cumulative + + return load_minutes + + +def _test_dataset_creation(): + """Test dataset creation from load minute data with train/val split""" + predictor = LoadPredictor() + now_utc = datetime.now(timezone.utc) + + # Create synthetic load data: 7 days + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + + # Create dataset - now returns 5 values (train + val split) + X_train, y_train, train_weights, X_val, y_val = predictor._create_dataset(load_data, now_utc, time_decay_days=7) + + # Should have valid training samples + assert X_train is not None, "Training X should not be None" + assert X_train.shape[0] > 0, "Training should have samples" + assert X_train.shape[0] == y_train.shape[0], "X_train and y_train should have same number of samples" + assert train_weights.shape[0] == X_train.shape[0], "Train weights should match training samples" + + # Should have validation samples + assert X_val is not None, "Validation X should not be None" + assert X_val.shape[0] > 0, "Validation should have samples" + assert X_val.shape[0] == y_val.shape[0], "X_val and y_val should have same number of samples" + + # Feature dimension: TOTAL_FEATURES + assert X_train.shape[1] == TOTAL_FEATURES, f"Expected {TOTAL_FEATURES} features, got {X_train.shape[1]}" + + # Output dimension: OUTPUT_STEPS (1 for autoregressive) + assert y_train.shape[1] == OUTPUT_STEPS, f"Expected {OUTPUT_STEPS} outputs, got {y_train.shape[1]}" + + # Validation should be approximately 24h worth of samples (288 at 5-min intervals) + expected_val_samples = 24 * 60 // STEP_MINUTES + assert abs(X_val.shape[0] - expected_val_samples) < 10, f"Expected ~{expected_val_samples} val samples, got {X_val.shape[0]}" + + +def _test_dataset_with_pv(): + """Test dataset creation includes PV features correctly""" + predictor = LoadPredictor() + # Use a fixed daytime hour to ensure PV generation + now_utc = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc) # Noon on summer day + + # Create synthetic load and PV data + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + pv_data = _create_synthetic_pv_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only for training + + # Create dataset with PV data + X_train, y_train, train_weights, X_val, y_val = predictor._create_dataset(load_data, now_utc, pv_minutes=pv_data, time_decay_days=7) + + # Should have valid samples + assert X_train is not None, "Training X should not be None" + assert X_train.shape[0] > 0, "Training should have samples" + + # Feature dimension should include PV features: LOOKBACK_STEPS (load) + LOOKBACK_STEPS (PV) + LOOKBACK_STEPS (temp) + 4 (time) = TOTAL_FEATURES + from load_predictor import NUM_LOAD_FEATURES, NUM_PV_FEATURES, NUM_TEMP_FEATURES, NUM_TIME_FEATURES + + expected_features = NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES + NUM_TIME_FEATURES + assert X_train.shape[1] == expected_features, f"Expected {expected_features} features with PV, got {X_train.shape[1]}" + assert X_train.shape[1] == TOTAL_FEATURES, f"TOTAL_FEATURES should be {expected_features}, is {TOTAL_FEATURES}" + + # Verify PV features are not all zeros (unless no PV data provided) + # PV features are in the middle section: indices NUM_LOAD_FEATURES to NUM_LOAD_FEATURES+NUM_PV_FEATURES + pv_feature_section = X_train[:, NUM_LOAD_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES] + # At least some PV values should be non-zero (during daylight hours) + assert np.any(pv_feature_section > 0), "PV features should contain some non-zero values" + + # Temperature features should be all zeros since we didn't provide temp_minutes + temp_feature_section = X_train[:, NUM_LOAD_FEATURES + NUM_PV_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES] + assert np.all(temp_feature_section == 0), "Temperature features should be zero when no temp data provided" + + +def _test_dataset_with_temp(): + """Test dataset creation includes temperature features correctly""" + predictor = LoadPredictor() + now_utc = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc) + + # Create synthetic load and temperature data + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + temp_data = _create_synthetic_temp_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only + + # Create dataset with temperature data + X_train, y_train, train_weights, X_val, y_val = predictor._create_dataset(load_data, now_utc, temp_minutes=temp_data, time_decay_days=7) + + # Should have valid samples + assert X_train is not None, "Training X should not be None" + assert X_train.shape[0] > 0, "Training should have samples" + + # Feature dimension should include temperature features + from load_predictor import NUM_LOAD_FEATURES, NUM_PV_FEATURES, NUM_TEMP_FEATURES, NUM_TIME_FEATURES + + expected_features = NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES + NUM_TIME_FEATURES + assert X_train.shape[1] == expected_features, f"Expected {expected_features} features with temp, got {X_train.shape[1]}" + assert X_train.shape[1] == TOTAL_FEATURES, f"TOTAL_FEATURES should be {expected_features}, is {TOTAL_FEATURES}" + + # Verify temperature features are not all zeros + # Temperature features are after load and PV: indices NUM_LOAD_FEATURES+NUM_PV_FEATURES to NUM_LOAD_FEATURES+NUM_PV_FEATURES+NUM_TEMP_FEATURES + temp_feature_section = X_train[:, NUM_LOAD_FEATURES + NUM_PV_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES] + # At least some temperature values should be non-zero + assert np.any(temp_feature_section != 0), "Temperature features should contain non-zero values" + # Check temperature values are in reasonable range (after normalization they won't be in Celsius range) + assert np.min(temp_feature_section) > -50, "Temperature features should be reasonable" + assert np.max(temp_feature_section) < 50, "Temperature features should be reasonable" + + # PV features should be all zeros since we didn't provide pv_minutes + pv_feature_section = X_train[:, NUM_LOAD_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES] + assert np.all(pv_feature_section == 0), "PV features should be zero when no PV data provided" + + +def _test_normalization(): + """Test Z-score normalization correctness""" + predictor = LoadPredictor() + + # Create test data + np.random.seed(42) + X = np.random.randn(100, TOTAL_FEATURES).astype(np.float32) * 10 + 5 # Mean ~5, std ~10 + + # Normalize with fit + X_norm = predictor._normalize_features(X, fit=True) + + # Check mean ~0 and std ~1 along each feature + assert np.allclose(np.mean(X_norm, axis=0), 0, atol=0.1), "Normalized mean should be ~0" + assert np.allclose(np.std(X_norm, axis=0), 1, atol=0.1), "Normalized std should be ~1" + + # Test target normalization + y = np.random.randn(100, OUTPUT_STEPS).astype(np.float32) * 2 + 3 + y_norm = predictor._normalize_targets(y, fit=True) + + # Check denormalization + y_denorm = predictor._denormalize_predictions(y_norm) + assert np.allclose(y, y_denorm, atol=1e-5), "Denormalization should recover original" + + +def _test_adam_optimizer(): + """Test Adam optimizer update step""" + predictor = LoadPredictor(learning_rate=0.01) + predictor._initialize_weights() + + # Store original weights + orig_weight = predictor.weights[0].copy() + + # Create dummy gradients + weight_grads = [np.ones_like(w) * 0.1 for w in predictor.weights] + bias_grads = [np.ones_like(b) * 0.1 for b in predictor.biases] + + # Perform Adam update + predictor._adam_update(weight_grads, bias_grads) + + # Weight should have changed + assert not np.allclose(orig_weight, predictor.weights[0]), "Adam update should change weights" + + # adam_t should have incremented + assert predictor.adam_t == 1, "Adam timestep should be 1" + + +def _test_training_convergence(): + """Test that training converges on simple synthetic data""" + predictor = LoadPredictor(learning_rate=0.01) + now_utc = datetime.now(timezone.utc) + + # Create simple repeating daily pattern + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + + # Train with few epochs + val_mae = predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=10, time_decay_days=7) + + # Training should complete and return a validation MAE + assert val_mae is not None, "Training should return validation MAE" + assert predictor.model_initialized, "Model should be initialized after training" + assert predictor.epochs_trained > 0, "Should have trained some epochs" + + +def _test_training_with_pv(): + """Test that training works correctly with PV input features""" + predictor = LoadPredictor(learning_rate=0.01) + now_utc = datetime.now(timezone.utc) + + # Create load and PV data + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + pv_data = _create_synthetic_pv_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only for training + + # Train with PV data + val_mae = predictor.train(load_data, now_utc, pv_minutes=pv_data, is_initial=True, epochs=10, time_decay_days=7) + + # Training should complete successfully + assert val_mae is not None, "Training with PV should return validation MAE" + assert predictor.model_initialized, "Model should be initialized after training with PV" + assert predictor.epochs_trained > 0, "Should have trained some epochs with PV data" + + # Verify the model can accept correct input size (with PV features) + test_input = np.random.randn(1, TOTAL_FEATURES).astype(np.float32) + output, _, _ = predictor._forward(test_input) + assert output.shape == (1, OUTPUT_STEPS), "Model should produce correct output shape with PV features" + + +def _test_training_with_temp(): + """Test that training works correctly with temperature input features""" + predictor = LoadPredictor(learning_rate=0.01) + now_utc = datetime.now(timezone.utc) + + # Create load and temperature data + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + temp_data = _create_synthetic_temp_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only for training + + # Train with temperature data + val_mae = predictor.train(load_data, now_utc, temp_minutes=temp_data, is_initial=True, epochs=10, time_decay_days=7) + + # Training should complete successfully + assert val_mae is not None, "Training with temperature should return validation MAE" + assert predictor.model_initialized, "Model should be initialized after training with temperature" + assert predictor.epochs_trained > 0, "Should have trained some epochs with temperature data" + + # Verify the model can accept correct input size (with temperature features) + test_input = np.random.randn(1, TOTAL_FEATURES).astype(np.float32) + output, _, _ = predictor._forward(test_input) + assert output.shape == (1, OUTPUT_STEPS), "Model should produce correct output shape with temperature features" + + +def _test_model_persistence(): + """Test model save/load with version check""" + predictor = LoadPredictor(learning_rate=0.005) + now_utc = datetime.now(timezone.utc) + + # Train briefly + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=5, now_utc=now_utc) + predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=5, time_decay_days=7) + + # Save to temp file + with tempfile.NamedTemporaryFile(suffix=".npz", delete=False) as f: + temp_path = f.name + + try: + predictor.save(temp_path) + + # Load into new predictor + predictor2 = LoadPredictor(learning_rate=0.005) + success = predictor2.load(temp_path) + + assert success, "Model load should succeed" + assert predictor2.model_initialized, "Loaded model should be marked as initialized" + + # Compare weights + for w1, w2 in zip(predictor.weights, predictor2.weights): + assert np.allclose(w1, w2), "Weights should match after load" + + # Test prediction produces same result + np.random.seed(123) + test_input = np.random.randn(1, TOTAL_FEATURES).astype(np.float32) + out1, _, _ = predictor._forward(test_input) + out2, _, _ = predictor2._forward(test_input) + assert np.allclose(out1, out2), "Predictions should match after load" + + finally: + if os.path.exists(temp_path): + os.unlink(temp_path) + + +def _test_cold_start(): + """Test cold start with insufficient data returns None""" + predictor = LoadPredictor() + now_utc = datetime.now(timezone.utc) + + # Only 1 day of data (insufficient for 48h horizon + lookback) + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=1, now_utc=now_utc) + + # Training should fail or return None + val_mae = predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=5, time_decay_days=7) + + # With only 1 day of data, we can't create a valid dataset for 48h prediction + # The result depends on actual data coverage + # Just verify it doesn't crash + assert True, "Cold start should not crash" + + +def _test_fine_tune(): + """Test fine-tuning on recent data only""" + predictor = LoadPredictor(learning_rate=0.01) + now_utc = datetime.now(timezone.utc) + + # Initial training on 7 days + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=5, time_decay_days=7) + + # Store original weights + orig_weights = [w.copy() for w in predictor.weights] + + # Fine-tune with same data but as fine-tune mode + # Note: Fine-tune uses is_finetune=True which only looks at last 24h + # For the test to work, we need enough data for the full training + predictor.train(load_data, now_utc, pv_minutes=None, is_initial=False, epochs=3, time_decay_days=7) + + # Even if fine-tune has insufficient data, initial training should have worked + # The test validates that fine-tune doesn't crash and model is still valid + assert predictor.model_initialized, "Model should still be initialized after fine-tune attempt" + + +def _test_prediction(): + """Test end-to-end prediction""" + predictor = LoadPredictor(learning_rate=0.01) + now_utc = datetime.now(timezone.utc) + midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + + # Train on synthetic data + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=10, time_decay_days=7) + + # Make prediction + predictions = predictor.predict(load_data, now_utc, midnight_utc, pv_minutes=None) + + # Should return dict with minute keys + if predictions: # May return empty dict if validation fails + assert isinstance(predictions, dict), "Predictions should be a dict" + # Check some predictions exist + assert len(predictions) > 0, "Should have some predictions" + # All values should be non-negative + for minute, val in predictions.items(): + assert val >= 0, f"Prediction at minute {minute} should be non-negative" + + +def _test_prediction_with_pv(): + """Test end-to-end prediction with PV forecast data""" + predictor = LoadPredictor(learning_rate=0.01) + now_utc = datetime.now(timezone.utc) + midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + + # Create load and PV data (with 48h forecast) + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + pv_data = _create_synthetic_pv_data(n_days=7, now_utc=now_utc, forecast_hours=48) # Include forecast + + # Train with PV data + predictor.train(load_data, now_utc, pv_minutes=pv_data, is_initial=True, epochs=10, time_decay_days=7) + + # Make prediction with PV forecast + predictions = predictor.predict(load_data, now_utc, midnight_utc, pv_minutes=pv_data) + + # Should return predictions + if predictions: + assert isinstance(predictions, dict), "Predictions should be a dict" + assert len(predictions) > 0, "Should have predictions with PV data" + + # Verify all values are non-negative + for minute, val in predictions.items(): + assert val >= 0, f"Prediction at minute {minute} should be non-negative" + + # Verify predictions span 48 hours (576 steps at 5-min intervals) + max_minute = max(predictions.keys()) + assert max_minute >= 2800, f"Predictions should span ~48h (2880 min), got {max_minute} min" + + +def _test_prediction_with_temp(): + """Test end-to-end prediction with temperature forecast data""" + predictor = LoadPredictor(learning_rate=0.01) + now_utc = datetime.now(timezone.utc) + midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + + # Create load and temperature data (with 48h forecast) + np.random.seed(42) + load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc) + temp_data = _create_synthetic_temp_data(n_days=7, now_utc=now_utc, forecast_hours=48) # Include forecast + + # Train with temperature data + predictor.train(load_data, now_utc, temp_minutes=temp_data, is_initial=True, epochs=10, time_decay_days=7) + + # Make prediction with temperature forecast + predictions = predictor.predict(load_data, now_utc, midnight_utc, temp_minutes=temp_data) + + # Should return predictions + if predictions: + assert isinstance(predictions, dict), "Predictions should be a dict" + assert len(predictions) > 0, "Should have predictions with temperature data" + + # Verify all values are non-negative + for minute, val in predictions.items(): + assert val >= 0, f"Prediction at minute {minute} should be non-negative" + + # Verify predictions span 48 hours (576 steps at 5-min intervals) + max_minute = max(predictions.keys()) + assert max_minute >= 2800, f"Predictions should span ~48h (2880 min), got {max_minute} min" + + +def _test_real_data_training(): + """ + Test training on real load_minutes_debug.json data and generate comparison chart + """ + import json + import os + + # Try to load the input_train_data.json which has real PV and temperature + input_train_paths = ["../coverage/input_train_data.json", "coverage/input_train_data.json", "input_train_data.json"] + + load_data = None + pv_data = None + temp_data = None + + for json_path in input_train_paths: + if os.path.exists(json_path): + with open(json_path, "r") as f: + train_data = json.load(f) + # Format: [load_minutes_new, age_days, load_minutes_now, pv_data, temperature_data] + if len(train_data) >= 5: + # Convert string keys to integers + load_data = {int(k): float(v) for k, v in train_data[0].items()} + pv_data = {int(k): float(v) for k, v in train_data[3].items()} if train_data[3] else {} + temp_data = {int(k): float(v) for k, v in train_data[4].items()} if train_data[4] else {} + print(f" Loaded training data from {json_path}") + print(f" Load: {len(load_data)} datapoints") + print(f" PV: {len(pv_data)} datapoints") + print(f" Temperature: {len(temp_data)} datapoints") + break + + if load_data is None: + print(" WARNING: No training data found, skipping real data test") + return + + # Initialize predictor with lower learning rate for better convergence + predictor = LoadPredictor(learning_rate=0.0005, max_load_kw=20.0) + now_utc = datetime.now(timezone.utc) + midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + + # Calculate how many days of data we have + max_minute = max(load_data.keys()) + n_days = max_minute / (24 * 60) + print(f" Data spans {n_days:.1f} days ({max_minute} minutes)") + + # Generate synthetic data only if real data wasn't loaded + if pv_data is None or len(pv_data) == 0: + print(f" Generating synthetic PV data for {n_days:.1f} days...") + pv_data = _create_synthetic_pv_data(n_days=int(n_days) + 1, now_utc=now_utc, forecast_hours=48) + print(f" Generated {len(pv_data)} PV datapoints") + + if temp_data is None or len(temp_data) == 0: + print(f" Generating synthetic temperature data for {n_days:.1f} days...") + temp_data = _create_synthetic_temp_data(n_days=int(n_days) + 1, now_utc=now_utc, forecast_hours=48) + print(f" Generated {len(temp_data)} temperature datapoints") + + # Train on full dataset with more epochs for larger network + data_source = "real" if (pv_data and len(pv_data) > 100 and temp_data and len(temp_data) > 100) else "synthetic" + print(f" Training on real load + {data_source} PV/temperature with {len(load_data)} points...") + success = predictor.train(load_data, now_utc, pv_minutes=pv_data, temp_minutes=temp_data, is_initial=True, epochs=50, time_decay_days=7) + + assert success, "Training on real data should succeed" + assert predictor.model_initialized, "Model should be initialized after training" + + # Make predictions + print(" Generating predictions with PV + temperature forecasts...") + predictions = predictor.predict(load_data, now_utc, midnight_utc, pv_minutes=pv_data, temp_minutes=temp_data) + + assert isinstance(predictions, dict), "Predictions should be a dict" + assert len(predictions) > 0, "Should have predictions" + + print(f" Generated {len(predictions)} predictions") + + # Create comparison chart using matplotlib + try: + import matplotlib + + matplotlib.use("Agg") # Non-interactive backend + import matplotlib.pyplot as plt + + # Chart layout: 7 days of history (negative hours) + 2 days of predictions (positive hours) + # X-axis: -168 to +48 hours (0 = now) + history_hours = 7 * 24 # 7 days back + prediction_hours = 48 # 2 days forward + + # Convert historical load_data (cumulative kWh) to energy per 5-min step (kWh) + # Going backwards in time: minute 0 is now, higher minutes are past + historical_minutes = [] + historical_energy = [] + max_history_minutes = min(history_hours * 60, max_minute) + + for minute in range(0, max_history_minutes, STEP_MINUTES): + if minute in load_data and (minute + STEP_MINUTES) in load_data: + energy_kwh = max(0, load_data[minute] - load_data.get(minute + STEP_MINUTES, load_data[minute])) + historical_minutes.append(minute) + historical_energy.append(energy_kwh) + + # Extract validation period actual data (most recent 24h = day 7) + # This is the data the model was validated against + val_actual_minutes = [] + val_actual_energy = [] + val_period_hours = 24 # Most recent 24h + for minute in range(0, val_period_hours * 60, STEP_MINUTES): + if minute in load_data and (minute + STEP_MINUTES) in load_data: + energy_kwh = max(0, load_data[minute] - load_data.get(minute + STEP_MINUTES, load_data[minute])) + val_actual_minutes.append(minute) + val_actual_energy.append(energy_kwh) + + # Generate validation predictions: what would the model predict for day 7 + # using only data from day 2-7 (excluding most recent 24h)? + # Simulate predicting from 24h ago + val_pred_minutes = [] + val_pred_energy = [] + + # Create a modified load_data that excludes the most recent 24h + # This simulates predicting "yesterday" from "2 days ago" + val_holdout_minutes = val_period_hours * 60 + shifted_load_data = {} + for minute, cum_kwh in load_data.items(): + if minute >= val_holdout_minutes: + # Shift back by 24h so model predicts into "held out" period + shifted_load_data[minute - val_holdout_minutes] = cum_kwh + + # Make validation prediction (predict next 24h from shifted data) + if shifted_load_data: + shifted_now = now_utc - timedelta(hours=val_period_hours) + shifted_midnight = shifted_now.replace(hour=0, minute=0, second=0, microsecond=0) + + # Create shifted PV data for validation prediction + shifted_pv_data = {} + for minute, cum_kwh in pv_data.items(): + if minute >= val_holdout_minutes: + shifted_pv_data[minute - val_holdout_minutes] = cum_kwh + + # Create shifted temperature data for validation prediction + shifted_temp_data = {} + for minute, temp in temp_data.items(): + if minute >= val_holdout_minutes: + shifted_temp_data[minute - val_holdout_minutes] = temp + + val_predictions = predictor.predict(shifted_load_data, shifted_now, shifted_midnight, pv_minutes=shifted_pv_data, temp_minutes=shifted_temp_data) + + # Extract first 24h of validation predictions + val_pred_keys = sorted(val_predictions.keys()) + for i, minute in enumerate(val_pred_keys): + if minute >= val_period_hours * 60: + break + if i == 0: + energy_kwh = val_predictions[minute] + else: + prev_minute = val_pred_keys[i - 1] + energy_kwh = max(0, val_predictions[minute] - val_predictions[prev_minute]) + val_pred_minutes.append(minute) + val_pred_energy.append(energy_kwh) + + # Convert predictions (cumulative kWh) to energy per step (kWh) + # predictions dict is: {0: cum0, 5: cum5, 10: cum10, ...} representing FUTURE + pred_minutes = [] + pred_energy = [] + pred_keys = sorted(predictions.keys()) + for i, minute in enumerate(pred_keys): + if minute >= prediction_hours * 60: + break + if i == 0: + # First step - use the value directly as energy + energy_kwh = predictions[minute] + else: + # Subsequent steps - calculate difference from previous + prev_minute = pred_keys[i - 1] + energy_kwh = max(0, predictions[minute] - predictions[prev_minute]) + pred_minutes.append(minute) + pred_energy.append(energy_kwh) + + # Convert PV data to energy per step for plotting + # Historical PV (positive minutes, going back in time) + pv_historical_minutes = [] + pv_historical_energy = [] + for minute in range(0, max_history_minutes, STEP_MINUTES): + if minute in pv_data and (minute + STEP_MINUTES) in pv_data: + energy_kwh = max(0, pv_data[minute] - pv_data.get(minute + STEP_MINUTES, pv_data[minute])) + pv_historical_minutes.append(minute) + pv_historical_energy.append(energy_kwh) + + # Future PV forecasts (negative minutes in pv_data dict, representing future) + pv_forecast_minutes = [] + pv_forecast_energy = [] + for minute in range(-prediction_hours * 60, 0, STEP_MINUTES): + if minute in pv_data and (minute + STEP_MINUTES) in pv_data: + energy_kwh = max(0, pv_data[minute] - pv_data.get(minute + STEP_MINUTES, pv_data[minute])) + pv_forecast_minutes.append(minute) + pv_forecast_energy.append(energy_kwh) + + # Extract temperature data (non-cumulative, so we use raw values) + # Historical temperature (positive minutes in temp_data dict, going back in time) + temp_historical_minutes = [] + temp_historical_celsius = [] + for minute in range(0, max_history_minutes, STEP_MINUTES): + if minute in temp_data: + temp_celsius = temp_data[minute] + temp_historical_minutes.append(minute) + temp_historical_celsius.append(temp_celsius) + + # Future temperature forecasts (negative minutes in temp_data dict, representing future) + temp_forecast_minutes = [] + temp_forecast_celsius = [] + for minute in range(-prediction_hours * 60, 0, STEP_MINUTES): + if minute in temp_data: + temp_celsius = temp_data[minute] + temp_forecast_minutes.append(minute) + temp_forecast_celsius.append(temp_celsius) + + # Create figure with single plot showing timeline + fig, ax = plt.subplots(1, 1, figsize=(16, 6)) + + # Create secondary y-axis for temperature + ax2 = ax.twinx() + + # Plot PV data first (in background) + # Historical PV (negative hours, going back in time) + if pv_historical_minutes: + pv_hist_hours = [-m / 60 for m in pv_historical_minutes] # Negative for past + ax.plot(pv_hist_hours, pv_historical_energy, "orange", linewidth=0.8, label="Historical PV (7 days)", alpha=0.3, linestyle="--") + + # Future PV forecasts (positive hours, going forward) + if pv_forecast_minutes: + # Convert negative minutes to positive hours for future + pv_forecast_hours = [-m / 60 for m in pv_forecast_minutes] # Negative minutes become positive hours + ax.plot(pv_forecast_hours, pv_forecast_energy, "orange", linewidth=1.2, label="PV Forecast (48h)", alpha=0.5, linestyle="--") + + # Plot temperature data on secondary y-axis + # Historical temperature (negative hours, going back in time) + if temp_historical_minutes: + temp_hist_hours = [-m / 60 for m in temp_historical_minutes] # Negative for past + ax2.plot(temp_hist_hours, temp_historical_celsius, "purple", linewidth=0.8, label="Historical Temp (7 days)", alpha=0.4, linestyle="-.") + + # Future temperature forecasts (positive hours, going forward) + if temp_forecast_minutes: + # Convert negative minutes to positive hours for future + temp_forecast_hours = [-m / 60 for m in temp_forecast_minutes] # Negative minutes become positive hours + ax2.plot(temp_forecast_hours, temp_forecast_celsius, "purple", linewidth=1.2, label="Temp Forecast (48h)", alpha=0.6, linestyle="-.") + + # Plot historical data (negative hours, going back in time) + # minute 0 = now (hour 0), minute 60 = 1 hour ago (hour -1) + if historical_minutes: + hist_hours = [-m / 60 for m in historical_minutes] # Negative for past + ax.plot(hist_hours, historical_energy, "b-", linewidth=0.8, label="Historical Load (7 days)", alpha=0.5) + + # Highlight validation period actual data (most recent 24h) with thicker line + if val_actual_minutes: + val_actual_hours = [-m / 60 for m in val_actual_minutes] # Negative for past + ax.plot(val_actual_hours, val_actual_energy, "b-", linewidth=1.5, label="Actual Day 7 (validation)", alpha=0.9) + + # Plot validation predictions (what model predicted for day 7) + if val_pred_minutes: + # These predictions map to the validation period (most recent 24h) + # val_pred minute 0 -> actual minute 0 -> hour 0, etc. + val_pred_hours = [-m / 60 for m in val_pred_minutes] # Same position as actual + ax.plot(val_pred_hours, val_pred_energy, "g-", linewidth=1.5, label="ML Prediction (day 7)", alpha=0.9) + + # Plot future predictions (positive hours, going forward) + if pred_minutes: + pred_hours = [m / 60 for m in pred_minutes] # Positive for future + ax.plot(pred_hours, pred_energy, "r-", linewidth=1.5, label="ML Prediction (48h future)", alpha=0.9) + + # Add vertical line at "now" + ax.axvline(x=0, color="black", linestyle="--", linewidth=2, label="Now", alpha=0.8) + + # Shade the validation region (most recent 24h) + ax.axvspan(-24, 0, alpha=0.1, color="green", label="Validation Period") + + # Formatting + ax.set_xlabel("Hours (negative = past, positive = future)", fontsize=12) + ax.set_ylabel("Load (kWh per 5 min)", fontsize=12) + ax2.set_ylabel("Temperature (°C)", fontsize=12, color="purple") + ax2.tick_params(axis="y", labelcolor="purple") + ax.set_title("ML Load Predictor with PV + Temperature Input: Validation (Day 7) + 48h Forecast", fontsize=14, fontweight="bold") + + # Combine legends from both axes + lines1, labels1 = ax.get_legend_handles_labels() + lines2, labels2 = ax2.get_legend_handles_labels() + ax.legend(lines1 + lines2, labels1 + labels2, loc="upper right", fontsize=10) + ax.grid(True, alpha=0.3) + ax.set_xlim(-history_hours, prediction_hours) + + # Add day markers + for day in range(-7, 3): + hour = day * 24 + if -history_hours <= hour <= prediction_hours: + ax.axvline(x=hour, color="gray", linestyle=":", linewidth=0.5, alpha=0.5) + + plt.tight_layout() + + # Save to coverage directory + chart_paths = ["../coverage/ml_prediction_chart.png", "coverage/ml_prediction_chart.png", "ml_prediction_chart.png"] + for chart_path in chart_paths: + try: + plt.savefig(chart_path, dpi=150, bbox_inches="tight") + print(f" Chart saved to {chart_path}") + break + except: + continue + + plt.close() + + except ImportError: + print(" WARNING: matplotlib not available, skipping chart generation") + + +def _test_component_fetch_load_data(): + """Test LoadMLComponent._fetch_load_data method""" + import asyncio + from datetime import datetime, timezone + from load_ml_component import LoadMLComponent + from unittest.mock import MagicMock + + # Helper to run async tests + def run_async(coro): + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop.run_until_complete(coro) + + # Create mock base object with all necessary properties + class MockBase: + def __init__(self): + self.prefix = "predbat" + self.config_root = None + self.now_utc = datetime.now(timezone.utc) + self.midnight_utc = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) + self.minutes_now = (self.now_utc - self.midnight_utc).seconds // 60 + self.local_tz = timezone.utc + self.args = {} + self.log_messages = [] + + def log(self, msg): + self.log_messages.append(msg) + + def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None): + return { + "load_today": ["sensor.load_today"], + "load_power": None, # Disable load_power to simplify test + "car_charging_energy": None, # Disable car charging to simplify test + "load_scaling": 1.0, + "car_charging_energy_scale": 1.0, + }.get(key, default) + + def get_state_wrapper(self, entity_id, default=None, attribute=None, refresh=False, required_unit=None, raw=False): + """Mock get_state_wrapper - returns None for temperature by default""" + return default + + def fetch_pv_forecast(self): + """Mock fetch_pv_forecast - returns empty forecasts""" + return {}, {} + + # Create synthetic load data (28 days worth) + def create_load_minutes(days=28, all_minutes=False): + """ + Create cumulative load data going backwards from minute 0 + + Args: + days: Number of days of data to create + all_minutes: If True, create entries for every minute (not just 5-min intervals) + """ + load_data = {} + cumulative = 0.0 + + if all_minutes: + # Create entry for every minute (for car charging test) + for minute in range(days * 24 * 60, -1, -1): + energy_step = 0.1 / 5 # Scale down since we have 5x more entries + cumulative += energy_step + load_data[minute] = cumulative + else: + # Create entries at 5-minute intervals (normal case) + for minute in range(days * 24 * 60, -1, -5): + energy_step = 0.1 # 0.1 kWh per 5 min + cumulative += energy_step + load_data[minute] = cumulative + + return load_data, days + + # Test 1: Successful fetch with minimal config + async def test_basic_fetch(): + mock_base = MockBase() + load_data, age = create_load_minutes(28) + mock_base.minute_data_load = MagicMock(return_value=(load_data, age)) + mock_base.minute_data_import_export = MagicMock(return_value=None) + # Mock the fill_load_from_power method - it should just return the load_minutes unchanged + mock_base.fill_load_from_power = MagicMock(side_effect=lambda x, y: x) + + component = LoadMLComponent(mock_base, load_ml_enable=True) + # Override default values for testing + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is not None, "Should return load data" + assert result_age == 28, f"Expected 28 days, got {result_age}" + assert len(result_data) > 0, "Load data should not be empty" + assert result_now >= 0, f"Current load should be non-negative, got {result_now}" + print(" ✓ Basic fetch successful") + + # Test 2: Missing sensor (should return None) + async def test_missing_sensor(): + class MockBaseNoSensor: + def __init__(self): + self.prefix = "predbat" + self.config_root = None + self.now_utc = datetime.now(timezone.utc) + self.local_tz = timezone.utc + self.args = {} + + def log(self, msg): + pass + + def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None): + return default + + mock_base_no_sensor = MockBaseNoSensor() + + component = LoadMLComponent(mock_base_no_sensor, load_ml_enable=True) + # Override default values for testing + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is None, "Should return None when sensor missing" + assert result_age == 0, "Age should be 0 when sensor missing" + assert result_now == 0, "Current load should be 0 when sensor missing" + print(" ✓ Missing sensor handled correctly") + + # Test 3: Car charging subtraction + async def test_car_charging_subtraction(): + mock_base_with_car = MockBase() + + # Create load data with entries for EVERY minute (not just 5-min intervals) + # This is required because the component's car charging subtraction loop + # iterates over every minute from 1 to max_minute + original_load_data, age = create_load_minutes(7, all_minutes=True) + car_charging_data = {i: i * 0.001 for i in range(0, 7 * 24 * 60 + 1)} # Small cumulative car charging (0.001 kWh/min) + + # Override get_arg to enable car_charging_energy + def mock_get_arg_with_car(key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None): + return { + "load_today": ["sensor.load_today"], + "load_power": None, + "car_charging_energy": ["sensor.car_charging"], # Enable car charging + "load_scaling": 1.0, + "car_charging_energy_scale": 1.0, + }.get(key, default) + + mock_base_with_car.get_arg = mock_get_arg_with_car + + # Return a copy of the data so the original isn't modified + mock_base_with_car.minute_data_load = MagicMock(return_value=(dict(original_load_data), age)) + mock_base_with_car.minute_data_import_export = MagicMock(return_value=car_charging_data) + + component = LoadMLComponent(mock_base_with_car, load_ml_enable=True) + # Override default values for testing + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is not None, f"Should return load data" + assert result_age > 0, f"Should have valid age (got {result_age})" + assert len(result_data) > 0, "Result data should not be empty" + assert result_now >= 0, f"Current load should be non-negative, got {result_now}" + + # Verify car charging was called + assert mock_base_with_car.minute_data_import_export.called, "minute_data_import_export should be called" + + # Verify all values are non-negative after subtraction + for minute, value in result_data.items(): + assert value >= 0, f"Load at minute {minute} should be non-negative, got {value}" + + print(" ✓ Car charging subtraction works") + + # Test 4: Load power fill + async def test_load_power_fill(): + mock_base_with_power = MockBase() + + # Override get_arg to enable load_power + def mock_get_arg_with_power(key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None): + return { + "load_today": ["sensor.load_today"], + "load_power": ["sensor.load_power"], # Enable load_power + "car_charging_energy": None, + "load_scaling": 1.0, + "car_charging_energy_scale": 1.0, + }.get(key, default) + + mock_base_with_power.get_arg = mock_get_arg_with_power + + load_data, age = create_load_minutes(7) + load_power_data, _ = create_load_minutes(7) + + mock_base_with_power.minute_data_load = MagicMock(side_effect=[(load_data, age), (load_power_data, age)]) # First call for load_today # Second call for load_power + mock_base_with_power.minute_data_import_export = MagicMock(return_value=None) + mock_base_with_power.fill_load_from_power = MagicMock(return_value=load_data) + + component = LoadMLComponent(mock_base_with_power, load_ml_enable=True) + # Override default values for testing + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is not None, "Should return load data" + assert mock_base_with_power.fill_load_from_power.called, "fill_load_from_power should be called" + assert result_now >= 0, f"Current load should be non-negative, got {result_now}" + print(" ✓ Load power fill invoked") + + # Test 5: Exception handling + async def test_exception_handling(): + mock_base = MockBase() + mock_base.minute_data_load = MagicMock(side_effect=Exception("Test exception")) + + component = LoadMLComponent(mock_base, load_ml_enable=True) + # Override default values for testing + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is None, "Should return None on exception" + assert result_age == 0, "Age should be 0 on exception" + assert result_now == 0, "Current load should be 0 on exception" + print(" ✓ Exception handling works") + + # Test 6: Empty load data + async def test_empty_load_data(): + mock_base = MockBase() + mock_base.minute_data_load = MagicMock(return_value=(None, 0)) + mock_base.minute_data_import_export = MagicMock(return_value=None) + + component = LoadMLComponent(mock_base, load_ml_enable=True) + # Override default values for testing + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is None, "Should return None when load data is empty" + assert result_age == 0, "Age should be 0 when load data is empty" + assert result_now == 0, "Current load should be 0 when load data is empty" + print(" ✓ Empty load data handled correctly") + + # Test 7: Temperature data fetch with future predictions only + async def test_temperature_data_fetch(): + from datetime import timedelta + + mock_base_with_temp = MockBase() + + # Create mock temperature data (dict with timestamp strings as keys) + # This simulates future temperature predictions from sensor.predbat_temperature attribute "results" + base_time = mock_base_with_temp.now_utc + temp_predictions = {} + for hours_ahead in range(1, 49): # 48 hours of predictions + timestamp = base_time + timedelta(hours=hours_ahead) + timestamp_str = timestamp.strftime("%Y-%m-%dT%H:%M:%S%z") + temp_predictions[timestamp_str] = 15.0 + (hours_ahead % 12) # Simulated temperature pattern + + # Override get_state_wrapper using MagicMock to return temperature predictions + def mock_get_state_wrapper_side_effect(entity_id, default=None, attribute=None, refresh=False, required_unit=None, raw=False): + if entity_id == "sensor.predbat_temperature" and attribute == "results": + return temp_predictions + return default + + mock_base_with_temp.get_state_wrapper = MagicMock(side_effect=mock_get_state_wrapper_side_effect) + + load_data, age = create_load_minutes(7) + + # Mock minute_data_load to return load data + mock_base_with_temp.minute_data_load = MagicMock(return_value=(load_data, age)) + mock_base_with_temp.minute_data_import_export = MagicMock(return_value={}) + mock_base_with_temp.fill_load_from_power = MagicMock(side_effect=lambda x, y: x) + + component = LoadMLComponent(mock_base_with_temp, load_ml_enable=True) + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is not None, "Should return load data" + assert result_temp is not None, "Should return temperature data" + assert isinstance(result_temp, dict), "Temperature data should be a dict" + assert len(result_temp) > 0, "Temperature data should not be empty" + + # Verify we have future temperature data (positive minutes from midnight) + # Note: minute_data with backwards=False returns positive minute keys + # These represent minutes from midnight forward (future predictions) + assert len(result_temp) > 0, "Should have future temperature predictions" + + # Verify get_state_wrapper was called correctly + assert mock_base_with_temp.get_state_wrapper.called, "get_state_wrapper should be called" + + print(" ✓ Temperature data fetch (future predictions) works") + + # Test 8: Temperature data with no predictions (None return) + async def test_temperature_no_data(): + mock_base_no_temp = MockBase() + + load_data, age = create_load_minutes(7) + mock_base_no_temp.minute_data_load = MagicMock(return_value=(load_data, age)) + mock_base_no_temp.minute_data_import_export = MagicMock(return_value={}) + mock_base_no_temp.fill_load_from_power = MagicMock(side_effect=lambda x, y: x) + + # get_state_wrapper returns None (default behavior) + + component = LoadMLComponent(mock_base_no_temp, load_ml_enable=True) + component.ml_learning_rate = 0.001 + component.ml_epochs_initial = 10 + component.ml_epochs_update = 2 + component.ml_min_days = 1 + component.ml_validation_threshold = 2.0 + component.ml_time_decay_days = 7 + component.ml_max_load_kw = 23.0 + component.ml_max_model_age_hours = 48 + + result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data() + + assert result_data is not None, "Should return load data" + assert result_temp is not None, "Should return temperature data (empty dict)" + assert isinstance(result_temp, dict), "Temperature data should be a dict" + assert len(result_temp) == 0, "Temperature data should be empty when no predictions available" + + print(" ✓ Temperature data with no predictions handled correctly") + + # Run all sub-tests + print(" Running LoadMLComponent._fetch_load_data tests:") + run_async(test_basic_fetch()) + run_async(test_missing_sensor()) + run_async(test_car_charging_subtraction()) + run_async(test_load_power_fill()) + run_async(test_exception_handling()) + run_async(test_empty_load_data()) + run_async(test_temperature_data_fetch()) + run_async(test_temperature_no_data()) + print(" All _fetch_load_data tests passed!") + + +def _test_component_publish_entity(): + """Test LoadMLComponent._publish_entity method""" + from datetime import datetime, timezone, timedelta + from load_ml_component import LoadMLComponent + from unittest.mock import MagicMock + from const import TIME_FORMAT + + # Create mock base object + class MockBase: + def __init__(self): + self.prefix = "predbat" + self.config_root = None + self.now_utc = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + self.midnight_utc = datetime(2026, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + self.minutes_now = 720 # 12:00 = 720 minutes since midnight + self.local_tz = timezone.utc + self.args = {} + self.log_messages = [] + self.dashboard_calls = [] + + def log(self, msg): + self.log_messages.append(msg) + + def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None): + return { + "load_today": ["sensor.load_today"], + "load_power": None, + "car_charging_energy": None, + "load_scaling": 1.0, + "car_charging_energy_scale": 1.0, + }.get(key, default) + + # Test 1: Basic entity publishing with predictions + print(" Testing _publish_entity:") + mock_base = MockBase() + + component = LoadMLComponent(mock_base, load_ml_enable=True) + + # Mock dashboard_item to capture calls + def mock_dashboard_item(entity_id, state, attributes, app): + mock_base.dashboard_calls.append({"entity_id": entity_id, "state": state, "attributes": attributes, "app": app}) + + component.dashboard_item = mock_dashboard_item + + # Set up test data + component.load_minutes_now = 10.5 # Current load today + component.current_predictions = { + 0: 0.1, # Now (delta from "before predictions" to now = 0.1) + 5: 0.2, # 5 minutes from now + 60: 1.3, # 1 hour from now (load_today_h1) + 480: 9.7, # 8 hours from now (load_today_h8) + 1440: 28.9, # 24 hours from now + } + + # Set up predictor state + component.predictor.validation_mae = 0.5 + component.predictor.get_model_age_hours = MagicMock(return_value=2.0) # Mock model age calculation + component.last_train_time = datetime(2026, 1, 1, 10, 0, 0, tzinfo=timezone.utc) + component.load_data_age_days = 7.0 + component.model_status = "active" + component.predictor.epochs_trained = 50 + + # Call _publish_entity + component._publish_entity() + + # Verify dashboard_item was called (now twice - for main entity and accuracy entity) + assert len(mock_base.dashboard_calls) == 2, "dashboard_item should be called twice" + + call = mock_base.dashboard_calls[0] + call2 = mock_base.dashboard_calls[1] + + # Verify entity_id + assert call["entity_id"] == "sensor.predbat_load_ml_forecast", f"Expected sensor.predbat_load_ml_forecast, got {call['entity_id']}" + assert call2["entity_id"] == "sensor.predbat_load_ml_stats", f"Expected sensor.predbat_load_ml_stats, got {call2['entity_id']}" + # Verify state (max prediction value) + assert call2["state"] == 28.9, f"Expected state 28.9, got {call2['state']}" + + # Verify app + assert call2["app"] == "load_ml", f"Expected app 'load_ml', got {call2['app']}" + + # Verify attributes + attrs = call["attributes"] + attrs2 = call2["attributes"] + + # Check results format + assert "results" in attrs, "results should be in attributes" + results = attrs["results"] + assert isinstance(results, dict), "results should be a dict" + + # Verify results are timestamp-formatted and include load_minutes_now offset + # predictions are relative to now, so minute 60 = 1 hour from now = 13:00 + expected_timestamp_60 = (mock_base.midnight_utc + timedelta(minutes=60 + 720)).strftime(TIME_FORMAT) + assert expected_timestamp_60 in results, f"Expected timestamp {expected_timestamp_60} in results" + # Value should be prediction (1.3) + load_minutes_now (10.5) = 11.8 + assert abs(results[expected_timestamp_60] - 11.8) < 0.01, f"Expected value 11.8 at {expected_timestamp_60}, got {results[expected_timestamp_60]}" + + # Check load_today (current load) + assert "load_today" in attrs2, "load_today should be in attributes" + assert attrs2["load_today"] == 10.5, f"Expected load_today 10.5, got {attrs2['load_today']}" + + # Check load_today_h1 (1 hour ahead) + assert "load_today_h1" in attrs2, "load_today_h1 should be in attributes" + assert abs(attrs2["load_today_h1"] - 11.8) < 0.01, f"Expected load_today_h1 11.8, got {attrs2['load_today_h1']}" + + # Check load_today_h8 (8 hours ahead) + assert "load_today_h8" in attrs2, "load_today_h8 should be in attributes" + assert abs(attrs2["load_today_h8"] - 20.2) < 0.01, f"Expected load_today_h8 20.2 (9.7+10.5), got {attrs2['load_today_h8']}" + # Check MAE + assert "mae_kwh" in attrs2, "mae_kwh should be in attributes" + assert attrs2["mae_kwh"] == 0.5, f"Expected mae_kwh 0.5, got {attrs2['mae_kwh']}" + + # Check last_trained + assert "last_trained" in attrs2, "last_trained should be in attributes" + assert attrs2["last_trained"] == "2026-01-01T10:00:00+00:00", f"Expected last_trained 2026-01-01T10:00:00+00:00, got {attrs2['last_trained']}" + + # Check model_age_hours (12:00 - 10:00 = 2 hours) + assert "model_age_hours" in attrs2, "model_age_hours should be in attributes" + assert attrs2["model_age_hours"] == 2.0, f"Expected model_age_hours 2.0, got {attrs2['model_age_hours']}" + + # Check training_days + assert "training_days" in attrs2, "training_days should be in attributes" + assert attrs2["training_days"] == 7.0, f"Expected training_days 7.0, got {attrs2['training_days']}" + + # Check status + assert "status" in attrs2, "status should be in attributes" + assert attrs2["status"] == "active", f"Expected status 'active', got {attrs2['status']}" + + # Check model_version + assert "model_version" in attrs2, "model_version should be in attributes" + from load_predictor import MODEL_VERSION + + assert attrs2["model_version"] == MODEL_VERSION, f"Expected model_version {MODEL_VERSION}, got {attrs2['model_version']}" + + # Check epochs_trained + assert "epochs_trained" in attrs2, "epochs_trained should be in attributes" + assert attrs2["epochs_trained"] == 50, f"Expected epochs_trained 50, got {attrs2['epochs_trained']}" + + # Check power_today values (instantaneous power in kW) + assert "power_today_now" in attrs2, "power_today_now should be in attributes" + assert "power_today_h1" in attrs2, "power_today_h1 should be in attributes" + assert "power_today_h8" in attrs2, "power_today_h8 should be in attributes" + + # power_today_now: delta from start (prev_value=0) to minute 0 (0.1 kWh) / 5 min * 60 = 1.2 kW + expected_power_now = (0.1 - 0.0) / 5 * 60 + assert abs(attrs2["power_today_now"] - expected_power_now) < 0.01, f"Expected power_today_now {expected_power_now:.2f}, got {attrs2['power_today_now']}" + + # power_today_h1: delta from minute 55 to minute 60 + # We need to interpolate - predictions are sparse, so the actual delta will depend on what's in the dict + # For minute 60, prev_value in the loop would be the value at minute 55 (or closest) + # Since we don't have minute 55 in our test data, prev_value when reaching minute 60 will be from minute 5 + # So delta = (1.3 - 0.2) / 5 * 60 = 13.2 kW + expected_power_h1 = (1.3 - 0.2) / 5 * 60 + assert abs(attrs2["power_today_h1"] - expected_power_h1) < 0.01, f"Expected power_today_h1 {expected_power_h1:.2f}, got {attrs2['power_today_h1']}" + + # power_today_h8: delta from minute 475 to minute 480 + # prev_value would be from minute 60, so delta = (9.7 - 1.3) / 5 * 60 = 100.8 kW + expected_power_h8 = (9.7 - 1.3) / 5 * 60 + assert abs(attrs2["power_today_h8"] - expected_power_h8) < 0.01, f"Expected power_today_h8 {expected_power_h8:.2f}, got {attrs2['power_today_h8']}" + + # Check friendly_name + assert attrs["friendly_name"] == "ML Load Forecast", "friendly_name should be 'ML Load Forecast'" + assert attrs2["friendly_name"] == "ML Load Stats", "friendly_name should be 'ML Load Stats'" + # Check state_class + assert attrs2["state_class"] == "measurement", "state_class should be 'measurement'" + + # Check unit_of_measurement + assert attrs2["unit_of_measurement"] == "kWh", "unit_of_measurement should be 'kWh'" + + # Check icon + assert attrs["icon"] == "mdi:chart-line", "icon should be 'mdi:chart-line'" + assert attrs2["icon"] == "mdi:chart-line", "icon should be 'mdi:chart-line'" + + print(" ✓ Entity published with correct attributes") + + # Test 2: Empty predictions + mock_base.dashboard_calls = [] + component.current_predictions = {} + component._publish_entity() + + assert len(mock_base.dashboard_calls) == 2, "dashboard_item should be called even with empty predictions" + call = mock_base.dashboard_calls[0] + call2 = mock_base.dashboard_calls[1] + assert call2["state"] == 0, "State should be 0 with empty predictions" + assert call["attributes"]["results"] == {}, "results should be empty dict" + + print(" ✓ Empty predictions handled correctly") + + print(" All _publish_entity tests passed!") diff --git a/apps/predbat/tests/test_minute_data_import_export.py b/apps/predbat/tests/test_minute_data_import_export.py index 5850e9ff1..abe65cdfa 100644 --- a/apps/predbat/tests/test_minute_data_import_export.py +++ b/apps/predbat/tests/test_minute_data_import_export.py @@ -66,7 +66,7 @@ def mock_get_history_wrapper(entity_id, days): # Test with array containing real entities and '0' fixed value entity_ids = ["sensor.import_1", "0", "sensor.import_2"] - result = my_predbat.minute_data_import_export(now_utc=now, key=entity_ids[0], scale=1.0, required_unit="kWh") # Pass first entity directly + result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key=entity_ids[0], scale=1.0, required_unit="kWh") # Pass first entity directly # Verify we got data from entity1 if len(result) == 0: @@ -76,7 +76,7 @@ def mock_get_history_wrapper(entity_id, days): # Now test with the config approach using an array my_predbat.args["import_today_test"] = entity_ids - result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh") + result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh") # Verify data was accumulated from both real entities if len(result) == 0: @@ -99,7 +99,7 @@ def mock_get_history_wrapper(entity_id, days): my_predbat.args["import_today_test2"] = ["0", "1", "5"] - result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test2", scale=1.0, required_unit="kWh") + result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test2", scale=1.0, required_unit="kWh") if len(result) != 0: print("ERROR: Test 2 failed - should return empty dict for fixed values only, got {} entries".format(len(result))) @@ -110,7 +110,7 @@ def mock_get_history_wrapper(entity_id, days): my_predbat.args["import_today_test3"] = [None, "", "sensor.import_1"] - result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test3", scale=1.0, required_unit="kWh") + result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test3", scale=1.0, required_unit="kWh") # Should only get data from sensor.import_1 if len(result) == 0: @@ -120,9 +120,9 @@ def mock_get_history_wrapper(entity_id, days): # Test 4: Verify scaling works with accumulated data print("Test 4: Scaling with accumulated data") - result_scaled = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test", scale=2.0, required_unit="kWh") + result_scaled = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test", scale=2.0, required_unit="kWh") - result_unscaled = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh") + result_unscaled = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh") if 0 in result_scaled and 0 in result_unscaled: expected_scaled = result_unscaled[0] * 2.0 @@ -136,7 +136,7 @@ def mock_get_history_wrapper(entity_id, days): # Test 5: Single entity passed directly (not from config) print("Test 5: Single entity passed directly") - result = my_predbat.minute_data_import_export(now_utc=now, key="sensor.import_1", scale=1.0, required_unit="kWh") + result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="sensor.import_1", scale=1.0, required_unit="kWh") if len(result) == 0: print("ERROR: Test 5 failed - no data returned for direct entity") @@ -147,7 +147,7 @@ def mock_get_history_wrapper(entity_id, days): my_predbat.args["import_today_test6"] = ["sensor.nonexistent", "sensor.import_1"] - result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test6", scale=1.0, required_unit="kWh") + result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test6", scale=1.0, required_unit="kWh") # Should still get data from sensor.import_1 if len(result) == 0: @@ -159,7 +159,7 @@ def mock_get_history_wrapper(entity_id, days): my_predbat.args["import_today_test7"] = "sensor.import_1" - result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test7", scale=1.0, required_unit="kWh") + result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test7", scale=1.0, required_unit="kWh") if len(result) == 0: print("ERROR: Test 7 failed - no data returned for single string entity") diff --git a/apps/predbat/tests/test_temperature.py b/apps/predbat/tests/test_temperature.py new file mode 100644 index 000000000..aa26b3ab4 --- /dev/null +++ b/apps/predbat/tests/test_temperature.py @@ -0,0 +1,419 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2025 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt: off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +""" +Temperature API Component Tests + +Comprehensive test suite for the External Temperature API component. +Tests all major functionality including: +- Initialization and configuration with zone.home fallback +- URL placeholder replacement for latitude/longitude +- API data fetching with retry logic and error handling +- Timezone offset conversion (positive and negative) +- Sensor creation with current temperature and forecast data +- Cache persistence on API failures +- HA timestamp format conversion +""" + +from temperature import TemperatureAPI +from datetime import datetime, timezone + + +class MockTemperatureAPI(TemperatureAPI): + """Mock TemperatureAPI class for testing without ComponentBase dependencies""" + + def __init__(self, temperature_latitude, temperature_longitude, temperature_url): + # Don't call parent __init__ to avoid ComponentBase + self.last_updated_timestamp = None + self.failures_total = 0 + self.dashboard_items = {} + self.log_messages = [] + self.prefix = "predbat" + self._last_updated_time = None + self.state_storage = {} + self.initialize( + temperature_enable=True, + temperature_latitude=temperature_latitude, + temperature_longitude=temperature_longitude, + temperature_url=temperature_url + ) + + def log(self, message): + self.log_messages.append(message) + + def dashboard_item(self, entity_id, state, attributes, app=None): + self.dashboard_items[entity_id] = {"state": state, "attributes": attributes, "app": app} + + def update_success_timestamp(self): + self._last_updated_time = datetime.now(timezone.utc) + + def last_updated_time(self): + return self._last_updated_time + + def get_state_wrapper(self, entity_id, default=None, attribute=None): + """Mock get_state_wrapper""" + if entity_id in self.state_storage: + if attribute: + return self.state_storage[entity_id].get("attributes", {}).get(attribute, default) + return self.state_storage[entity_id].get("state", default) + return default + + def set_state(self, entity_id, state, attributes=None): + """Mock set_state""" + self.state_storage[entity_id] = {"state": state, "attributes": attributes or {}} + + +def _test_temperature_initialization(my_predbat): + """Test TemperatureAPI initialization with various configurations""" + print(" Testing TemperatureAPI initialization...") + + # Test with explicit coordinates + temp_component = MockTemperatureAPI( + temperature_latitude=51.5074, + temperature_longitude=-0.1278, + temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m" + ) + + if temp_component.temperature_latitude != 51.5074: + print(" ERROR: Incorrect latitude: {}".format(temp_component.temperature_latitude)) + return 1 + + if temp_component.temperature_longitude != -0.1278: + print(" ERROR: Incorrect longitude: {}".format(temp_component.temperature_longitude)) + return 1 + + print(" PASS: Initialization with explicit coordinates") + return 0 + + +def _test_temperature_zone_home_fallback(my_predbat): + """Test zone.home coordinate fallback""" + print(" Testing zone.home coordinate fallback...") + + # Initialize without explicit coordinates + temp_component = MockTemperatureAPI( + temperature_latitude=None, + temperature_longitude=None, + temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m" + ) + + # Set zone.home with coordinates + temp_component.set_state("zone.home", state="home", attributes={"latitude": 52.52, "longitude": 13.41}) + + # Test coordinate resolution + lat, lon = temp_component.get_coordinates() + + if lat != 52.52 or lon != 13.41: + print(" ERROR: Failed to fallback to zone.home coordinates: lat={}, lon={}".format(lat, lon)) + return 1 + + print(" PASS: zone.home fallback works correctly") + return 0 + + +def _test_temperature_url_placeholder_replacement(my_predbat): + """Test URL placeholder replacement with coordinates""" + print(" Testing URL placeholder replacement...") + + temp_component = MockTemperatureAPI( + temperature_latitude=51.5074, + temperature_longitude=-0.1278, + temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m" + ) + + url = temp_component.build_api_url(51.5074, -0.1278) + expected_url = "https://api.open-meteo.com/v1/forecast?latitude=51.5074&longitude=-0.1278&hourly=temperature_2m¤t=temperature_2m" + if url != expected_url: + print(" ERROR: URL placeholder replacement failed") + print(" Expected: {}".format(expected_url)) + print(" Got: {}".format(url)) + return 1 + + print(" PASS: URL placeholders replaced correctly") + return 0 + + +def _test_temperature_timezone_offset_conversion(my_predbat): + """Test timezone offset conversion from seconds to ±HH:MM format""" + print(" Testing timezone offset conversion...") + + my_predbat.args["temperature_latitude"] = 51.5074 + my_predbat.args["temperature_longitude"] = -0.1278 + + temp_component = MockTemperatureAPI( + temperature_latitude=51.5074, + temperature_longitude=-0.1278, + temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m" + ) + + # Test UTC (0 offset) + offset_str = temp_component.convert_timezone_offset(0) + if offset_str != "+00:00": + print(" ERROR: Failed to convert 0 seconds to +00:00, got: {}".format(offset_str)) + return 1 + + # Test positive offset (CET) + offset_str = temp_component.convert_timezone_offset(3600) + if offset_str != "+01:00": + print(" ERROR: Failed to convert 3600 seconds to +01:00, got: {}".format(offset_str)) + return 1 + + # Test negative offset (EST) + offset_str = temp_component.convert_timezone_offset(-18000) + if offset_str != "-05:00": + print(" ERROR: Failed to convert -18000 seconds to -05:00, got: {}".format(offset_str)) + return 1 + + # Test offset with minutes (IST) + offset_str = temp_component.convert_timezone_offset(19800) # +05:30 + if offset_str != "+05:30": + print(" ERROR: Failed to convert 19800 seconds to +05:30, got: {}".format(offset_str)) + return 1 + + print(" PASS: Timezone offset conversion works correctly") + return 0 + + +def _test_temperature_sensor_creation(my_predbat): + """Test sensor creation with current temperature and forecast""" + print(" Testing sensor creation with temperature data...") + + temp_component = MockTemperatureAPI( + temperature_latitude=51.5074, + temperature_longitude=-0.1278, + temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m" + ) + + # Mock API response data + mock_data = { + "latitude": 51.5, + "longitude": -0.12, + "utc_offset_seconds": 0, + "timezone": "GMT", + "current": { + "time": "2026-02-07T10:30", + "temperature_2m": 9.5 + }, + "hourly": { + "time": [ + "2026-02-07T00:00", + "2026-02-07T01:00", + "2026-02-07T02:00", + "2026-02-07T03:00" + ], + "temperature_2m": [8.2, 8.5, 8.8, 9.1] + } + } + + # Set the data and publish sensor + temp_component.temperature_data = mock_data + temp_component.last_updated_timestamp = datetime.now() + temp_component.publish_temperature_sensor() + + # Verify sensor was created + sensor_entity = "sensor.predbat_temperature" + if sensor_entity not in temp_component.dashboard_items: + print(" ERROR: Temperature sensor was not created") + return 1 + + sensor_state = temp_component.dashboard_items[sensor_entity]["state"] + if sensor_state != 9.5: + print(" ERROR: Incorrect sensor state: {} (expected 9.5)".format(sensor_state)) + return 1 + + # Verify attributes + sensor_attrs = temp_component.dashboard_items[sensor_entity]["attributes"] + results = sensor_attrs.get("results") + if results is None: + print(" ERROR: results attribute not set") + return 1 + + # Check forecast has correct HA timestamp format + expected_keys = [ + "2026-02-07T00:00:00+00:00", + "2026-02-07T01:00:00+00:00", + "2026-02-07T02:00:00+00:00", + "2026-02-07T03:00:00+00:00" + ] + + for key in expected_keys: + if key not in results: + print(" ERROR: Missing results key: {}".format(key)) + print(" Available keys: {}".format(list(results.keys()))) + return 1 + + # Verify temperature values + if results["2026-02-07T00:00:00+00:00"] != 8.2: + print(" ERROR: Incorrect results value for first hour") + return 1 + + print(" PASS: Sensor created with correct state and forecast") + return 0 + + +def _test_temperature_cache_persistence(my_predbat): + """Test that cached data persists on API failure""" + print(" Testing cache persistence on API failure...") + + my_predbat.args["temperature_latitude"] = 51.5074 + my_predbat.args["temperature_longitude"] = -0.1278 + temp_component = MockTemperatureAPI( + temperature_latitude=51.5074, + temperature_longitude=-0.1278, + temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m" + ) + + # Set initial cached data + initial_data = { + "utc_offset_seconds": 0, + "current": {"temperature_2m": 10.0}, + "hourly": {"time": ["2026-02-07T00:00"], "temperature_2m": [9.5]} + } + + temp_component.temperature_data = initial_data + temp_component.last_updated_timestamp = datetime(2026, 2, 7, 10, 0) + initial_time = temp_component.last_updated_timestamp + + # Publish sensor with initial data + temp_component.publish_temperature_sensor() + + # Verify initial sensor state + sensor_entity = "sensor.predbat_temperature" + if sensor_entity not in temp_component.dashboard_items: + print(" ERROR: Sensor not created") + return 1 + + sensor_state = temp_component.dashboard_items[sensor_entity]["state"] + if sensor_state != 10.0: + print(" ERROR: Initial sensor state incorrect: {}".format(sensor_state)) + return 1 + + # Simulate API failure by keeping old data + temp_component.temperature_data = initial_data # Keep old data + temp_component.publish_temperature_sensor() + + # Verify sensor still has old data (10.0) + sensor_state = temp_component.dashboard_items[sensor_entity]["state"] + if sensor_state != 10.0: + print(f" ERROR: Sensor state changed when it shouldn't - got {sensor_state}") + return 1 + + # Verify last_updated timestamp hasn't changed + if temp_component.last_updated_timestamp != initial_time: + print(" ERROR: last_updated timestamp changed when it shouldn't") + return 1 + + print(" PASS: Cached data persists on API failure") + return 0 + + +def _test_temperature_negative_timezone_offset(my_predbat): + """Test negative timezone offset handling (e.g., US timezones)""" + print(" Testing negative timezone offset handling...") + + my_predbat.args["temperature_latitude"] = 40.7128 + my_predbat.args["temperature_longitude"] = -74.0060 + temp_component = MockTemperatureAPI( + temperature_latitude=40.7128, + temperature_longitude=-74.0060, + temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m" + ) + + # Mock API response with negative timezone offset (EST) + mock_data = { + "utc_offset_seconds": -18000, # -05:00 + "current": {"temperature_2m": 5.5}, + "hourly": { + "time": ["2026-02-07T00:00"], + "temperature_2m": [4.8] + } + } + + temp_component.temperature_data = mock_data + temp_component.last_updated_timestamp = datetime.now() + temp_component.publish_temperature_sensor() + + # Verify sensor attributes have correct timezone + sensor_entity = "sensor.predbat_temperature" + if sensor_entity not in temp_component.dashboard_items: + print(" ERROR: Sensor not created") + return 1 + + forecast = temp_component.dashboard_items[sensor_entity]["attributes"].get("results", {}) + if not forecast: + print(" ERROR: Forecast not found in sensor attributes") + return 1 + + # Check for negative timezone offset in timestamp + expected_key = "2026-02-07T00:00:00-05:00" + if expected_key not in forecast: + print(" ERROR: Expected key {} not found in forecast".format(expected_key)) + print(" Available keys: {}".format(list(forecast.keys()))) + return 1 + + print(" PASS: Negative timezone offset handled correctly") + return 0 + + +def test_temperature(my_predbat=None): + """ + Comprehensive test suite for External Temperature API. + + Tests all major functionality including: + - Initialization and configuration + - zone.home coordinate fallback + - URL placeholder replacement + - Timezone offset conversion (positive and negative) + - Sensor creation with current temperature and forecast + - Cache persistence on API failures + - HA timestamp format conversion + """ + + # Registry of all sub-tests + sub_tests = [ + ("initialization", _test_temperature_initialization, "Temperature API initialization"), + ("zone_home_fallback", _test_temperature_zone_home_fallback, "zone.home coordinate fallback"), + ("url_placeholder", _test_temperature_url_placeholder_replacement, "URL placeholder replacement"), + ("timezone_offset", _test_temperature_timezone_offset_conversion, "Timezone offset conversion"), + ("sensor_creation", _test_temperature_sensor_creation, "Sensor creation with forecast data"), + ("cache_persistence", _test_temperature_cache_persistence, "Cache persistence on failure"), + ("negative_timezone", _test_temperature_negative_timezone_offset, "Negative timezone offset handling"), + ] + + print("\n" + "=" * 70) + print("EXTERNAL TEMPERATURE API TEST SUITE") + print("=" * 70) + + failed = 0 + passed = 0 + + for test_name, test_func, test_desc in sub_tests: + print("\n[{}] {}".format(test_name, test_desc)) + try: + test_result = test_func(my_predbat) + if test_result: + failed += 1 + print(" ❌ FAILED") + else: + passed += 1 + print(" ✅ PASSED") + except Exception as e: + print(" ❌ EXCEPTION: {}".format(e)) + import traceback + traceback.print_exc() + failed += 1 + + print("\n" + "=" * 70) + print("TEMPERATURE API TEST RESULTS") + print(" Passed: {}".format(passed)) + print(" Failed: {}".format(failed)) + print("=" * 70) + + return failed diff --git a/apps/predbat/unit_test.py b/apps/predbat/unit_test.py index a005b3157..a12f9649f 100644 --- a/apps/predbat/unit_test.py +++ b/apps/predbat/unit_test.py @@ -96,6 +96,8 @@ from tests.test_ohme import test_ohme from tests.test_component_base import test_component_base_all from tests.test_solis import run_solis_tests +from tests.test_load_ml import test_load_ml +from tests.test_temperature import test_temperature # Mock the components and plugin system @@ -244,6 +246,10 @@ def main(): ("component_base", test_component_base_all, "ComponentBase tests (all)", False), # Solis Cloud API unit tests ("solis", run_solis_tests, "Solis Cloud API tests (V1/V2 time window writes, change detection)", False), + # ML Load Forecaster tests + ("load_ml", test_load_ml, "ML Load Forecaster tests (MLP, training, persistence, validation)", False), + # External Temperature API tests + ("temperature", test_temperature, "External Temperature API tests (initialization, zone.home fallback, timezone conversion, caching)", False), ("optimise_levels", run_optimise_levels_tests, "Optimise levels tests", False), ("optimise_windows", run_optimise_all_windows_tests, "Optimise all windows tests", True), ("debug_cases", run_debug_cases, "Debug case file tests", True), diff --git a/apps/predbat/utils.py b/apps/predbat/utils.py index 9125c2704..f2c8b718a 100644 --- a/apps/predbat/utils.py +++ b/apps/predbat/utils.py @@ -41,7 +41,7 @@ def get_now_from_cumulative(data, minutes_now, backwards): return max(value, 0) -def prune_today(data, now_utc, midnight_utc, prune=True, group=15, prune_future=False, intermediate=False): +def prune_today(data, now_utc, midnight_utc, prune=True, group=15, prune_future=False, prune_future_days=0, prune_past_days=0, intermediate=False, offset_minutes=0): """ Remove data from before today """ @@ -54,18 +54,19 @@ def prune_today(data, now_utc, midnight_utc, prune=True, group=15, prune_future= timekey = datetime.strptime(key, TIME_FORMAT_SECONDS) else: timekey = datetime.strptime(key, TIME_FORMAT) - if last_time and (timekey - last_time).seconds < group * 60: + if last_time and (timekey - last_time).total_seconds() < group * 60: continue - if intermediate and last_time and ((timekey - last_time).seconds > group * 60): + if intermediate and last_time and ((timekey - last_time).total_seconds() > group * 60): # Large gap, introduce intermediate data point seconds_gap = int((timekey - last_time).total_seconds()) for i in range(1, seconds_gap // int(group * 60)): - new_time = last_time + timedelta(seconds=i * group * 60) - results[new_time.strftime(TIME_FORMAT)] = prev_value - if not prune or (timekey > midnight_utc): - if prune_future and (timekey > now_utc): + new_time = last_time + timedelta(seconds=i * group * 60) + timedelta(minutes=offset_minutes) + results[new_time.isoformat()] = prev_value + if not prune or (timekey > (midnight_utc - timedelta(days=prune_past_days))): + if prune_future and (timekey > (now_utc + timedelta(days=prune_future_days))): continue - results[key] = data[key] + new_time = timekey + timedelta(minutes=offset_minutes) + results[new_time.isoformat()] = data[key] last_time = timekey prev_value = data[key] return results diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 2a9efd2ee..2ab56472e 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -57,7 +57,7 @@ get_dashboard_collapsible_js, ) -from utils import calc_percent_limit, str2time, dp0, dp2, format_time_ago, get_override_time_from_string, history_attribute, prune_today +from utils import calc_percent_limit, str2time, dp0, dp2, dp4, format_time_ago, get_override_time_from_string, history_attribute, prune_today from const import TIME_FORMAT, TIME_FORMAT_DAILY, TIME_FORMAT_HA from predbat import THIS_VERSION from component_base import ComponentBase @@ -142,6 +142,7 @@ async def start(self): app.router.add_get("/internals", self.html_internals) app.router.add_get("/api/internals", self.html_api_internals) app.router.add_get("/api/internals/download", self.html_api_internals_download) + app.router.add_get("/api/status", self.html_api_get_status) # Notify plugin system that web interface is ready if hasattr(self.base, "plugin_system") and self.base.plugin_system: @@ -1794,6 +1795,24 @@ async def html_api_post_state(self, request): else: return web.Response(content_type="application/json", text='{"result": "error"}') + async def html_api_get_status(self, request): + """ + Get current Predbat status (calculating state and battery info) + """ + try: + calculating = self.get_arg("active", False) + if self.base.update_pending: + calculating = True + + battery_icon = self.get_battery_status_icon() + + status_data = {"calculating": calculating, "battery_html": battery_icon} + + return web.Response(content_type="application/json", text=json.dumps(status_data)) + except Exception as e: + self.log("Error getting status: {}".format(e)) + return web.Response(status=500, content_type="application/json", text=json.dumps({"error": str(e)})) + async def html_api_ping(self, request): """ Check if Predbat is running @@ -2570,6 +2589,96 @@ def get_chart(self, chart): {"name": "Forecast CL", "data": pv_today_forecastCL, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#e90a0a"}, ] text += self.render_chart(series_data, "kW", "Solar Forecast", now_str) + elif chart == "LoadML": + # Get historical load data for last 24 hours + load_today = prune_today(history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 1, required=False), attributes=True, state_key="load_today"), self.now_utc, self.midnight_utc, prune=False) + load_today_h1 = prune_today( + history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 1, required=False), attributes=True, state_key="load_today_h1"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 1 + ) + load_today_h8 = prune_today( + history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 1, required=False), attributes=True, state_key="load_today_h8"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 8 + ) + + # Get ML forecast from load_forecast_ml entity results + load_ml_forecast = self.get_entity_results("sensor." + self.prefix + "_load_ml_forecast") + + series_data = [ + {"name": "Load (Actual)", "data": load_today, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#3291a8"}, + {"name": "Forecast (+1h)", "data": load_today_h1, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "color": "#f5a442"}, + {"name": "Forecast (+8h)", "data": load_today_h8, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "color": "#9b59b6"}, + {"name": "Load (ML Forecast)", "data": load_ml_forecast, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#eb2323"}, + ] + text += self.render_chart(series_data, "kWh", "ML Load Forecast", now_str) + elif chart == "LoadMLPower": + # Get historical load power + load_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".load_power", 7, required=False)) + load_power = prune_today(load_power_hist, self.now_utc, self.midnight_utc, prune=False) + + # Get ML predicted load energy (cumulative) and convert to power (kW) + load_ml_forecast_energy = self.get_entity_results("sensor." + self.prefix + "_load_ml_forecast") + load_ml_forecast_power = {} + + power_today = prune_today(history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 7, required=False), attributes=True, state_key="power_today"), self.now_utc, self.midnight_utc, prune=False) + power_today_h1 = prune_today( + history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 7, required=False), attributes=True, state_key="power_today_h1"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 1 + ) + power_today_h8 = prune_today( + history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 7, required=False), attributes=True, state_key="power_today_h8"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 8 + ) + + # Sort timestamps and calculate deltas to get energy per interval + if load_ml_forecast_energy: + from datetime import datetime + + sorted_timestamps = sorted(load_ml_forecast_energy.keys()) + prev_energy = 0 + prev_timestamp = None + for timestamp in sorted_timestamps: + energy = load_ml_forecast_energy[timestamp] + energy_delta = max(energy - prev_energy, 0) + + # Calculate actual interval in hours between this and previous timestamp + if prev_timestamp: + # Parse timestamps and calculate difference in hours + curr_dt = datetime.strptime(timestamp, TIME_FORMAT) + prev_dt = datetime.strptime(prev_timestamp, TIME_FORMAT) + interval_hours = (curr_dt - prev_dt).total_seconds() / 3600.0 + load_ml_forecast_power[timestamp] = dp4(energy_delta / interval_hours) + + prev_energy = energy + prev_timestamp = timestamp + + # Get historical PV power + pv_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".pv_power", 1, required=False)) + pv_power = prune_today(pv_power_hist, self.now_utc, self.midnight_utc, prune=False) + + # Get temperature prediction data and limit to 48 hours forward + temperature_forecast = prune_today(self.get_entity_results("sensor." + self.prefix + "_temperature"), self.now_utc, self.midnight_utc, prune_future=True, prune_future_days=2, prune=True, prune_past_days=7) + + series_data = [ + {"name": "Load Power (Actual)", "data": load_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#3291a8", "unit": "kW"}, + {"name": "Load Power (ML Predicted Future)", "data": load_ml_forecast_power, "opacity": "0.5", "stroke_width": "3", "chart_type": "area", "stroke_curve": "smooth", "color": "#eb2323", "unit": "kW"}, + {"name": "Load Power (Used)", "data": load_power_best, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"}, + {"name": "Load Power ML History", "data": power_today, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"}, + {"name": "Load Power ML History +1h", "data": power_today_h1, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"}, + {"name": "Load Power ML History +8h", "data": power_today_h8, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"}, + {"name": "PV Power (Actual)", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d", "unit": "kW"}, + {"name": "PV Power (Predicted)", "data": pv_power_best, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#ffa500", "unit": "kW"}, + {"name": "Temperature", "data": temperature_forecast, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "color": "#ff6b6b", "unit": "°C"}, + ] + + # Configure secondary axis for temperature + secondary_axis = [ + { + "title": "°C", + "series_name": "Temperature", + "decimals": 1, + "opposite": True, + "labels_formatter": "return val.toFixed(1) + '°C';", + } + ] + + text += self.render_chart(series_data, "kW", "ML Load & PV Power with Temperature", now_str, extra_yaxis=secondary_axis) else: text += "

Unknown chart type

" @@ -2594,6 +2703,8 @@ async def html_charts(self, request): active_inday = "" active_pv = "" active_pv7 = "" + active_loadml = "" + active_loadmlpower = "" if chart == "Battery": active_battery = "active" @@ -2609,6 +2720,10 @@ async def html_charts(self, request): active_pv = "active" elif chart == "PV7": active_pv7 = "active" + elif chart == "LoadML": + active_loadml = "active" + elif chart == "LoadMLPower": + active_loadmlpower = "active" text += '
' text += "

Charts

" @@ -2619,6 +2734,10 @@ async def html_charts(self, request): text += f'InDay' text += f'PV' text += f'PV7' + # Only show LoadML chart if ML is enabled + if self.base.get_arg("load_ml_enable", False): + text += f'LoadML' + text += f'LoadMLPower' text += "
" text += '
' diff --git a/apps/predbat/web_helper.py b/apps/predbat/web_helper.py index c25f0d6d1..b01052f6d 100644 --- a/apps/predbat/web_helper.py +++ b/apps/predbat/web_helper.py @@ -6648,9 +6648,52 @@ def get_menu_html(calculating, default_page, arg_errors, THIS_VERSION, battery_s } } +// Live status update functionality +let statusUpdateInterval = null; + +function updateLiveStatus() { + fetch('./api/status') + .then(response => response.json()) + .then(data => { + // Update calculating/idle icon + const statusIcon = document.getElementById('status-icon'); + if (statusIcon) { + if (data.calculating) { + statusIcon.innerHTML = ''; + } else { + statusIcon.innerHTML = ''; + } + } + + // Update battery status + const batteryStatus = document.getElementById('battery-status'); + if (batteryStatus && data.battery_html) { + batteryStatus.innerHTML = data.battery_html; + } + }) + .catch(error => { + console.error('Error updating status:', error); + }); +} + +function startStatusUpdates() { + // Initial update + updateLiveStatus(); + // Update every 5 seconds + statusUpdateInterval = setInterval(updateLiveStatus, 5000); +} + +function stopStatusUpdates() { + if (statusUpdateInterval) { + clearInterval(statusUpdateInterval); + statusUpdateInterval = null; + } +} + // Initialize menu on page load document.addEventListener("DOMContentLoaded", function() { setActiveMenuItem(); +startStatusUpdates(); // For each menu item, add click handler to set it as active const menuLinks = document.querySelectorAll('.menu-bar a'); @@ -6708,10 +6751,10 @@ def get_menu_html(calculating, default_page, arg_errors, THIS_VERSION, battery_s onclick="flyBat()" style="cursor: pointer;" > - """ + """ + status_icon - + """ -
+ + """ +
""" + battery_status_icon + """ diff --git a/coverage/analyze_data.py b/coverage/analyze_data.py new file mode 100644 index 000000000..c19237caf --- /dev/null +++ b/coverage/analyze_data.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import json +import statistics + +# Load the data +with open("load_minutes_debug.json", "r") as f: + load_data = {int(k): float(v) for k, v in json.load(f).items()} + +# Convert to energy per step (like predictor does) +STEP_MINUTES = 5 +energy_per_step = {} +sorted_minutes = sorted(load_data.keys()) + +for minute in sorted_minutes: + if minute + STEP_MINUTES in load_data: + energy = max(0, load_data[minute] - load_data[minute + STEP_MINUTES]) + energy_per_step[minute] = energy + +# Get statistics +energies = list(energy_per_step.values()) +print(f"Energy per step statistics:") +print(f" Count: {len(energies)}") +print(f" Min: {min(energies):.4f} kWh") +print(f" Max: {max(energies):.4f} kWh") +print(f" Mean: {statistics.mean(energies):.4f} kWh") +print(f" Median: {statistics.median(energies):.4f} kWh") +print(f" Std: {statistics.stdev(energies):.4f} kWh") +energies_sorted = sorted(energies) +print(f" 25th percentile: {energies_sorted[len(energies)//4]:.4f} kWh") +print(f" 75th percentile: {energies_sorted[3*len(energies)//4]:.4f} kWh") +print(f" 95th percentile: {energies_sorted[95*len(energies)//100]:.4f} kWh") + +# Show first 24 hours of data +print(f"\nFirst 24 hours of data (minute 0-1440):") +for minute in range(0, min(1440, max(energy_per_step.keys())), 60): + if minute in energy_per_step: + print(f" Minute {minute}: {energy_per_step[minute]:.4f} kWh") + +# Check what the training data looks like +print(f"\nTraining window analysis (for predicting minute 0-2880):") +print(f"Looking at samples from minute 2880 onwards...") +for sample_minute in range(2880, min(2880 + 1440, max(energy_per_step.keys())), 60): + if sample_minute in energy_per_step: + print(f" Sample at minute {sample_minute} (lookback from here): {energy_per_step[sample_minute]:.4f} kWh") diff --git a/coverage/analyze_periods.py b/coverage/analyze_periods.py new file mode 100644 index 000000000..5b1ad4aef --- /dev/null +++ b/coverage/analyze_periods.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +import json + +# Load the data +with open("load_minutes_debug.json", "r") as f: + load_data = {int(k): float(v) for k, v in json.load(f).items()} + +# Convert to energy per step +STEP_MINUTES = 5 +energy_per_step = {} +sorted_minutes = sorted(load_data.keys()) + +for minute in sorted_minutes: + if minute + STEP_MINUTES in load_data: + energy = max(0, load_data[minute] - load_data[minute + STEP_MINUTES]) + energy_per_step[minute] = energy + +# Analyze different time periods +periods = [("Recent (0-1440min, 0-24h)", 0, 1440), ("Recent (0-2880min, 0-48h)", 0, 2880), ("Training window (2880-10080min, 2-7 days ago)", 2880, 10080), ("Full dataset", 0, max(energy_per_step.keys()))] + +for name, start, end in periods: + values = [energy_per_step[m] for m in energy_per_step.keys() if start <= m < end] + if values: + mean_val = sum(values) / len(values) + max_val = max(values) + median_val = sorted(values)[len(values) // 2] + print(f"{name}:") + print(f" Count: {len(values)}, Mean: {mean_val:.4f} kWh, Median: {median_val:.4f} kWh, Max: {max_val:.4f} kWh") + else: + print(f"{name}: No data") diff --git a/coverage/debug_model.py b/coverage/debug_model.py new file mode 100644 index 000000000..eb2023d87 --- /dev/null +++ b/coverage/debug_model.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Debug script to analyze what the model is learning""" +import json +import sys + +sys.path.insert(0, "../apps/predbat") +from load_predictor import LoadPredictor +from datetime import datetime, timezone + +# Load data +with open("load_minutes_debug.json", "r") as f: + load_data = {int(k): float(v) for k, v in json.load(f).items()} + +# Train model +predictor = LoadPredictor(learning_rate=0.001, max_load_kw=20.0) +now_utc = datetime.now(timezone.utc) + +print("Training model...") +predictor.train(load_data, now_utc, is_initial=True, epochs=30, time_decay_days=7) + +# Check normalization parameters +print(f"\nNormalization parameters:") +print(f" Feature mean (first 12): {predictor.feature_mean[:12]}") # Lookback values +print(f" Feature mean (last 4): {predictor.feature_mean[12:]}") # Time features +print(f" Feature std (first 12): {predictor.feature_std[:12]}") +print(f" Feature std (last 4): {predictor.feature_std[12:]}") +print(f" Target mean: {predictor.target_mean:.4f} kWh") +print(f" Target std: {predictor.target_std:.4f} kWh") + +# Check first layer weights to see feature importance +print(f"\nFirst layer weight magnitudes (input importance):") +w1 = predictor.weights[0] # Shape: (16, 32) +for i in range(16): + mag = float((w1[i, :] ** 2).sum() ** 0.5) + feat_name = f"lookback_{i}" if i < 12 else ["sin_minute", "cos_minute", "sin_day", "cos_day"][i - 12] + print(f" {feat_name:15s}: {mag:.4f}") diff --git a/coverage/debug_predict.py b/coverage/debug_predict.py new file mode 100644 index 000000000..9e8931d81 --- /dev/null +++ b/coverage/debug_predict.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Debug the prediction issue""" +import sys + +sys.path.insert(0, "../apps/predbat") + +import json +import numpy as np +from datetime import datetime, timezone, timedelta +from load_predictor import LoadPredictor, LOOKBACK_STEPS, STEP_MINUTES + +# Load data +with open("load_minutes_debug.json", "r") as f: + load_data = {int(k): float(v) for k, v in json.load(f).items()} + +# Quick mode - just check final energies +if len(sys.argv) > 1 and sys.argv[1] == "--quick": + predictor = LoadPredictor(learning_rate=0.001, max_load_kw=20.0) + now_utc = datetime.now(timezone.utc) + midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + + predictor.train(load_data, now_utc, is_initial=True, epochs=30, time_decay_days=7) + predictions = predictor.predict(load_data, now_utc, midnight_utc) + + pred_keys = sorted(predictions.keys()) + energies = [] + for i, minute in enumerate(pred_keys): + if i == 0: + energies.append(predictions[minute]) + else: + energies.append(predictions[minute] - predictions[pred_keys[i - 1]]) + + print("Energy stats:") + print(f" Min: {min(energies):.4f}, Max: {max(energies):.4f}, Mean: {np.mean(energies):.4f}") + print(f" Steps 0-20: {[round(e, 4) for e in energies[0:20]]}") + print(f" Steps 200-220: {[round(e, 4) for e in energies[200:220]]}") + print(f" Steps 400-420: {[round(e, 4) for e in energies[400:420]]}") + print(f" Steps 550-576: {[round(e, 4) for e in energies[550:576]]}") + sys.exit(0) + +# Train model +predictor = LoadPredictor(learning_rate=0.001, max_load_kw=20.0) +now_utc = datetime.now(timezone.utc) +midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0) + +print("Training model...") +predictor.train(load_data, now_utc, is_initial=True, epochs=30, time_decay_days=7) + +# Check normalization parameters +print(f"\n=== Normalization Parameters ===") +print(f"Feature mean (first 10 lookback): {predictor.feature_mean[:10]}") +print(f"Feature std (first 10 lookback): {predictor.feature_std[:10]}") +print(f"Target mean: {predictor.target_mean:.6f}") +print(f"Target std: {predictor.target_std:.6f}") + +# Get the energy per step for historical data +energy_per_step = predictor._load_to_energy_per_step(load_data) + +# Look at the initial lookback buffer +print(f"\n=== Initial Lookback Buffer ===") +lookback_buffer = [] +for lb_offset in range(LOOKBACK_STEPS): + lb_minute = lb_offset * STEP_MINUTES + if lb_minute in energy_per_step: + lookback_buffer.append(energy_per_step[lb_minute]) + else: + lookback_buffer.append(0) + +print(f"First 10 values: {lookback_buffer[:10]}") +print(f"Mean: {np.mean(lookback_buffer):.6f}, Std: {np.std(lookback_buffer):.6f}") +print(f"Min: {np.min(lookback_buffer):.6f}, Max: {np.max(lookback_buffer):.6f}") + +# Now trace through a few prediction steps +print(f"\n=== Prediction Step-by-Step ===") +predictions_energy = [] + +for step_idx in range(200): # First 200 steps (16+ hours) + target_time = now_utc + timedelta(minutes=(step_idx + 1) * STEP_MINUTES) + minute_of_day = target_time.hour * 60 + target_time.minute + day_of_week = target_time.weekday() + time_features = predictor._create_time_features(minute_of_day, day_of_week) + + # Combine features + features = np.concatenate([np.array(lookback_buffer, dtype=np.float32), time_features]) + + # Normalize + features_norm = predictor._normalize_features(features.reshape(1, -1), fit=False) + + # Forward pass + pred_norm, _, _ = predictor._forward(features_norm) + + # Denormalize + pred_energy = predictor._denormalize_predictions(pred_norm[0]) + + # Clip + pred_clipped = predictor._clip_predictions(pred_energy) + energy_value = float(pred_clipped[0]) + + print(f"Step {step_idx}: lb_mean={np.mean(lookback_buffer):.4f}, " f"pred_norm={pred_norm[0][0]:.4f}, pred_denorm={pred_energy[0]:.4f}, " f"pred_clipped={energy_value:.4f}") + + predictions_energy.append(energy_value) + + # Update lookback buffer + lookback_buffer.insert(0, energy_value) + lookback_buffer.pop() + +# Check for the issue - when does it first go to zero? +print(f"\n=== Full Prediction Analysis ===") +full_predictions = predictor.predict(load_data, now_utc, midnight_utc) + +# Show cumulative values +pred_keys = sorted(full_predictions.keys()) +print("\nFirst 20 cumulative values:") +for i in range(20): + print(f" minute {pred_keys[i]}: {full_predictions[pred_keys[i]]:.4f}") + +print("\nAround step 120-140:") +for i in range(120, 140): + print(f" minute {pred_keys[i]}: {full_predictions[pred_keys[i]]:.4f}") + +# Convert to energy +pred_energy_list = [] +sorted_minutes = sorted(full_predictions.keys()) +prev_cum = 0 +for minute in sorted_minutes: + cum = full_predictions[minute] + energy = cum - prev_cum + pred_energy_list.append(energy) + prev_cum = cum + +print(f"\nPrediction minutes: {sorted_minutes[:10]}...{sorted_minutes[-3:]}") +print(f"First 20 energies: {[f'{e:.4f}' for e in pred_energy_list[:20]]}") +print(f"Middle energies (140-160): {[f'{e:.4f}' for e in pred_energy_list[140:160]]}") +print(f"Late energies (200-220): {[f'{e:.4f}' for e in pred_energy_list[200:220]]}") + +# Check for zeros or near-zeros +zeros = [(i, e) for i, e in enumerate(pred_energy_list) if e < 0.01] +print(f"\nSteps with energy < 0.01: {len(zeros)}") +if zeros: + print(f"First 10: {zeros[:10]}") + +# Stats +print(f"\nOverall stats:") +print(f" Min: {min(pred_energy_list):.4f}") +print(f" Max: {max(pred_energy_list):.4f}") +print(f" Mean: {np.mean(pred_energy_list):.4f}") +print(f" Std: {np.std(pred_energy_list):.4f}") diff --git a/docs/components.md b/docs/components.md index 4917b352c..05288b5e9 100644 --- a/docs/components.md +++ b/docs/components.md @@ -21,6 +21,8 @@ This document provides a comprehensive overview of all Predbat components, their - [Solis Cloud API (Solis)](#solis-cloud-api-solis) - [Alert Feed (alert_feed)](#alert-feed-alert_feed) - [Carbon Intensity API (carbon)](#carbon-intensity-api-carbon) + - [Temperature API (temperature)](#temperature-api-temperature) + - [ML Load Prediction (load_ml)](#ml-load-prediction-load_ml) - [Managing Components](#managing-components) - [Checking Component Status](#checking-component-status) - [Restarting Components](#restarting-components) @@ -617,6 +619,152 @@ Note: To use the carbon data in Predbat you also have to turn on **switch.predba --- +### Temperature API (temperature) + +**Can be restarted:** Yes + +#### What it does (temperature) + +Fetches temperature forecasts from the Open-Meteo API to provide accurate temperature predictions for the next 48+ hours. +This temperature data is used by the ML Load Prediction component to improve load forecasting accuracy, especially for homes with electric heating or air conditioning systems. + +#### When to enable (temperature) + +- You are using ML Load Prediction and want improved accuracy +- Your energy consumption is significantly affected by temperature (heating/cooling) +- You want temperature forecasts available for other automations + +#### How it works (temperature) + +- Fetches temperature data from Open-Meteo API every hour +- Uses your location coordinates (from `temperature_latitude`/`temperature_longitude` or defaults to `zone.home`) +- Provides current temperature and hourly forecasts +- Publishes data to `sensor.predbat_temperature` with forecasts in the `results` attribute +- Automatically retries on API failures with exponential backoff + +**Important**: This component is **recommended** when using ML Load Prediction, as temperature data cam improve prediction accuracy for households with electric/heat-pump heating. + +#### Configuration Options (temperature) + +| Option | Type | Required | Default | Config Key | Description | +| ------ | ---- | -------- | ------- | ---------- | ----------- | +| `temperature_enable` | Boolean | Yes | False | `temperature_enable` | Set to `true` to enable temperature forecasts | +| `temperature_url` | String | No | Open-Meteo API URL | `temperature_url` | API URL with LATITUDE/LONGITUDE placeholders | +| `temperature_latitude` | Float | No | Uses zone.home | `temperature_latitude` | Latitude for temperature forecast location | +| `temperature_longitude` | Float | No | Uses zone.home | `temperature_longitude` | Longitude for temperature forecast location | + +#### Configuration example (temperature) + +```yaml +predbat: + # Enable temperature forecasts (recommended for ML load prediction) + temperature_enable: true + + # Optional: specify location (defaults to zone.home) + # temperature_latitude: 51.5074 + # temperature_longitude: -0.1278 +``` + +#### Accessing temperature data (temperature) + +Temperature data is published to: + +- `sensor.predbat_temperature` - Current temperature with forecast in `results` attribute + +The `results` attribute contains a dictionary of timestamp strings (ISO format with timezone) to temperature values in °C. + +--- + +### ML Load Prediction (load_ml) + +**Can be restarted:** Yes + +#### What it does (load_ml) + +Uses a neural network to predict your household energy consumption for the next 48 hours based on historical patterns, time-of-day, day-of-week, and optionally temperature and PV generation data. +This provides more accurate load predictions than simple averaging, especially for households with variable usage patterns. + +#### When to enable (load_ml) + +- You want more accurate load predictions than historical averages +- Your energy consumption has regular daily/weekly patterns +- You have at least 1 day of historical load data (7+ days recommended) +- You want Predbat to automatically adapt to changing consumption patterns + +#### How it works (load_ml) + +- Fetches historical load data from your configured `load_today` sensor +- Optionally incorporates PV generation and temperature forecast data +- Trains a multi-layer neural network on your historical patterns +- Makes autoregressive predictions for 48 hours ahead in 5-minute intervals +- Fine-tunes periodically (every 2 hours) to adapt to changing patterns +- Validates predictions and falls back gracefully if accuracy is poor +- Publishes predictions to `sensor.predbat_load_ml_forecast` + +**Important**: For best results, enable the Temperature component (`temperature_enable: true`) as temperature data significantly improves prediction accuracy. + +For a detailed explanation of how the neural network works and comprehensive configuration guidance, see the [ML Load Prediction documentation](load-ml.md). + +#### Configuration Options (load_ml) + +| Option | Type | Required | Default | Config Key | Description | +| ------ | ---- | -------- | ------- | ---------- | ----------- | +| `load_ml_enable` | Boolean | Yes | False | `load_ml_enable` | Set to `True` to enable ML load prediction | +| `load_ml_source` | Boolean | Yes | False | `load_ml_source` | Set to `True` to use the ML load prediction in Predbat | + +Note: load_today, pv_today and car_charging_energy apps.yaml configuration items are also used, but these should already be set in Predbat. + +#### Configuration example (load_ml) + +```yaml +predbat: + # Enable ML load prediction + load_ml_enable: True + # Use the data in Predbat, can be false while exploring the predictions but not using them + load_ml_source: True + + # Optional but recommended: enable temperature forecasts + temperature_enable: true +``` + +#### Understanding model status (load_ml) + +The ML component tracks several status indicators: + +- **not_initialized**: Model has not been created yet +- **training**: Model is currently training on historical data +- **active**: Model is trained and making predictions +- **validation_failed**: Predictions are disabled due to high validation error +- **stale**: Model hasn't been trained in 48+ hours and needs retraining + +Check Predbat logs for training progress and validation metrics: + +```text +ML Component: Starting initial training +ML Predictor: Training complete, final val_mae=0.3245 kWh +ML Component: Model status: active +``` + +#### Accessing predictions (load_ml) + +ML load predictions are published to: + +- `sensor.predbat_load_ml_forecast` - Contains 48-hour prediction in `results` attribute + +Predbat automatically uses these predictions when making battery charge/discharge decisions. + +#### For more information (load_ml) + +See the comprehensive [ML Load Prediction documentation](load-ml.md) for: + +- Detailed explanation of neural network architecture +- Training process and parameters +- Expected accuracy metrics +- Troubleshooting guide +- Advanced configuration options + +--- + ## Managing Components ### Checking Component Status diff --git a/docs/load-ml.md b/docs/load-ml.md new file mode 100644 index 000000000..cebc9ad38 --- /dev/null +++ b/docs/load-ml.md @@ -0,0 +1,303 @@ +# ML Load Prediction + +Predbat includes a neural network-based machine learning component that can predict your household energy consumption for the next 48 hours. +This prediction is based on historical load patterns, time-of-day patterns, day-of-week patterns, and optionally PV generation history and temperature forecasts. + +## Table of Contents + +- [Overview](#overview) +- [How the Neural Network Works](#how-the-neural-network-works) +- [Configuration](#configuration) +- [Setup Instructions](#setup-instructions) +- [Understanding the Model](#understanding-the-model) +- [Monitoring and Troubleshooting](#monitoring-and-troubleshooting) +- [Model Persistence](#model-persistence) + +## Overview + +The ML Load Prediction component uses a lightweight multi-layer perceptron (MLP) neural network implemented in pure NumPy. It learns from your historical energy consumption patterns and makes predictions about future load. + +**Key Features:** + +- Predicts 48 hours of load data in 5-minute intervals +- Learns daily and weekly patterns automatically +- Supports historical PV generation data as an input feature +- Supports temperature forecast data for improved accuracy +- Automatically trains on historical data (requires at least 1 day, recommended 7+ days) +- Fine-tunes periodically to adapt to changing patterns +- Model persists across restarts +- Falls back gracefully if predictions are unreliable + +## How the Neural Network Works + +### Architecture + +The ML Load Predictor uses a deep neural network with an input layer, some hidden layers and an output layer. + +### Input Features + +The neural network uses several types of input features to make predictions: + +1. **Historical Load Lookback** + - Past 24 hours of energy consumption at 5-minute intervals + - Helps the network understand recent usage patterns + +2. **Historical PV Generation** + - Past 24 hours of solar PV generation at 5-minute intervals + - Helps correlate solar production with consumption patterns + - Requires `pv_today` sensor to be configured + +3. **Historical Temperature** + - Past 7 days and future 2 days of temperature data at 5-minute intervals + - Helps correlate temperature with energy usage (heating/cooling) + - **Requires the Temperature component to be enabled** + +4. **Cyclical Time Features** (4 features) + - Sin/Cos encoding of hour-of-day (captures daily patterns) + - Sin/Cos encoding of day-of-week (captures weekly patterns) + - These features help the network understand that 23:55 is close to 00:05 + +### Prediction Process + +The model uses an autoregressive approach: + +1. Takes the last 24 hours of historical data +2. Predicts the next 5-minute step +3. Adds that prediction to the history window +4. Shifts the window forward and repeats +5. Continues for 576 steps to cover 48 hours + +To prevent drift in long-range predictions, the model blends autoregressive predictions with historical daily patterns. + +### Training Process + +**Initial Training:** + +- Requires at least 1 day of historical data (7+ days recommended) +- Uses 50 epochs with early stopping +- Validates on the last 24 hours of data +- Saves model to disk: `predbat_ml_model.npz` + +**Fine-tuning:** + +- Runs every 2 hours if enabled +- Uses last 24 hours of data +- Uses 2 epochs to quickly adapt to recent changes +- Preserves learned patterns while adapting to new ones + +**Model Validation:** + +- Model is validated after each training session +- If validation error exceeds threshold (default 2.0 kWh MAE), predictions are disabled +- Model is considered stale after 48 hours and requires retraining + +## Configuration + +### Basic Setup + +To enable ML load prediction, add to your `apps.yaml`: + +```yaml +predbat: + module: predbat + class: PredBat + + # Enable ML load prediction + load_ml_enable: True + # Use the output data in Predbat (can be False to explore the use without using the data) + load_ml_source: True +``` + +For best results: + +- Ensure you have a least a weeks worth of data before enabling load_ml_source. +- Make sure you do not have PredAI enabled at the same time +- Disable in day adjustment (switch.predbat_calculate_inday_adjustment) as the AI model will do that for you. + +### Recommended: Enable Temperature Predictions + +**For best results, enable the Temperature component to provide temperature forecasts:** + +```yaml +predbat: + # ... other config ... + + # Enable temperature predictions (RECOMMENDED for ML load prediction) + temperature_enable: true + + # Optional: specify coordinates (defaults to zone.home) + # temperature_latitude: 51.5074 + # temperature_longitude: -0.1278 +``` + +The temperature data significantly improves prediction accuracy for homes with heating/cooling systems, as energy consumption is often correlated with outside temperature. + +### Optional: Add PV Generation Data + +Your PV data will be picked from the pv_today setting in Predbat already + +```yaml +predbat: + # ... other config ... + + pv_today: + - sensor.my_solar_generation_today +``` + +### Optional: Subtract Car Charging + +If you have an EV charger set in Predbat then this will be subtracted from predictions. +If this is not set then the default EV charging threshold is used if car_charging_hold is True. + +```yaml +predbat: + # ... other config ... + + # Optional: subtract car charging from load history + car_charging_energy: + - sensor.my_ev_charger_energy +``` + +## Setup Instructions + +### Step 1: Verify Prerequisites + +Before enabling ML load prediction: + +1. Ensure you have a `load_today` sensor that tracks cumulative daily energy consumption +2. Optionally configure `pv_today` if you have solar panels +3. **Recommended**: Enable the Temperature component (Temperature Component in components documentation) +4. Ensure you have at least 1 day of historical data (7+ days recommended) + +### Step 2: Enable the Component + +Add `load_ml_enable: true` to your `apps.yaml` and restart Predbat. + +### Step 3: Wait for Initial Training + +On first run, the component will: + +1. Fetch historical load data (default: 7 days) +2. Train the neural network (takes 1-5 minutes depending on data) +3. Validate the model +4. Begin making predictions if validation passes + +Check the Predbat logs for training progress: + +```text +ML Component: Starting initial training +ML Predictor: Starting initial training with 50 epochs +ML Predictor: Training complete, final val_mae=0.3245 kWh +ML Component: Initial training completed, validation MAE=0.3245 kWh +``` + +### Step 4: Monitor Predictions + +Once trained, the component publishes predictions to: + +- `sensor.predbat_load_ml_forecast` - Contains 48-hour prediction in `results` attribute + +You can visualize these predictions in the Predbat web interface or by creating charts in Home Assistant. + +## Understanding the Model + +### Model Status + +The ML component tracks several status indicators: + +- **Model Status**: `not_initialized`, `training`, `active`, `validation_failed`, `stale` +- **Validation MAE**: Mean Absolute Error on validation data (in kWh per 5-min step) +- **Model Age**: How long since the model was last trained + +You can check model status in the Predbat logs or via the component status page in the web interface. + +### What Makes Good Predictions? + +Good predictions require: + +1. **Sufficient Historical Data**: At least 7 days recommended +2. **Consistent Patterns**: Regular daily/weekly routines improve accuracy +3. **Temperature Data**: Especially important for homes with electric heating/cooling +4. **Clean Data**: Avoid gaps or incorrect readings in historical data +5. **Recent Training**: Model should be retrained periodically (happens automatically) + +### Expected Accuracy + +Typical validation MAE values: + +- **Excellent**: < 0.3 kWh per 5-min step (~ 3.6 kW average) +- **Good**: 0.3 - 0.5 kWh per 5-min step +- **Fair**: 0.5 - 1.0 kWh per 5-min step +- **Poor**: > 1.0 kWh per 5-min step (may indicate issues) + +If validation MAE exceeds the threshold (default 2.0 kWh), predictions are disabled and the model will attempt to retrain. + +## Monitoring and Troubleshooting + +### Check Model Status + +View model status in Predbat logs: + +```text +ML Component: Model status: active, last trained: 2024-02-07 10:30:00 +ML Component: Validation MAE: 0.3245 kWh +``` + +### Common Issues + +**Issue**: Model never trains + +- **Cause**: Insufficient historical data +- **Solution**: Wait until you have at least 1 day of data, preferably 7+ days + +**Issue**: Validation MAE too high (predictions disabled) + +- **Cause**: Inconsistent load patterns, poor data quality, or insufficient training data +- **Solution**: + - Ensure historical data is accurate + - Add temperature data if not already enabled + - Wait for more historical data to accumulate + - Check for gaps or anomalies in historical data + +**Issue**: Model becomes stale + +- **Cause**: No training for 48+ hours +- **Solution**: Check logs for training failures, ensure Predbat is running continuously + +**Issue**: Predictions seem inaccurate + +- **Cause**: Changing household patterns, insufficient features, or missing temperature data +- **Solution**: + - Enable temperature predictions for better accuracy + - Wait for fine-tuning to adapt to new patterns + - Verify historical data quality + - Consider adding PV data if you have solar panels + +### Viewing Predictions + +Access predictions via: + +1. **Web Interface**: Navigate to the battery plan view to see ML predictions +2. **Home Assistant**: Check `sensor.predbat_load_ml_forecast` and its `results` attribute +3. **Logs**: Look for "ML Predictor: Generated predictions" messages + +## Model Persistence + +The trained model is saved to disk as `predbat_ml_model.npz` in your Predbat config directory. This file contains: + +- Network weights and biases +- Normalization parameters (mean, standard deviation) +- Training metadata (epochs, timestamp, version) + +The model is automatically loaded on Predbat restart, allowing predictions to continue immediately without retraining. + +If the model becomes unstable you can also delete this file to start again. + +--- + +## See Also + +- [Components Documentation](components.md) - Overview of all Predbat components +- [Configuration Guide](configuration-guide.md) - General configuration guidance +- [Temperature Component](components.md#temperature-api-temperature) - Setup guide for temperature forecasts +- [Customisation Guide](customisation.md) - Advanced customisation options diff --git a/mkdocs.yml b/mkdocs.yml index 1dc956759..ccf2f8afd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - energy-rates.md - apps-yaml.md - components.md + - load-ml.md - car-charging.md - configuration-guide.md - customisation.md