diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt
index 2a7b451f4..c27014566 100644
--- a/.cspell/custom-dictionary-workspace.txt
+++ b/.cspell/custom-dictionary-workspace.txt
@@ -7,6 +7,7 @@ AIO
AIO's
aiohttp
Alertfeed
+allclose
Anson
apexcharts
appdaemon
@@ -19,6 +20,11 @@ autoflake
automations
autopep
autoupdate
+axvline
+axvspan
+backprop
+Backpropagate
+backpropagation
Basepath
Batpred
battemperature
@@ -62,6 +68,7 @@ dayname
daynumber
daysymbol
dend
+denorm
devcontainer
devcontainers
dexport
@@ -91,6 +98,7 @@ energythroughput
epod
euids
evse
+exog
exportlimit
fdpwr
fdsoc
@@ -163,12 +171,17 @@ kvar
kvarh
kwargs
kwhb
+labelcolor
linebreak
+linestyle
+loadml
+loadmlpower
loadspower
localfolder
lockstep
logdata
loglines
+lookback
luxpower
markdownlint
matplotlib
@@ -254,6 +267,7 @@ pylint
pyproject
pytest
pytz
+randn
rarr
recp
Redownload
@@ -271,6 +285,7 @@ rstart
rtype
ruamel
saverestore
+savez
scalarstring
searr
securetoken
@@ -327,11 +342,13 @@ timekey
timelapse
timenow
timeobj
+timestep
timestr
timezone
tojson
Trefor
treforsiphone
+twinx
unsmoothed
unstaged
useid
@@ -349,6 +366,7 @@ wrongsha
xaxis
xaxistooltip
xlabel
+xlim
xload
xticks
yaxis
diff --git a/apps/predbat/components.py b/apps/predbat/components.py
index 749b754da..110e30d52 100644
--- a/apps/predbat/components.py
+++ b/apps/predbat/components.py
@@ -14,6 +14,7 @@
from ohme import OhmeAPI
from octopus import OctopusAPI
from carbon import CarbonAPI
+from temperature import TemperatureAPI
from axle import AxleAPI
from solax import SolaxAPI
from solis import SolisAPI
@@ -23,6 +24,7 @@
from db_manager import DatabaseManager
from fox import FoxAPI
from web_mcp import PredbatMCPServer
+from load_ml_component import LoadMLComponent
from datetime import datetime, timezone, timedelta
import asyncio
import os
@@ -220,6 +222,17 @@
},
"phase": 1,
},
+ "temperature": {
+ "class": TemperatureAPI,
+ "name": "External Temperature API",
+ "args": {
+ "temperature_enable": {"required_true": True, "config": "temperature_enable", "default": False},
+ "temperature_latitude": {"required": False, "config": "temperature_latitude", "default": None},
+ "temperature_longitude": {"required": False, "config": "temperature_longitude", "default": None},
+ "temperature_url": {"required": False, "config": "temperature_url", "default": "https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m&past_days=7"},
+ },
+ "phase": 1,
+ },
"axle": {
"class": AxleAPI,
"name": "Axle Energy",
@@ -265,6 +278,17 @@
"phase": 1,
"can_restart": True,
},
+ "load_ml": {
+ "class": LoadMLComponent,
+ "name": "ML Load Forecaster",
+ "event_filter": "predbat_load_ml_",
+ "args": {
+ "load_ml_enable": {"required_true": True, "config": "load_ml_enable", "default": False},
+ "load_ml_source": {"required": False, "config": "load_ml_source", "default": False},
+ },
+ "phase": 1,
+ "can_restart": True,
+ },
}
diff --git a/apps/predbat/config.py b/apps/predbat/config.py
index d1bf4c121..2d2bf4fd9 100644
--- a/apps/predbat/config.py
+++ b/apps/predbat/config.py
@@ -2101,4 +2101,5 @@
"forecast_solar_max_age": {"type": "float"},
"enable_coarse_fine_levels": {"type": "boolean"},
"load_power_fill_enable": {"type": "boolean"},
+ "load_ml_enable": {"type": "boolean"},
}
diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py
index 1a3bf8b8f..86517286a 100644
--- a/apps/predbat/fetch.py
+++ b/apps/predbat/fetch.py
@@ -507,7 +507,7 @@ def get_from_incrementing(self, data, index, backwards=True):
else:
return max(data.get(index + 1, 0) - data.get(index, 0), 0)
- def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True):
+ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True):
"""
Download one or more entities for import/export data
"""
@@ -529,7 +529,7 @@ def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None,
continue
try:
- history = self.get_history_wrapper(entity_id=entity_id, days=self.max_days_previous)
+ history = self.get_history_wrapper(entity_id=entity_id, days=max_days_previous)
except (ValueError, TypeError) as exc:
self.log("Warn: No history data found for {} : {}".format(entity_id, exc))
history = []
@@ -537,7 +537,7 @@ def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None,
if history and len(history) > 0:
import_today, _ = minute_data(
history[0],
- self.max_days_previous,
+ max_days_previous,
now_utc,
"state",
"last_updated",
@@ -674,8 +674,15 @@ def fetch_sensor_data(self, save=True):
self.iboost_today = dp2(abs(self.iboost_energy_today[0] - self.iboost_energy_today[self.minutes_now]))
self.log("iBoost energy today from sensor reads {} kWh".format(self.iboost_today))
+ # Fetch ML forecast if enabled
+ load_ml_forecast = {}
+ if self.get_arg("load_ml_enable", False) and self.get_arg("load_ml_source", False):
+ load_ml_forecast = self.fetch_ml_load_forecast(self.now_utc)
+ if load_ml_forecast:
+ self.load_forecast_only = True # Use only ML forecast for load if enabled and we have data
+
# Fetch extra load forecast
- self.load_forecast, self.load_forecast_array = self.fetch_extra_load_forecast(self.now_utc)
+ self.load_forecast, self.load_forecast_array = self.fetch_extra_load_forecast(self.now_utc, load_ml_forecast)
# Load previous load data
if self.get_arg("ge_cloud_data", False):
@@ -712,28 +719,28 @@ def fetch_sensor_data(self, save=True):
# Load import today data
if "import_today" in self.args:
- self.import_today = self.minute_data_import_export(self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
+ self.import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
self.import_today_now = get_now_from_cumulative(self.import_today, self.minutes_now, backwards=True)
else:
self.log("Warn: You have not set import_today in apps.yaml, you will have no previous import data")
# Load export today data
if "export_today" in self.args:
- self.export_today = self.minute_data_import_export(self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
+ self.export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
self.export_today_now = get_now_from_cumulative(self.export_today, self.minutes_now, backwards=True)
else:
self.log("Warn: You have not set export_today in apps.yaml, you will have no previous export data")
# PV today data
if "pv_today" in self.args:
- self.pv_today = self.minute_data_import_export(self.now_utc, "pv_today", required_unit="kWh")
+ self.pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh")
self.pv_today_now = get_now_from_cumulative(self.pv_today, self.minutes_now, backwards=True)
else:
self.log("Warn: You have not set pv_today in apps.yaml, you will have no previous PV data")
# Battery temperature
if "battery_temperature_history" in self.args:
- self.battery_temperature_history = self.minute_data_import_export(self.now_utc, "battery_temperature_history", scale=1.0, increment=False, smoothing=False)
+ self.battery_temperature_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, "battery_temperature_history", scale=1.0, increment=False, smoothing=False)
data = []
for minute in range(0, 24 * 60, 5):
data.append({minute: self.battery_temperature_history.get(minute, 0)})
@@ -1059,8 +1066,8 @@ def fetch_sensor_data(self, save=True):
# Fetch PV forecast if enabled, today must be enabled, other days are optional
self.pv_forecast_minute, self.pv_forecast_minute10 = self.fetch_pv_forecast()
- # Apply modal filter to historical data
if self.load_minutes and not self.load_forecast_only:
+ # Apply modal filter to historical data
self.previous_days_modal_filter(self.load_minutes)
self.log("Historical days now {} weight {}".format(self.days_previous, self.days_previous_weight))
@@ -1203,17 +1210,17 @@ def download_ge_data(self, now_utc):
self.log("GECloudData load_last_period from immediate sensor is {} kW".format(dp2(self.load_last_period)))
if "import_today" in self.args:
- import_today = self.minute_data_import_export(self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
+ import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
self.import_today_now = get_now_from_cumulative(import_today, self.minutes_now, backwards=True)
# Load export today data
if "export_today" in self.args:
- export_today = self.minute_data_import_export(self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
+ export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
self.export_today_now = get_now_from_cumulative(export_today, self.minutes_now, backwards=True)
# PV today data
if "pv_today" in self.args:
- pv_today = self.minute_data_import_export(self.now_utc, "pv_today", required_unit="kWh")
+ pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh")
self.pv_today_now = get_now_from_cumulative(pv_today, self.minutes_now, backwards=True)
self.log("Downloaded {} datapoints from GECloudData going back {} days".format(len(self.load_minutes), self.load_minutes_age))
@@ -1769,13 +1776,51 @@ def get_car_charging_planned(self):
)
)
- def fetch_extra_load_forecast(self, now_utc):
+ def fetch_ml_load_forecast(self, now_utc):
+ """
+ Fetches ML load forecast from sensor
+ and returns it as a minute_data dictionary
+ """
+ # Use ML Model for load prediction
+ load_ml_forecast = self.get_state_wrapper("sensor." + self.prefix + "_load_ml_forecast", attribute="results")
+ if load_ml_forecast:
+ self.log("Loading ML load forecast from sensor.sensor.{}_load_ml_forecast".format(self.prefix))
+ # Convert format from dict to array
+ if isinstance(load_ml_forecast, dict):
+ data_array = []
+ for key, value in load_ml_forecast.items():
+ data_array.append({"energy": value, "last_updated": key})
+
+ # Load data
+ load_forecast, _ = minute_data(
+ data_array,
+ self.forecast_days + 1,
+ self.midnight_utc,
+ "energy",
+ "last_updated",
+ backwards=False,
+ clean_increment=False,
+ smoothing=True,
+ divide_by=1.0,
+ scale=self.load_scaling,
+ )
+
+ if load_forecast:
+ self.log("Loaded the ML load forecast; now {}kWh to midnight {}kwh".format(load_forecast.get(self.minutes_now, 0), load_forecast.get(24 * 60 - PREDICT_STEP, 0)))
+ return load_forecast
+ return {}
+
+ def fetch_extra_load_forecast(self, now_utc, ml_forecast=None):
"""
Fetch extra load forecast, this is future load data
"""
load_forecast_final = {}
load_forecast_array = []
+ # Add ML forecast if available
+ if ml_forecast:
+ load_forecast_array.append(ml_forecast)
+
if "load_forecast" in self.args:
entity_ids = self.get_arg("load_forecast", indirect=False)
if isinstance(entity_ids, str):
@@ -1855,7 +1900,7 @@ def fetch_carbon_intensity(self, entity_id):
state = self.get_state_wrapper(entity_id=entity_id)
if state is not None:
try:
- carbon_history = self.minute_data_import_export(self.now_utc, entity_id, required_unit="g/kWh", increment=False, smoothing=False)
+ carbon_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, entity_id, required_unit="g/kWh", increment=False, smoothing=False)
except (ValueError, TypeError):
self.log("Warn: No carbon intensity history in sensor {}".format(entity_id))
else:
@@ -2166,7 +2211,7 @@ def load_car_energy(self, now_utc):
"""
self.car_charging_energy = {}
if "car_charging_energy" in self.args:
- self.car_charging_energy = self.minute_data_import_export(now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh")
+ self.car_charging_energy = self.minute_data_import_export(self.max_days_previous, now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh")
else:
self.log("Car charging hold {}, threshold {}kWh".format(self.car_charging_hold, self.car_charging_threshold * 60.0))
return self.car_charging_energy
diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py
new file mode 100644
index 000000000..b3f66d9d9
--- /dev/null
+++ b/apps/predbat/load_ml_component.py
@@ -0,0 +1,518 @@
+# -----------------------------------------------------------------------------
+# Predbat Home Battery System
+# Copyright Trefor Southwell 2025 - All Rights Reserved
+# This application maybe used for personal use only and not for commercial use
+# -----------------------------------------------------------------------------
+# ML Load Forecaster Component - ComponentBase wrapper for LoadPredictor
+# -----------------------------------------------------------------------------
+# fmt off
+# pylint: disable=consider-using-f-string
+# pylint: disable=line-too-long
+# pylint: disable=attribute-defined-outside-init
+
+import asyncio
+import os
+from datetime import datetime, timezone, timedelta
+from component_base import ComponentBase
+from utils import get_now_from_cumulative, dp2, minute_data
+from load_predictor import LoadPredictor, MODEL_VERSION
+from const import TIME_FORMAT, PREDICT_STEP
+import traceback
+
+# Training intervals
+RETRAIN_INTERVAL_SECONDS = 2 * 60 * 60 # 2 hours between training cycles
+PREDICTION_INTERVAL_SECONDS = 30 * 60 # 30 minutes between predictions
+
+
+class LoadMLComponent(ComponentBase):
+ """
+ ML Load Forecaster component that predicts household load for the next 48 hours.
+
+ This component:
+ - Fetches load history from configured sensor
+ - Optionally fills gaps using load_power sensor
+ - Subtracts configured sensors (e.g., car charging) from load
+ - Trains/fine-tunes an MLP model on historical load data
+ - Generates predictions in the same format as load_forecast
+ - Falls back to empty predictions when validation fails or model is stale
+ """
+
+ def initialize(self, load_ml_enable, load_ml_source=True):
+ """
+ Initialize the ML load forecaster component.
+
+ Args:
+ load_ml_enable: Whether ML forecasting is enabled
+ """
+ self.ml_enable = load_ml_enable
+ self.ml_source = load_ml_source
+ self.ml_load_sensor = self.get_arg("load_today", default=[], indirect=False)
+ self.ml_load_power_sensor = self.get_arg("load_power", default=[], indirect=False)
+ self.ml_pv_sensor = self.get_arg("pv_today", default=[], indirect=False)
+ self.ml_subtract_sensors = self.get_arg("car_charging_energy", default=[], indirect=False)
+ self.car_charging_hold = self.get_arg("car_charging_hold", True)
+ self.car_charging_threshold = float(self.get_arg("car_charging_threshold", 6.0)) / 60.0
+ self.car_charging_energy_scale = self.get_arg("car_charging_energy_scale", 1.0)
+ self.car_charging_rate = float(self.get_arg("car_charging_rate", 7.5)) / 60.0
+
+ self.ml_learning_rate = 0.001
+ self.ml_epochs_initial = 50
+ self.ml_epochs_update = 2
+ self.ml_min_days = 1
+ self.ml_validation_threshold = 2.0
+ self.ml_time_decay_days = 7
+ self.ml_max_load_kw = 50.0
+ self.ml_max_model_age_hours = 48
+
+ # Data state
+ self.load_data = None
+ self.load_data_age_days = 0
+ self.pv_data = None
+ self.temperature_data = None
+ self.data_ready = False
+ self.data_lock = asyncio.Lock()
+ self.last_data_fetch = None
+
+ # Model state
+ self.predictor = None
+ self.model_valid = False
+ self.model_status = "not_initialized"
+ self.last_train_time = None
+ self.initial_training_done = False
+
+ # Predictions cache
+ self.current_predictions = {}
+
+ # Model file path
+ self.model_filepath = None
+
+ # Validate configuration
+ if self.ml_enable and not self.ml_load_sensor:
+ self.log("Error: ML Component: ml_load_sensor must be configured when ml_enable is True")
+ self.ml_enable = False
+
+ # Initialize predictor
+ self._init_predictor()
+
+ def _init_predictor(self):
+ """Initialize or reinitialize the predictor."""
+ self.predictor = LoadPredictor(log_func=self.log, learning_rate=self.ml_learning_rate, max_load_kw=self.ml_max_load_kw)
+
+ # Determine model save path
+ if self.config_root:
+ self.model_filepath = os.path.join(self.config_root, "predbat_ml_model.npz")
+ else:
+ self.model_filepath = None
+
+ # Try to load existing model
+ if self.model_filepath and os.path.exists(self.model_filepath):
+ load_success = self.predictor.load(self.model_filepath)
+ if load_success:
+ self.log("ML Component: Loaded existing model")
+ # Check if model is still valid
+ is_valid, reason = self.predictor.is_valid(validation_threshold=self.ml_validation_threshold, max_age_hours=self.ml_max_model_age_hours)
+ if is_valid:
+ self.model_valid = True
+ self.model_status = "active"
+ self.initial_training_done = True
+ else:
+ self.log("ML Component: Loaded model is invalid ({}), will retrain".format(reason))
+ self.model_status = "fallback_" + reason
+ else:
+ # Model load failed (version mismatch, architecture change, etc.)
+ # Reinitialize predictor to ensure clean state
+ self.log("ML Component: Failed to load model, reinitializing predictor")
+ self.predictor = LoadPredictor(log_func=self.log, learning_rate=self.ml_learning_rate, max_load_kw=self.ml_max_load_kw)
+
+ async def _fetch_load_data(self):
+ """
+ Fetch and process load data from configured sensors.
+
+ Returns:
+ Tuple of (load_minutes_dict, age_days, load_minutes_now, pv_data) or (None, 0, 0, None) on failure
+ """
+ if not self.ml_load_sensor:
+ return None, 0, 0, None, None
+
+ try:
+ # Determine how many days of history to fetch, up to 7 days back
+ days_to_fetch = max(7, self.ml_min_days)
+
+ # Fetch load sensor history
+ self.log("ML Component: Fetching {} days of load history from {}".format(days_to_fetch, self.ml_load_sensor))
+
+ load_minutes, load_minutes_age = self.base.minute_data_load(self.now_utc, "load_today", days_to_fetch, required_unit="kWh", load_scaling=self.get_arg("load_scaling", 1.0), interpolate=True)
+ if not load_minutes:
+ self.log("Warn: ML Component: Failed to convert load history to minute data")
+ return None, 0, 0, None, None
+
+ if self.get_arg("load_power", default=None, indirect=False):
+ load_power_data, _ = self.base.minute_data_load(self.now_utc, "load_power", days_to_fetch, required_unit="W", load_scaling=1.0, interpolate=True)
+ load_minutes = self.base.fill_load_from_power(load_minutes, load_power_data)
+
+ # Get current cumulative load value
+ load_minutes_now = get_now_from_cumulative(load_minutes, self.minutes_now, backwards=True)
+
+ car_charging_energy = {}
+ if self.get_arg("car_charging_energy", default=None, indirect=False):
+ car_charging_energy = self.base.minute_data_import_export(days_to_fetch, self.now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh")
+
+ max_minute = max(load_minutes.keys()) if load_minutes else 0
+ max_minute = (max_minute // 5) * 5 # Align to 5-minute intervals
+ load_minutes_new = {}
+
+ # Subtract configured sensors (e.g., car charging)
+ total_load_energy = 0
+ car_delta = 0.0
+ STEP = PREDICT_STEP
+ for minute in range(max_minute, -STEP, -STEP):
+ if self.car_charging_hold and car_charging_energy:
+ car_delta = abs(car_charging_energy.get(minute, 0.0) - car_charging_energy.get(minute - STEP, car_charging_energy.get(minute, 0.0)))
+ elif self.car_charging_hold:
+ load_now = abs(load_minutes.get(minute, 0.0) - load_minutes.get(minute - STEP, load_minutes.get(minute, 0.0)))
+ if load_now >= self.car_charging_threshold * STEP:
+ car_delta = self.car_charging_rate * STEP
+ if car_delta > 0:
+ # When car is enable spread over 5 minutes due to alignment between car and house load data
+ load_delta = abs(load_minutes.get(minute, 0.0) - load_minutes.get(minute - STEP, load_minutes.get(minute, 0.0)))
+ load_delta = max(0.0, load_delta - car_delta)
+ for m in range(minute, minute - STEP, -1):
+ load_minutes_new[m] = total_load_energy + load_delta / STEP
+ total_load_energy += load_delta
+ else:
+ # Otherwise just copy load data
+ for m in range(minute, minute - STEP, -1):
+ load_delta = abs(load_minutes.get(minute, 0.0) - load_minutes.get(minute - 1, load_minutes.get(minute, 0.0)))
+ load_minutes_new[m] = total_load_energy
+ total_load_energy += load_delta
+
+ # Calculate age of data
+ age_days = max_minute / (24 * 60)
+
+ # PV Data
+ if self.ml_pv_sensor:
+ pv_data, _ = self.base.minute_data_load(self.now_utc, "pv_today", days_to_fetch, required_unit="kWh", load_scaling=1.0, interpolate=True)
+ else:
+ pv_data = {}
+
+ # Temperature predictions
+ temp_entity = "sensor." + self.prefix + "_temperature"
+ temperature_info = self.get_state_wrapper(temp_entity, attribute="results")
+ temperature_data = {}
+ if isinstance(temperature_info, dict):
+ data_array = []
+ for key, value in temperature_info.items():
+ data_array.append({"state": value, "last_updated": key})
+
+ # Load data from past and future predictions, base backwards around now_utc
+ # We also get the last 7 days in the past to help the model learn the daily pattern
+ temperature_data, _ = minute_data(
+ data_array,
+ days_to_fetch,
+ self.now_utc,
+ "state",
+ "last_updated",
+ backwards=True,
+ clean_increment=False,
+ smoothing=True,
+ divide_by=1.0,
+ scale=1.0,
+ )
+ self.log("ML Temperature data points: {}".format(len(temperature_data)))
+
+ self.log("ML Component: Fetched {} load data points, {:.1f} days of history".format(len(load_minutes_new), age_days))
+ # with open("input_train_data.json", "w") as f:
+ # import json
+ # json.dump([load_minutes_new, age_days, load_minutes_now, pv_data, temperature_data], f, indent=2)
+ return load_minutes_new, age_days, load_minutes_now, pv_data, temperature_data
+
+ except Exception as e:
+ self.log("Error: ML Component: Failed to fetch load data: {}".format(e))
+ self.log("Error: ML Component: {}".format(traceback.format_exc()))
+ return None, 0, 0, None, None
+
+ def get_current_prediction(self):
+ """
+ Returns the current ML load predictions.
+
+ Output format:
+ Dict of {minute: cumulative_kwh}
+ """
+ return self.current_predictions
+
+ def _get_predictions(self, now_utc, midnight_utc, exog_features=None):
+ """
+ Get current predictions for integration with load_forecast.
+
+ Called from fetch.py to retrieve ML predictions.
+
+ Args:
+ now_utc: Current UTC timestamp
+ midnight_utc: Today's midnight UTC timestamp
+ exog_features: Optional dict with future exogenous data
+
+ Returns:
+ Dict of {minute: cumulative_kwh} or empty dict on fallback
+ """
+ if not self.ml_enable:
+ return {}
+
+ if not self.data_ready:
+ self.log("ML Component: No load data available for prediction")
+ return {}
+
+ if not self.model_valid:
+ self.log("ML Component: Model not valid ({}), returning empty predictions".format(self.model_status))
+ return {}
+
+ # Generate predictions using current model
+ try:
+ predictions = self.predictor.predict(self.load_data, now_utc, midnight_utc, pv_minutes=self.pv_data, temp_minutes=self.temperature_data, exog_features=exog_features)
+
+ if predictions:
+ self.current_predictions = predictions
+ self.log("ML Component: Generated {} predictions (total {:.2f} kWh over 48h)".format(len(predictions), max(predictions.values()) if predictions else 0))
+
+ return predictions
+
+ except Exception as e:
+ self.log("Error: ML Component: Prediction failed: {}".format(e))
+ return {}
+
+ async def run(self, seconds, first):
+ """
+ Main component loop - handles data fetching, training and prediction cycles.
+
+ Args:
+ seconds: Seconds since component start
+ first: True if this is the first run
+
+ Returns:
+ True if successful, False otherwise
+ """
+ if not self.ml_enable:
+ self.api_started = True
+ return True
+
+ # Fetch fresh load data periodically (every 15 minutes)
+ should_fetch = first or ((seconds % PREDICTION_INTERVAL_SECONDS) == 0)
+
+ if should_fetch:
+ async with self.data_lock:
+ load_data, age_days, load_minutes_now, pv_data, temperature_data = await self._fetch_load_data()
+ if load_data:
+ self.load_data = load_data
+ self.load_data_age_days = age_days
+ self.load_minutes_now = load_minutes_now
+ self.data_ready = True
+ self.last_data_fetch = self.now_utc
+ pv_forecast_minute, pv_forecast_minute10 = self.base.fetch_pv_forecast()
+ # PV Data has the historical PV data (minute is the number of minutes in the past)
+ # PV forecast has the predicted PV generation for the next 24 hours (minute is the number of minutes from midnight forward
+ # Combine the two into a new dict where negative minutes are in the future and positive in the past
+ self.pv_data = pv_data
+ current_value = pv_data.get(0, 0)
+ if pv_forecast_minute:
+ max_minute = max(pv_forecast_minute.keys()) + PREDICT_STEP
+ for minute in range(self.minutes_now + PREDICT_STEP, max_minute, PREDICT_STEP):
+ current_value += pv_forecast_minute.get(minute, current_value)
+ pv_data[-minute + self.minutes_now] = current_value
+ self.temperature_data = temperature_data
+ else:
+ self.log("Warn: ML Component: Failed to fetch load data")
+
+ # Check if we have data
+ if not self.data_ready:
+ if first:
+ self.log("ML Component: Waiting for load data from sensors")
+ return True # Not an error, just waiting
+
+ # Check if we have enough data
+ if self.load_data_age_days < self.ml_min_days:
+ self.model_status = "insufficient_data"
+ self.model_valid = False
+ if first:
+ self.log("ML Component: Insufficient data ({:.1f} days, need {})".format(self.load_data_age_days, self.ml_min_days))
+ return True
+
+ # Determine if training is needed
+ should_train = False
+ is_initial = False
+
+ if not self.initial_training_done:
+ # First training
+ should_train = True
+ is_initial = True
+ self.log("ML Component: Starting initial training")
+ elif seconds % RETRAIN_INTERVAL_SECONDS == 0:
+ # Periodic fine-tuning every 2 hours
+ should_train = True
+ is_initial = False
+ self.log("ML Component: Starting fine-tune training (2h interval)")
+
+ if should_train:
+ await self._do_training(is_initial)
+
+ # Update model validity status
+ self._update_model_status()
+
+ if should_fetch:
+ self._get_predictions(self.now_utc, self.midnight_utc)
+ # Publish entity with current state
+ self._publish_entity()
+ self.log("ML Component: Prediction cycle completed")
+
+ self.update_success_timestamp()
+ return True
+
+ async def _do_training(self, is_initial):
+ """
+ Perform model training.
+
+ Args:
+ is_initial: True for full training, False for fine-tuning
+ """
+ async with self.data_lock:
+ if not self.load_data:
+ self.log("Warn: ML Component: No data for training")
+ return
+
+ # Warn if limited data
+ if self.load_data_age_days < 3:
+ self.log("Warn: ML Component: Training with only {} days of data, recommend 3+ days for better accuracy".format(self.load_data_age_days))
+
+ try:
+ # Run training in executor to avoid blocking
+ epochs = self.ml_epochs_initial if is_initial else self.ml_epochs_update
+
+ val_mae = self.predictor.train(self.load_data, self.now_utc, pv_minutes=self.pv_data, temp_minutes=self.temperature_data, is_initial=is_initial, epochs=epochs, time_decay_days=self.ml_time_decay_days)
+
+ if val_mae is not None:
+ self.last_train_time = datetime.now(timezone.utc)
+ self.initial_training_done = True
+
+ # Check validation threshold
+ if val_mae <= self.ml_validation_threshold:
+ self.model_valid = True
+ self.model_status = "active"
+ self.log("ML Component: Training successful, val_mae={:.4f} kWh".format(val_mae))
+ else:
+ self.model_valid = False
+ self.model_status = "fallback_validation"
+ self.log("Warn: ML Component: Validation MAE ({:.4f}) exceeds threshold ({:.4f})".format(val_mae, self.ml_validation_threshold))
+
+ # Save model
+ if self.model_filepath:
+ self.predictor.save(self.model_filepath)
+ else:
+ self.log("Warn: ML Component: Training failed")
+
+ except Exception as e:
+ self.log("Error: ML Component: Training exception: {}".format(e))
+ self.log("Error: " + traceback.format_exc())
+
+ def _update_model_status(self):
+ """Update model validity status based on current state."""
+ if not self.predictor or not self.predictor.model_initialized:
+ self.model_valid = False
+ self.model_status = "not_initialized"
+ return
+
+ is_valid, reason = self.predictor.is_valid(validation_threshold=self.ml_validation_threshold, max_age_hours=self.ml_max_model_age_hours)
+
+ if is_valid:
+ self.model_valid = True
+ self.model_status = "active"
+ else:
+ self.model_valid = False
+ self.model_status = "fallback_" + reason
+
+ def _publish_entity(self):
+ """Publish the load_forecast_ml entity with current predictions."""
+ # Convert predictions to timestamp format for entity
+ results = {}
+ reset_amount = 0
+ load_today_h1 = 0
+ load_today_h8 = 0
+ load_today_now = 0
+ power_today_now = 0
+ power_today_h1 = 0
+ power_today_h8 = 0
+ # Future predictions
+ if self.current_predictions:
+ prev_value = 0
+ for minute, value in self.current_predictions.items():
+ timestamp = self.midnight_utc + timedelta(minutes=minute + self.minutes_now)
+ timestamp_str = timestamp.strftime(TIME_FORMAT)
+ # Reset at midnight
+ if minute > 0 and ((minute + self.minutes_now) % (24 * 60) == 0):
+ reset_amount = value + self.load_minutes_now
+ output_value = round(value - reset_amount + self.load_minutes_now, 4)
+ results[timestamp_str] = output_value
+ delta_value = (value - prev_value) / PREDICT_STEP * 60.0
+ if minute == 0:
+ power_today_now = delta_value
+ if minute == 60:
+ load_today_h1 = output_value
+ power_today_h1 = delta_value
+ if minute == 60 * 8:
+ load_today_h8 = output_value
+ power_today_h8 = delta_value
+ prev_value = value
+
+ # Get model age
+ model_age_hours = self.predictor.get_model_age_hours() if self.predictor else None
+
+ # Calculate total predicted load
+ total_kwh = max(self.current_predictions.values()) if self.current_predictions else 0
+
+ self.dashboard_item(
+ "sensor." + self.prefix + "_load_ml_forecast",
+ state=self.model_status,
+ attributes={
+ "results": results,
+ "friendly_name": "ML Load Forecast",
+ "icon": "mdi:chart-line",
+ },
+ app="load_ml",
+ )
+ self.dashboard_item(
+ "sensor." + self.prefix + "_load_ml_stats",
+ state=round(total_kwh, 2),
+ attributes={
+ "load_today": dp2(self.load_minutes_now),
+ "load_today_h1": dp2(load_today_h1),
+ "load_today_h8": dp2(load_today_h8),
+ "load_total": dp2(total_kwh),
+ "power_today_now": dp2(power_today_now),
+ "power_today_h1": dp2(power_today_h1),
+ "power_today_h8": dp2(power_today_h8),
+ "mae_kwh": round(self.predictor.validation_mae, 4) if self.predictor and self.predictor.validation_mae else None,
+ "last_trained": self.last_train_time.isoformat() if self.last_train_time else None,
+ "model_age_hours": round(model_age_hours, 1) if model_age_hours else None,
+ "training_days": self.load_data_age_days,
+ "status": self.model_status,
+ "model_version": MODEL_VERSION,
+ "epochs_trained": self.predictor.epochs_trained if self.predictor else 0,
+ "friendly_name": "ML Load Stats",
+ "state_class": "measurement",
+ "unit_of_measurement": "kWh",
+ "icon": "mdi:chart-line",
+ },
+ app="load_ml",
+ )
+
+ def last_updated_time(self):
+ """Return last successful update time for component health check."""
+ return self.last_success_timestamp
+
+ def is_alive(self):
+ """Check if component is alive and functioning."""
+ if not self.ml_enable:
+ return True
+
+ if self.last_success_timestamp is None:
+ return False
+
+ age = datetime.now(timezone.utc) - self.last_success_timestamp
+ return age < timedelta(minutes=10)
diff --git a/apps/predbat/load_predictor.py b/apps/predbat/load_predictor.py
new file mode 100644
index 000000000..7ff27014a
--- /dev/null
+++ b/apps/predbat/load_predictor.py
@@ -0,0 +1,1059 @@
+# -----------------------------------------------------------------------------
+# Predbat Home Battery System
+# Copyright Trefor Southwell 2025 - All Rights Reserved
+# This application maybe used for personal use only and not for commercial use
+# -----------------------------------------------------------------------------
+# Lightweight ML Load Predictor - NumPy-only MLP implementation
+# -----------------------------------------------------------------------------
+# fmt off
+# pylint: disable=consider-using-f-string
+# pylint: disable=line-too-long
+# pylint: disable=attribute-defined-outside-init
+
+import numpy as np
+import json
+import os
+from datetime import datetime, timezone, timedelta
+
+# Architecture constants (not user-configurable)
+MODEL_VERSION = 5 # Bumped for temperature feature
+LOOKBACK_STEPS = 288 # 24 hours at 5-min intervals
+OUTPUT_STEPS = 1 # Single step output (autoregressive)
+PREDICT_HORIZON = 576 # 48 hours of predictions (576 * 5 min)
+HIDDEN_SIZES = [512, 256, 128, 64] # Deeper network with more capacity
+BATCH_SIZE = 128 # Smaller batches for better gradient estimates
+FINETUNE_HOURS = 24 # Hours of data for fine-tuning
+STEP_MINUTES = 5 # Minutes per step
+
+# Feature constants
+NUM_TIME_FEATURES = 4 # sin/cos minute-of-day, sin/cos day-of-week (for TARGET time)
+NUM_LOAD_FEATURES = LOOKBACK_STEPS # Historical load values
+NUM_PV_FEATURES = LOOKBACK_STEPS # Historical PV generation values
+NUM_TEMP_FEATURES = LOOKBACK_STEPS # Historical temperature values
+TOTAL_FEATURES = NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES + NUM_TIME_FEATURES
+
+
+def relu(x):
+ """ReLU activation function"""
+ return np.maximum(0, x)
+
+
+def relu_derivative(x):
+ """Derivative of ReLU"""
+ return (x > 0).astype(np.float32)
+
+
+def huber_loss(y_true, y_pred, delta=1.0):
+ """Huber loss - robust to outliers"""
+ error = y_true - y_pred
+ abs_error = np.abs(error)
+ quadratic = np.minimum(abs_error, delta)
+ linear = abs_error - quadratic
+ return np.mean(0.5 * quadratic**2 + delta * linear)
+
+
+def huber_loss_derivative(y_true, y_pred, delta=1.0):
+ """Derivative of Huber loss"""
+ error = y_pred - y_true
+ abs_error = np.abs(error)
+ return np.where(abs_error <= delta, error, delta * np.sign(error)) / y_true.shape[0]
+
+
+def mse_loss(y_true, y_pred):
+ """Mean Squared Error loss"""
+ return np.mean((y_true - y_pred) ** 2)
+
+
+def mse_loss_derivative(y_true, y_pred):
+ """Derivative of MSE loss"""
+ return 2 * (y_pred - y_true) / y_true.shape[0]
+
+
+class LoadPredictor:
+ """
+ Lightweight MLP-based load predictor using NumPy only.
+
+ Predicts household electrical load for the next 48 hours using:
+ - Historical load data (lookback window)
+ - Cyclical time encodings (hour-of-day, day-of-week)
+ - Placeholder for future exogenous features (temperature, solar)
+ """
+
+ def __init__(self, log_func=None, learning_rate=0.001, max_load_kw=23.0):
+ """
+ Initialize the load predictor.
+
+ Args:
+ log_func: Logging function (defaults to print)
+ learning_rate: Learning rate for Adam optimizer
+ max_load_kw: Maximum load in kW for clipping predictions
+ """
+ self.log = log_func if log_func else print
+ self.learning_rate = learning_rate
+ self.max_load_kw = max_load_kw
+
+ # Model weights (initialized on first train)
+ self.weights = None
+ self.biases = None
+
+ # Adam optimizer state
+ self.m_weights = None
+ self.v_weights = None
+ self.m_biases = None
+ self.v_biases = None
+ self.adam_t = 0
+
+ # Normalization parameters
+ self.feature_mean = None
+ self.feature_std = None
+ self.target_mean = None
+ self.target_std = None
+ self.pv_mean = None
+ self.pv_std = None
+
+ # Training metadata
+ self.training_timestamp = None
+ self.validation_mae = None
+ self.epochs_trained = 0
+ self.model_initialized = False
+
+ def _initialize_weights(self):
+ """Initialize network weights using Xavier initialization"""
+ np.random.seed(42) # For reproducibility
+
+ layer_sizes = [TOTAL_FEATURES] + HIDDEN_SIZES + [OUTPUT_STEPS]
+
+ self.weights = []
+ self.biases = []
+ self.m_weights = []
+ self.v_weights = []
+ self.m_biases = []
+ self.v_biases = []
+
+ for i in range(len(layer_sizes) - 1):
+ fan_in = layer_sizes[i]
+ fan_out = layer_sizes[i + 1]
+
+ # Xavier initialization
+ std = np.sqrt(2.0 / (fan_in + fan_out))
+ w = np.random.randn(fan_in, fan_out).astype(np.float32) * std
+ b = np.zeros(fan_out, dtype=np.float32)
+
+ self.weights.append(w)
+ self.biases.append(b)
+
+ # Adam optimizer momentum terms
+ self.m_weights.append(np.zeros_like(w))
+ self.v_weights.append(np.zeros_like(w))
+ self.m_biases.append(np.zeros_like(b))
+ self.v_biases.append(np.zeros_like(b))
+
+ self.adam_t = 0
+ self.model_initialized = True
+
+ def _forward(self, X):
+ """
+ Forward pass through the network.
+
+ Args:
+ X: Input features (batch_size, TOTAL_FEATURES)
+
+ Returns:
+ Output predictions and list of layer activations for backprop
+ """
+ activations = [X]
+ pre_activations = []
+
+ current = X
+ for i, (w, b) in enumerate(zip(self.weights, self.biases)):
+ z = np.dot(current, w) + b
+ pre_activations.append(z)
+
+ # Apply ReLU for hidden layers, linear for output
+ if i < len(self.weights) - 1:
+ current = relu(z)
+ else:
+ current = z # Linear output
+
+ activations.append(current)
+
+ return current, activations, pre_activations
+
+ def _backward(self, y_true, activations, pre_activations, sample_weights=None):
+ """
+ Backward pass using backpropagation.
+
+ Args:
+ y_true: True target values
+ activations: Layer activations from forward pass
+ pre_activations: Pre-activation values from forward pass
+ sample_weights: Optional per-sample weights for weighted loss
+
+ Returns:
+ Gradients for weights and biases
+ """
+ batch_size = y_true.shape[0]
+
+ # Output layer gradient (MSE loss derivative)
+ delta = mse_loss_derivative(y_true, activations[-1])
+
+ # Apply sample weights to gradient if provided
+ if sample_weights is not None:
+ delta = delta * sample_weights.reshape(-1, 1)
+
+ weight_grads = []
+ bias_grads = []
+
+ # Backpropagate through layers
+ for i in range(len(self.weights) - 1, -1, -1):
+ # Gradient for weights and biases
+ weight_grads.insert(0, np.dot(activations[i].T, delta))
+ bias_grads.insert(0, np.sum(delta, axis=0))
+
+ if i > 0:
+ # Propagate gradient to previous layer
+ delta = np.dot(delta, self.weights[i].T) * relu_derivative(pre_activations[i - 1])
+
+ return weight_grads, bias_grads
+
+ def _adam_update(self, weight_grads, bias_grads, beta1=0.9, beta2=0.999, epsilon=1e-8):
+ """
+ Update weights using Adam optimizer.
+
+ Args:
+ weight_grads: Gradients for weights
+ bias_grads: Gradients for biases
+ beta1: Exponential decay rate for first moment
+ beta2: Exponential decay rate for second moment
+ epsilon: Small constant for numerical stability
+ """
+ self.adam_t += 1
+
+ for i in range(len(self.weights)):
+ # Update momentum for weights
+ self.m_weights[i] = beta1 * self.m_weights[i] + (1 - beta1) * weight_grads[i]
+ self.v_weights[i] = beta2 * self.v_weights[i] + (1 - beta2) * (weight_grads[i] ** 2)
+
+ # Bias correction
+ m_hat = self.m_weights[i] / (1 - beta1**self.adam_t)
+ v_hat = self.v_weights[i] / (1 - beta2**self.adam_t)
+
+ # Update weights
+ self.weights[i] -= self.learning_rate * m_hat / (np.sqrt(v_hat) + epsilon)
+
+ # Update momentum for biases
+ self.m_biases[i] = beta1 * self.m_biases[i] + (1 - beta1) * bias_grads[i]
+ self.v_biases[i] = beta2 * self.v_biases[i] + (1 - beta2) * (bias_grads[i] ** 2)
+
+ # Bias correction
+ m_hat = self.m_biases[i] / (1 - beta1**self.adam_t)
+ v_hat = self.v_biases[i] / (1 - beta2**self.adam_t)
+
+ # Update biases
+ self.biases[i] -= self.learning_rate * m_hat / (np.sqrt(v_hat) + epsilon)
+
+ def _create_time_features(self, minute_of_day, day_of_week):
+ """
+ Create cyclical time features.
+
+ Args:
+ minute_of_day: Minutes since midnight (0-1439)
+ day_of_week: Day of week (0-6, Monday=0)
+
+ Returns:
+ Array of 4 time features: sin/cos minute, sin/cos day
+ """
+ # Cyclical encoding for minute of day
+ minute_sin = np.sin(2 * np.pi * minute_of_day / 1440)
+ minute_cos = np.cos(2 * np.pi * minute_of_day / 1440)
+
+ # Cyclical encoding for day of week
+ day_sin = np.sin(2 * np.pi * day_of_week / 7)
+ day_cos = np.cos(2 * np.pi * day_of_week / 7)
+
+ return np.array([minute_sin, minute_cos, day_sin, day_cos], dtype=np.float32)
+
+ def _add_exog_features(self, X, exog_dict=None):
+ """
+ Placeholder for adding exogenous features (temperature, solar).
+
+ Args:
+ X: Current feature array
+ exog_dict: Dictionary with optional "temperature" and "solar" data
+
+ Returns:
+ Extended feature array (currently just returns X unchanged)
+ """
+ # Future expansion: add temperature/solar features here
+ if exog_dict:
+ pass # Placeholder for future implementation
+ return X
+
+ def _load_to_energy_per_step(self, load_minutes, step=STEP_MINUTES):
+ """
+ Convert cumulative load_minutes dict to energy per step (kWh per 5 min).
+
+ The load_minutes dict contains cumulative kWh values:
+ - Positive minutes: going backwards in time (historical data)
+ - Negative minutes: going forward in time (future forecasts)
+ Energy consumption for a period is the difference between start and end.
+
+ Args:
+ load_minutes: Dict of {minute: cumulative_kwh}
+ step: Step size in minutes
+
+ Returns:
+ Dict of {minute: energy_kwh_per_step}
+ """
+ energy_per_step = {}
+
+ if not load_minutes:
+ return energy_per_step
+
+ # Get both positive (historical) and negative (future) minute ranges
+ all_minutes = list(load_minutes.keys())
+ if not all_minutes:
+ return energy_per_step
+
+ max_minute = max(all_minutes)
+ min_minute = min(all_minutes)
+
+ # Process historical data (positive minutes, going backwards)
+ for minute in range(0, max_minute, step):
+ # Energy = cumulative_now - cumulative_later (going backwards)
+ val_now = load_minutes.get(minute, 0)
+ val_next = load_minutes.get(minute + step, 0)
+ energy = max(val_now - val_next, 0) # Ensure non-negative
+ energy_per_step[minute] = energy
+
+ # Process future data (negative minutes, going forwards)
+ if min_minute < 0:
+ # Need to go from min_minute (-XXX) towards 0 in positive steps
+ # So we go from min to 0-step in steps of +step
+ for minute in range(min_minute, -step + 1, step):
+ # For future: energy = cumulative_now - cumulative_later (cumulative decreases going forward)
+ val_now = load_minutes.get(minute, 0)
+ val_next = load_minutes.get(minute + step, 0)
+ energy = max(val_now - val_next, 0) # Ensure non-negative
+ energy_per_step[minute] = energy
+
+ return energy_per_step
+
+ def _compute_daily_pattern(self, energy_per_step, smoothing_window=6):
+ """
+ Compute average daily pattern from historical data.
+
+ Groups energy values by minute-of-day and computes rolling average.
+ Used to blend with predictions to prevent autoregressive drift.
+
+ Args:
+ energy_per_step: Dict of {minute: energy_kwh}
+ smoothing_window: Number of adjacent slots to smooth over
+
+ Returns:
+ Dict of {minute_of_day: avg_energy} for 288 slots in a day
+ """
+ # Collect energy values by minute-of-day (0 to 1435 in 5-min steps)
+ by_minute = {}
+ for minute, energy in energy_per_step.items():
+ minute_of_day = minute % (24 * 60) # 0-1439
+ # Align to 5-minute boundaries
+ slot = (minute_of_day // STEP_MINUTES) * STEP_MINUTES
+ if slot not in by_minute:
+ by_minute[slot] = []
+ by_minute[slot].append(energy)
+
+ # Compute mean for each slot
+ pattern = {}
+ for slot in range(0, 24 * 60, STEP_MINUTES):
+ if slot in by_minute and len(by_minute[slot]) > 0:
+ pattern[slot] = float(np.mean(by_minute[slot]))
+ else:
+ pattern[slot] = 0.05 # Default fallback
+
+ # Apply smoothing to reduce noise
+ slots = sorted(pattern.keys())
+ smoothed = {}
+ for i, slot in enumerate(slots):
+ values = []
+ for offset in range(-smoothing_window // 2, smoothing_window // 2 + 1):
+ idx = (i + offset) % len(slots)
+ values.append(pattern[slots[idx]])
+ smoothed[slot] = float(np.mean(values))
+
+ return smoothed
+
+ def _create_dataset(self, load_minutes, now_utc, pv_minutes=None, temp_minutes=None, is_finetune=False, time_decay_days=7, validation_holdout_hours=24):
+ """
+ Create training dataset from load_minutes dict.
+
+ For autoregressive prediction: each sample uses 24h lookback to predict
+ the next single 5-minute step. Time features are for the TARGET time.
+
+ Training uses all available data (from most recent to as far back as data goes).
+ Validation uses the most recent 24h as a subset of training data to check model fit.
+
+ Args:
+ load_minutes: Dict of {minute: cumulative_kwh} going backwards in time
+ now_utc: Current UTC timestamp
+ pv_minutes: Dict of {minute: cumulative_kwh} PV generation (backwards for history, negative for future)
+ temp_minutes: Dict of {minute: temperature_celsius} Temperature (backwards for history, negative for future)
+ is_finetune: If True, only use last 24 hours; else use full data with time-decay
+ time_decay_days: Time constant for exponential decay weighting
+ validation_holdout_hours: Hours of most recent data to hold out for validation
+
+ Returns:
+ X_train, y_train, train_weights: Training data
+ X_val, y_val: Validation data (most recent period)
+ """
+ # Convert to energy per step
+ energy_per_step = self._load_to_energy_per_step(load_minutes)
+ pv_energy_per_step = self._load_to_energy_per_step(pv_minutes) if pv_minutes else {}
+ # Temperature is not cumulative, so just use the raw values (already in correct format)
+ temp_values = temp_minutes if temp_minutes else {}
+
+ if not energy_per_step:
+ return None, None, None, None, None
+
+ max_minute = max(energy_per_step.keys())
+
+ # Determine data range
+ if is_finetune:
+ # Only use last 48 hours for fine-tuning (24h train + 24h for lookback)
+ start_minute = 0
+ end_minute = min(48 * 60, max_minute)
+ validation_holdout_hours = 12 # Smaller holdout for fine-tuning
+ else:
+ # Use 7 days of data for initial training
+ start_minute = 0
+ end_minute = min(7 * 24 * 60, max_minute)
+
+ # Need enough history for lookback plus validation holdout
+ min_required = LOOKBACK_STEPS * STEP_MINUTES + validation_holdout_hours * 60 + STEP_MINUTES
+
+ if end_minute < min_required:
+ self.log("Warn: Insufficient data for ML training, need {} minutes, have {}".format(min_required, end_minute))
+ return None, None, None, None, None
+
+ # Validation uses most recent data (minute 0 to validation_holdout)
+ # Training uses ALL data (minute 0 to end_minute), including validation period
+ validation_end = validation_holdout_hours * 60
+
+ X_train_list = []
+ y_train_list = []
+ weight_list = []
+ X_val_list = []
+ y_val_list = []
+
+ # Create training samples (from all available data, including most recent)
+ # These samples predict targets in the range [0, end_minute - lookback]
+ for target_minute in range(0, end_minute - LOOKBACK_STEPS * STEP_MINUTES, STEP_MINUTES):
+ # Lookback window starts at target_minute + STEP_MINUTES (one step after target)
+ lookback_start = target_minute + STEP_MINUTES
+
+ # Extract lookback window (24 hours of history before the target)
+ lookback_values = []
+ pv_lookback_values = []
+ temp_lookback_values = []
+ valid_sample = True
+
+ for lb_offset in range(LOOKBACK_STEPS):
+ lb_minute = lookback_start + lb_offset * STEP_MINUTES
+ if lb_minute in energy_per_step:
+ lookback_values.append(energy_per_step[lb_minute])
+ # Add PV generation for the same time period (0 if no PV data)
+ pv_lookback_values.append(pv_energy_per_step.get(lb_minute, 0.0))
+ # Add temperature for the same time period (0 if no temp data)
+ temp_lookback_values.append(temp_values.get(lb_minute, 0.0))
+ else:
+ valid_sample = False
+ break
+
+ if not valid_sample or len(lookback_values) != LOOKBACK_STEPS:
+ continue
+
+ # Target is the single next step we're predicting
+ if target_minute not in energy_per_step:
+ continue
+ target_value = energy_per_step[target_minute]
+
+ # Calculate time features for the TARGET time (what we're predicting)
+ target_time = now_utc - timedelta(minutes=target_minute)
+ minute_of_day = target_time.hour * 60 + target_time.minute
+ day_of_week = target_time.weekday()
+ time_features = self._create_time_features(minute_of_day, day_of_week)
+
+ # Combine features: [load_lookback..., pv_lookback..., temp_lookback..., time_features...]
+ features = np.concatenate([np.array(lookback_values, dtype=np.float32), np.array(pv_lookback_values, dtype=np.float32), np.array(temp_lookback_values, dtype=np.float32), time_features])
+
+ X_train_list.append(features)
+ y_train_list.append(np.array([target_value], dtype=np.float32))
+
+ # Time-decay weighting (older samples get lower weight)
+ age_days = target_minute / (24 * 60)
+ if is_finetune:
+ weight = 1.0 # Equal weight for fine-tuning
+ else:
+ weight = np.exp(-age_days / time_decay_days)
+ weight_list.append(weight)
+
+ # Create validation samples (from most recent data, minute 0 to validation_end)
+ # These samples use lookback from validation_end onwards to predict the holdout period
+ for target_minute in range(0, validation_end, STEP_MINUTES):
+ # Lookback window starts at target_minute + STEP_MINUTES
+ lookback_start = target_minute + STEP_MINUTES
+
+ # Extract lookback window
+ lookback_values = []
+ pv_lookback_values = []
+ temp_lookback_values = []
+ valid_sample = True
+
+ for lb_offset in range(LOOKBACK_STEPS):
+ lb_minute = lookback_start + lb_offset * STEP_MINUTES
+ if lb_minute in energy_per_step:
+ lookback_values.append(energy_per_step[lb_minute])
+ pv_lookback_values.append(pv_energy_per_step.get(lb_minute, 0.0))
+ temp_lookback_values.append(temp_values.get(lb_minute, 0.0))
+ else:
+ valid_sample = False
+ break
+
+ if not valid_sample or len(lookback_values) != LOOKBACK_STEPS:
+ continue
+
+ # Target value
+ if target_minute not in energy_per_step:
+ continue
+ target_value = energy_per_step[target_minute]
+
+ # Time features for target time
+ target_time = now_utc - timedelta(minutes=target_minute)
+ minute_of_day = target_time.hour * 60 + target_time.minute
+ day_of_week = target_time.weekday()
+ time_features = self._create_time_features(minute_of_day, day_of_week)
+
+ features = np.concatenate([np.array(lookback_values, dtype=np.float32), np.array(pv_lookback_values, dtype=np.float32), np.array(temp_lookback_values, dtype=np.float32), time_features])
+
+ X_val_list.append(features)
+ y_val_list.append(np.array([target_value], dtype=np.float32))
+
+ if not X_train_list:
+ return None, None, None, None, None
+
+ X_train = np.array(X_train_list, dtype=np.float32)
+ y_train = np.array(y_train_list, dtype=np.float32)
+ train_weights = np.array(weight_list, dtype=np.float32)
+
+ # Normalize weights to sum to number of samples
+ train_weights = train_weights * len(train_weights) / np.sum(train_weights)
+
+ X_val = np.array(X_val_list, dtype=np.float32) if X_val_list else None
+ y_val = np.array(y_val_list, dtype=np.float32) if y_val_list else None
+
+ return X_train, y_train, train_weights, X_val, y_val
+
+ def _normalize_features(self, X, fit=False):
+ """
+ Normalize features using z-score normalization.
+
+ Args:
+ X: Feature array
+ fit: If True, compute and store normalization parameters
+
+ Returns:
+ Normalized feature array
+ """
+ if fit:
+ self.feature_mean = np.mean(X, axis=0)
+ self.feature_std = np.std(X, axis=0)
+ # Prevent division by zero
+ self.feature_std = np.maximum(self.feature_std, 1e-8)
+
+ if self.feature_mean is None or self.feature_std is None:
+ return X
+
+ return (X - self.feature_mean) / self.feature_std
+
+ def _normalize_targets(self, y, fit=False):
+ """
+ Normalize targets using z-score normalization.
+
+ Args:
+ y: Target array
+ fit: If True, compute and store normalization parameters
+
+ Returns:
+ Normalized target array
+ """
+ if fit:
+ self.target_mean = np.mean(y)
+ self.target_std = np.std(y)
+ self.target_std = max(self.target_std, 1e-8)
+
+ if self.target_mean is None or self.target_std is None:
+ return y
+
+ return (y - self.target_mean) / self.target_std
+
+ def _denormalize_predictions(self, y_pred):
+ """
+ Denormalize predictions back to original scale.
+
+ Args:
+ y_pred: Normalized predictions
+
+ Returns:
+ Denormalized predictions in kWh
+ """
+ if self.target_mean is None or self.target_std is None:
+ return y_pred
+
+ return y_pred * self.target_std + self.target_mean
+
+ def _clip_predictions(self, predictions, lookback_buffer=None):
+ """
+ Apply physical constraints to predictions.
+
+ Args:
+ predictions: Raw predictions in kWh per 5 min
+ lookback_buffer: Optional recent values to compute minimum floor
+
+ Returns:
+ Clipped predictions
+ """
+ # Convert max kW to kWh per 5 minutes
+ max_kwh_per_step = self.max_load_kw * STEP_MINUTES / 60.0
+
+ # Compute minimum floor based on recent data (prevent collapse to zero)
+ # Use 10% of the recent minimum as a floor, but at least 0.01 kWh (120W average)
+ if lookback_buffer is not None and len(lookback_buffer) > 0:
+ recent_min = min(lookback_buffer)
+ recent_mean = sum(lookback_buffer) / len(lookback_buffer)
+ # Floor is the smaller of: 20% of recent mean, or recent minimum
+ min_floor = max(0.01, min(recent_min, recent_mean * 0.2))
+ else:
+ min_floor = 0.01 # ~120W baseline
+
+ # Clip to valid range with minimum floor
+ predictions = np.clip(predictions, min_floor, max_kwh_per_step)
+
+ return predictions
+
+ def train(self, load_minutes, now_utc, pv_minutes=None, temp_minutes=None, is_initial=True, epochs=50, time_decay_days=7, patience=5):
+ """
+ Train or fine-tune the model.
+
+ Training uses all available data (most recent to as far back as data goes).
+ Validation uses the most recent 24 hours (subset of training data) to check model fit.
+
+ Args:
+ load_minutes: Dict of {minute: cumulative_kwh}
+ now_utc: Current UTC timestamp
+ pv_minutes: Dict of {minute: cumulative_kwh} PV generation (backwards for history, negative for future)
+ temp_minutes: Dict of {minute: temperature_celsius} Temperature (backwards for history, negative for future)
+ is_initial: If True, full training; else fine-tuning on last 24h
+ epochs: Number of training epochs
+ time_decay_days: Time constant for sample weighting
+ patience: Early stopping patience
+
+ Returns:
+ Validation MAE or None if training failed
+ """
+ self.log("ML Predictor: Starting {} training with {} epochs".format("initial" if is_initial else "fine-tune", epochs))
+
+ # Create dataset with train/validation split
+ result = self._create_dataset(load_minutes, now_utc, pv_minutes=pv_minutes, temp_minutes=temp_minutes, is_finetune=not is_initial, time_decay_days=time_decay_days)
+
+ if result[0] is None:
+ self.log("Warn: ML Predictor: Failed to create dataset")
+ return None
+
+ X_train, y_train, train_weights, X_val, y_val = result
+
+ if len(X_train) < BATCH_SIZE:
+ self.log("Warn: ML Predictor: Insufficient training data ({} samples)".format(len(X_train)))
+ return None
+
+ self.log("ML Predictor: Created {} training samples, {} validation samples".format(len(X_train), len(X_val) if X_val is not None else 0))
+
+ # Check we have validation data
+ if X_val is None or len(X_val) == 0:
+ self.log("Warn: ML Predictor: No validation data available")
+ return None
+
+ # Normalize features and targets
+ X_train_norm = self._normalize_features(X_train, fit=is_initial or not self.model_initialized)
+ X_val_norm = self._normalize_features(X_val, fit=False)
+ y_train_norm = self._normalize_targets(y_train, fit=is_initial or not self.model_initialized)
+ y_val_norm = self._normalize_targets(y_val, fit=False)
+
+ # Initialize weights if needed
+ if not self.model_initialized or (is_initial and self.weights is None):
+ self._initialize_weights()
+
+ # Training loop
+ best_val_loss = float("inf")
+ patience_counter = 0
+
+ for epoch in range(epochs):
+ # Shuffle training data
+ indices = np.random.permutation(len(X_train_norm))
+ X_shuffled = X_train_norm[indices]
+ y_shuffled = y_train_norm[indices]
+ weights_shuffled = train_weights[indices]
+
+ # Mini-batch training
+ epoch_loss = 0
+ num_batches = 0
+
+ for batch_start in range(0, len(X_shuffled), BATCH_SIZE):
+ batch_end = min(batch_start + BATCH_SIZE, len(X_shuffled))
+ X_batch = X_shuffled[batch_start:batch_end]
+ y_batch = y_shuffled[batch_start:batch_end]
+ batch_weights = weights_shuffled[batch_start:batch_end]
+
+ # Forward pass
+ y_pred, activations, pre_activations = self._forward(X_batch)
+
+ # Compute unweighted loss for monitoring
+ batch_loss = mse_loss(y_batch, y_pred)
+ epoch_loss += batch_loss
+ num_batches += 1
+
+ # Backward pass with sample weights applied to gradient
+ weight_grads, bias_grads = self._backward(y_batch, activations, pre_activations, sample_weights=batch_weights)
+
+ # Adam update
+ self._adam_update(weight_grads, bias_grads)
+
+ epoch_loss /= num_batches
+
+ # Validation
+ val_pred, _, _ = self._forward(X_val_norm)
+ val_pred_denorm = self._denormalize_predictions(val_pred)
+ val_mae = np.mean(np.abs(y_val - val_pred_denorm))
+
+ self.log("ML Predictor: Epoch {}/{}: train_loss={:.4f} val_mae={:.4f} kWh".format(epoch + 1, epochs, epoch_loss, val_mae))
+
+ # Early stopping check
+ if val_mae < best_val_loss:
+ best_val_loss = val_mae
+ patience_counter = 0
+ else:
+ patience_counter += 1
+
+ if patience_counter >= patience:
+ self.log("ML Predictor: Early stopping at epoch {}".format(epoch + 1))
+ break
+
+ self.training_timestamp = datetime.now(timezone.utc)
+ self.validation_mae = best_val_loss
+ self.epochs_trained += epochs
+
+ self.log("ML Predictor: Training complete, final val_mae={:.4f} kWh".format(best_val_loss))
+
+ return best_val_loss
+
+ def predict(self, load_minutes, now_utc, midnight_utc, pv_minutes=None, temp_minutes=None, exog_features=None):
+ """
+ Generate predictions for the next 48 hours using autoregressive approach.
+
+ Each iteration predicts the next 5-minute step, then feeds that prediction
+ back into the lookback window for the next iteration. This allows the model
+ to use target-time features for each prediction.
+
+ To prevent autoregressive drift, predictions are blended with historical
+ daily patterns (average energy by time of day).
+
+ Args:
+ load_minutes: Dict of {minute: cumulative_kwh}
+ now_utc: Current UTC timestamp
+ midnight_utc: Today's midnight UTC timestamp
+ pv_minutes: Dict of {minute: cumulative_kwh} PV generation (backwards for history, negative for future)
+ temp_minutes: Dict of {minute: temperature_celsius} Temperature (backwards for history, negative for future)
+ exog_features: Optional dict with future exogenous data
+
+ Returns:
+ Dict of {minute: cumulative_kwh} in incrementing format for future, or empty dict on failure
+ """
+ if not self.model_initialized or self.weights is None:
+ self.log("Warn: ML Predictor: Model not trained, cannot predict")
+ return {}
+
+ # Convert to energy per step for extracting lookback
+ energy_per_step = self._load_to_energy_per_step(load_minutes)
+ pv_energy_per_step = self._load_to_energy_per_step(pv_minutes) if pv_minutes else {}
+ # Temperature is not cumulative, so just use the raw values
+ temp_values = temp_minutes if temp_minutes else {}
+
+ if not energy_per_step:
+ self.log("Warn: ML Predictor: No load data available for prediction")
+ return {}
+
+ # Compute historical daily patterns for blending (prevents autoregressive drift)
+ # Group historical energy by minute-of-day and compute average
+ historical_pattern = self._compute_daily_pattern(energy_per_step)
+
+ # Build initial lookback window from historical data (most recent 24 hours)
+ # This will be updated as we make predictions (autoregressive)
+ lookback_buffer = []
+ pv_lookback_buffer = []
+ temp_lookback_buffer = []
+ for lb_offset in range(LOOKBACK_STEPS):
+ lb_minute = lb_offset * STEP_MINUTES
+ if lb_minute in energy_per_step:
+ lookback_buffer.append(energy_per_step[lb_minute])
+ else:
+ lookback_buffer.append(0) # Fallback to zero
+ # Add PV generation (0 if no data)
+ pv_lookback_buffer.append(pv_energy_per_step.get(lb_minute, 0.0))
+ # Add temperature (0 if no data)
+ temp_lookback_buffer.append(temp_values.get(lb_minute, 0.0))
+
+ # Autoregressive prediction loop: predict one step at a time
+ predictions_energy = []
+
+ # Blending parameters: model weight decreases as we go further into future
+ # At step 0: 100% model, at step PREDICT_HORIZON: blend_floor% model
+ blend_floor = 0.5 # Minimum model weight at horizon (keep more model influence)
+
+ for step_idx in range(PREDICT_HORIZON):
+ # Calculate target time for this prediction step
+ target_time = now_utc + timedelta(minutes=(step_idx + 1) * STEP_MINUTES)
+ minute_of_day = target_time.hour * 60 + target_time.minute
+ day_of_week = target_time.weekday()
+ time_features = self._create_time_features(minute_of_day, day_of_week)
+
+ # Get PV value for the next step from forecast (negative minutes are future)
+ # For future predictions, use forecast; for past, it's already in pv_energy_per_step
+ future_minute = -(step_idx + 1) * STEP_MINUTES # Negative = future
+ next_pv_value = pv_energy_per_step.get(future_minute, 0.0)
+ # Get temperature value for the next step from forecast (negative minutes are future)
+ next_temp_value = temp_values.get(future_minute, 0.0)
+
+ # Combine features: [load_lookback..., pv_lookback..., temp_lookback..., time_features...]
+ features = np.concatenate([np.array(lookback_buffer, dtype=np.float32), np.array(pv_lookback_buffer, dtype=np.float32), np.array(temp_lookback_buffer, dtype=np.float32), time_features])
+ features = self._add_exog_features(features, exog_features)
+
+ # Normalize and forward pass
+ features_norm = self._normalize_features(features.reshape(1, -1), fit=False)
+ pred_norm, _, _ = self._forward(features_norm)
+ pred_energy = self._denormalize_predictions(pred_norm[0])
+
+ # Apply physical constraints
+ pred_energy = self._clip_predictions(pred_energy)
+ model_pred = float(pred_energy[0]) # Single output
+
+ # Get historical pattern value for this time of day
+ slot = (minute_of_day // STEP_MINUTES) * STEP_MINUTES
+ hist_value = historical_pattern.get(slot, model_pred)
+
+ # Blend model prediction with historical pattern
+ # Linear decay: model weight goes from 1.0 to blend_floor over horizon
+ progress = step_idx / PREDICT_HORIZON
+ model_weight = 1.0 - progress * (1.0 - blend_floor)
+ energy_value = model_weight * model_pred + (1.0 - model_weight) * hist_value
+
+ # Re-apply constraints after blending
+ max_kwh_per_step = self.max_load_kw * STEP_MINUTES / 60.0
+ energy_value = max(0.01, min(energy_value, max_kwh_per_step))
+
+ predictions_energy.append(energy_value)
+
+ # Update lookback buffer for next iteration (shift and add new prediction)
+ # Lookback[0] is most recent, so insert at front and remove from end
+ lookback_buffer.insert(0, energy_value)
+ lookback_buffer.pop() # Remove oldest value
+
+ # Update PV lookback buffer with next forecast value
+ pv_lookback_buffer.insert(0, next_pv_value)
+ pv_lookback_buffer.pop() # Remove oldest value
+
+ # Update temperature lookback buffer with next forecast value
+ temp_lookback_buffer.insert(0, next_temp_value)
+ temp_lookback_buffer.pop() # Remove oldest value
+
+ # Convert to cumulative kWh format (incrementing into future)
+ # Format matches fetch_extra_load_forecast output
+ result = {}
+ cumulative = 0
+
+ for step_idx in range(PREDICT_HORIZON):
+ minute = step_idx * STEP_MINUTES
+ energy = predictions_energy[step_idx]
+ cumulative += energy
+ result[minute] = round(cumulative, 4)
+
+ return result
+
+ def save(self, filepath):
+ """
+ Save model to file.
+
+ Args:
+ filepath: Path to save model (without extension)
+ """
+ if not self.model_initialized:
+ self.log("Warn: ML Predictor: No model to save")
+ return False
+
+ try:
+ # Prepare metadata
+ metadata = {
+ "model_version": MODEL_VERSION,
+ "lookback_steps": LOOKBACK_STEPS,
+ "output_steps": OUTPUT_STEPS,
+ "predict_horizon": PREDICT_HORIZON,
+ "hidden_sizes": HIDDEN_SIZES,
+ "training_timestamp": self.training_timestamp.isoformat() if self.training_timestamp else None,
+ "validation_mae": float(self.validation_mae) if self.validation_mae else None,
+ "epochs_trained": self.epochs_trained,
+ "learning_rate": self.learning_rate,
+ "max_load_kw": self.max_load_kw,
+ "feature_mean": self.feature_mean.tolist() if self.feature_mean is not None else None,
+ "feature_std": self.feature_std.tolist() if self.feature_std is not None else None,
+ "target_mean": float(self.target_mean) if self.target_mean is not None else None,
+ "target_std": float(self.target_std) if self.target_std is not None else None,
+ "pv_mean": float(self.pv_mean) if self.pv_mean is not None else None,
+ "pv_std": float(self.pv_std) if self.pv_std is not None else None,
+ }
+
+ # Save weights and metadata
+ save_dict = {
+ "metadata_json": json.dumps(metadata),
+ }
+
+ for i, (w, b) in enumerate(zip(self.weights, self.biases)):
+ save_dict[f"weight_{i}"] = w
+ save_dict[f"bias_{i}"] = b
+
+ # Save Adam optimizer state
+ for i in range(len(self.weights)):
+ save_dict[f"m_weight_{i}"] = self.m_weights[i]
+ save_dict[f"v_weight_{i}"] = self.v_weights[i]
+ save_dict[f"m_bias_{i}"] = self.m_biases[i]
+ save_dict[f"v_bias_{i}"] = self.v_biases[i]
+
+ save_dict["adam_t"] = np.array([self.adam_t])
+
+ np.savez(filepath, **save_dict)
+ self.log("ML Predictor: Model saved to {}".format(filepath))
+ return True
+
+ except Exception as e:
+ self.log("Error: ML Predictor: Failed to save model: {}".format(e))
+ return False
+
+ def load(self, filepath):
+ """
+ Load model from file.
+
+ Args:
+ filepath: Path to model file
+
+ Returns:
+ True if successful, False otherwise
+ """
+ try:
+ if not os.path.exists(filepath):
+ self.log("ML Predictor: No saved model found at {}".format(filepath))
+ return False
+
+ data = np.load(filepath, allow_pickle=True)
+
+ # Load metadata
+ metadata = json.loads(str(data["metadata_json"]))
+
+ # Check version compatibility
+ saved_version = metadata.get("model_version", 0)
+ if saved_version != MODEL_VERSION:
+ self.log("Warn: ML Predictor: Model version mismatch (saved={}, current={}), retraining from scratch".format(saved_version, MODEL_VERSION))
+ return False
+
+ # Check architecture compatibility
+ if metadata.get("lookback_steps") != LOOKBACK_STEPS or metadata.get("output_steps") != OUTPUT_STEPS or metadata.get("hidden_sizes") != HIDDEN_SIZES:
+ self.log("Warn: ML Predictor: Architecture mismatch, retraining from scratch")
+ return False
+
+ # Load weights
+ self.weights = []
+ self.biases = []
+ self.m_weights = []
+ self.v_weights = []
+ self.m_biases = []
+ self.v_biases = []
+
+ layer_count = len(HIDDEN_SIZES) + 1
+ for i in range(layer_count):
+ self.weights.append(data[f"weight_{i}"])
+ self.biases.append(data[f"bias_{i}"])
+ self.m_weights.append(data[f"m_weight_{i}"])
+ self.v_weights.append(data[f"v_weight_{i}"])
+ self.m_biases.append(data[f"m_bias_{i}"])
+ self.v_biases.append(data[f"v_bias_{i}"])
+
+ self.adam_t = int(data["adam_t"][0])
+
+ # Load normalization parameters
+ if metadata.get("feature_mean"):
+ self.feature_mean = np.array(metadata["feature_mean"], dtype=np.float32)
+ if metadata.get("feature_std"):
+ self.feature_std = np.array(metadata["feature_std"], dtype=np.float32)
+ if metadata.get("target_mean") is not None:
+ self.target_mean = metadata["target_mean"]
+ if metadata.get("target_std") is not None:
+ self.target_std = metadata["target_std"]
+ if metadata.get("pv_mean") is not None:
+ self.pv_mean = metadata["pv_mean"]
+ if metadata.get("pv_std") is not None:
+ self.pv_std = metadata["pv_std"]
+
+ # Load training metadata
+ if metadata.get("training_timestamp"):
+ self.training_timestamp = datetime.fromisoformat(metadata["training_timestamp"])
+ self.validation_mae = metadata.get("validation_mae")
+ self.epochs_trained = metadata.get("epochs_trained", 0)
+
+ self.model_initialized = True
+
+ self.log("ML Predictor: Model loaded from {} (trained {}, val_mae={:.4f})".format(filepath, self.training_timestamp.strftime("%Y-%m-%d %H:%M") if self.training_timestamp else "unknown", self.validation_mae if self.validation_mae else 0))
+ return True
+
+ except Exception as e:
+ self.log("Error: ML Predictor: Failed to load model: {}".format(e))
+ return False
+
+ def get_model_age_hours(self):
+ """Get the age of the model in hours since last training."""
+ if self.training_timestamp is None:
+ return None
+
+ age = datetime.now(timezone.utc) - self.training_timestamp
+ return age.total_seconds() / 3600
+
+ def is_valid(self, validation_threshold=2.0, max_age_hours=48):
+ """
+ Check if model is valid for predictions.
+
+ Args:
+ validation_threshold: Maximum acceptable validation MAE in kWh
+ max_age_hours: Maximum model age in hours
+
+ Returns:
+ Tuple of (is_valid, reason_if_invalid)
+ """
+ if not self.model_initialized:
+ return False, "not_initialized"
+
+ if self.weights is None:
+ return False, "no_weights"
+
+ if self.validation_mae is not None and self.validation_mae > validation_threshold:
+ return False, "validation_threshold"
+
+ age_hours = self.get_model_age_hours()
+ if age_hours is not None and age_hours > max_age_hours:
+ return False, "stale"
+
+ return True, None
diff --git a/apps/predbat/predbat.py b/apps/predbat/predbat.py
index 68c906d10..55816897d 100644
--- a/apps/predbat/predbat.py
+++ b/apps/predbat/predbat.py
@@ -27,10 +27,10 @@
import requests
import asyncio
-THIS_VERSION = "v8.32.14"
+THIS_VERSION = "v8.33.0"
# fmt: off
-PREDBAT_FILES = ["predbat.py", "const.py", "hass.py", "config.py", "prediction.py", "gecloud.py", "utils.py", "inverter.py", "ha.py", "download.py", "web.py", "web_helper.py", "predheat.py", "futurerate.py", "octopus.py", "solcast.py", "execute.py", "plan.py", "fetch.py", "output.py", "userinterface.py", "energydataservice.py", "alertfeed.py", "compare.py", "db_manager.py", "db_engine.py", "plugin_system.py", "ohme.py", "components.py", "fox.py", "carbon.py", "web_mcp.py", "component_base.py", "axle.py", "solax.py", "solis.py", "unit_test.py"]
+PREDBAT_FILES = ["predbat.py", "const.py", "hass.py", "config.py", "prediction.py", "gecloud.py", "utils.py", "inverter.py", "ha.py", "download.py", "web.py", "web_helper.py", "predheat.py", "futurerate.py", "octopus.py", "solcast.py", "execute.py", "plan.py", "fetch.py", "output.py", "userinterface.py", "energydataservice.py", "alertfeed.py", "compare.py", "db_manager.py", "db_engine.py", "plugin_system.py", "ohme.py", "components.py", "fox.py", "carbon.py", "temperature.py", "web_mcp.py", "component_base.py", "axle.py", "solax.py", "solis.py", "unit_test.py"]
# fmt: on
from download import predbat_update_move, predbat_update_download, check_install
diff --git a/apps/predbat/temperature.py b/apps/predbat/temperature.py
new file mode 100644
index 000000000..6e53c2364
--- /dev/null
+++ b/apps/predbat/temperature.py
@@ -0,0 +1,210 @@
+# -----------------------------------------------------------------------------
+# Predbat Home Battery System
+# Copyright Trefor Southwell 2025 - All Rights Reserved
+# This application maybe used for personal use only and not for commercial use
+# -----------------------------------------------------------------------------
+# fmt: off
+# pylint: disable=consider-using-f-string
+# pylint: disable=line-too-long
+# pylint: disable=attribute-defined-outside-init
+
+import aiohttp
+import asyncio
+from datetime import datetime
+from utils import dp1
+from component_base import ComponentBase
+
+
+class TemperatureAPI(ComponentBase):
+ def initialize(self, temperature_enable, temperature_latitude, temperature_longitude, temperature_url):
+ """Initialize the Temperature API component"""
+ self.temperature_enable = temperature_enable
+ self.temperature_latitude = temperature_latitude
+ self.temperature_longitude = temperature_longitude
+ self.temperature_url = temperature_url
+ self.temperature_cache = {}
+ self.temperature_data = None
+ self.last_updated_timestamp = None
+ self.failures_total = 0
+
+ async def select_event(self, entity_id, value):
+ pass
+
+ async def number_event(self, entity_id, value):
+ pass
+
+ async def switch_event(self, entity_id, service):
+ pass
+
+ async def run(self, seconds, first):
+ """
+ Main run loop - polls API every hour
+ """
+ try:
+ if not self.temperature_enable:
+ return True
+ if first or (seconds % (60 * 60) == 0):
+ # Fetch temperature data every hour
+ temperature_data = await self.fetch_temperature_data()
+ if temperature_data is not None:
+ self.temperature_data = temperature_data
+ self.last_updated_timestamp = datetime.now()
+ self.publish_temperature_sensor()
+ if self.temperature_data is not None:
+ self.update_success_timestamp()
+ self.publish_temperature_sensor()
+ except Exception as e:
+ self.log("Warn: TemperatureAPI: Exception in run loop: {}".format(e))
+ # Still return True to keep component alive
+ if self.temperature_data is not None:
+ # Keep publishing old data even on error
+ self.publish_temperature_sensor()
+
+ return True
+
+ def get_coordinates(self):
+ """
+ Get latitude and longitude, with fallback to zone.home
+ """
+ # Try config values first
+ latitude = self.temperature_latitude
+ longitude = self.temperature_longitude
+
+ # If latitude and longitude are not provided, use zone.home
+ if latitude is None:
+ latitude = self.get_state_wrapper("zone.home", attribute="latitude")
+ if longitude is None:
+ longitude = self.get_state_wrapper("zone.home", attribute="longitude")
+
+ if latitude is not None and longitude is not None:
+ self.log("TemperatureAPI: Using coordinates latitude {}, longitude {}".format(dp1(latitude), dp1(longitude)))
+ return latitude, longitude
+ else:
+ self.log("Warn: TemperatureAPI: No latitude or longitude found, cannot fetch temperature data")
+ return None, None
+
+ def build_api_url(self, latitude, longitude):
+ """
+ Build the API URL with latitude and longitude placeholders replaced
+ """
+ url = self.temperature_url.replace("LATITUDE", str(latitude)).replace("LONGITUDE", str(longitude))
+ return url
+
+ def convert_timezone_offset(self, utc_offset_seconds):
+ """
+ Convert UTC offset in seconds to ±HH:MM format
+ Handles negative offsets correctly
+ """
+ if utc_offset_seconds >= 0:
+ sign = "+"
+ else:
+ sign = "-"
+ utc_offset_seconds = abs(utc_offset_seconds)
+
+ offset_hours = utc_offset_seconds // 3600
+ offset_minutes = (utc_offset_seconds % 3600) // 60
+
+ return "{}{:02d}:{:02d}".format(sign, offset_hours, offset_minutes)
+
+ async def fetch_temperature_data(self):
+ """
+ Fetch temperature data from Open-Meteo API with retry logic
+ """
+ latitude, longitude = self.get_coordinates()
+ if latitude is None or longitude is None:
+ return None
+
+ url = self.build_api_url(latitude, longitude)
+
+ # Try up to 3 times with exponential backoff
+ max_retries = 3
+ for attempt in range(max_retries):
+ try:
+ timeout = aiohttp.ClientTimeout(total=30)
+ async with aiohttp.ClientSession(timeout=timeout) as session:
+ async with session.get(url) as response:
+ if response.status == 200:
+ data = await response.json()
+ self.log("TemperatureAPI: Successfully fetched temperature data from Open-Meteo API")
+ self.update_success_timestamp()
+ return data
+ else:
+ self.log("Warn: TemperatureAPI: Failed to fetch data, status code {}".format(response.status))
+ if attempt < max_retries - 1:
+ sleep_time = 2 ** attempt
+ self.log("Warn: TemperatureAPI: Retrying in {} seconds...".format(sleep_time))
+ await asyncio.sleep(sleep_time)
+ else:
+ self.failures_total += 1
+ return None
+ except (aiohttp.ClientError, asyncio.TimeoutError) as e:
+ if attempt < max_retries - 1:
+ sleep_time = 2 ** attempt
+ self.log("Warn: TemperatureAPI: Request attempt {} failed: {}. Retrying in {}s...".format(attempt + 1, e, sleep_time))
+ await asyncio.sleep(sleep_time)
+ else:
+ self.log("Warn: TemperatureAPI: Request failed after {} attempts: {}".format(max_retries, e))
+ self.failures_total += 1
+ return None
+ except Exception as e:
+ self.log("Warn: TemperatureAPI: Unexpected error fetching temperature data: {}".format(e))
+ self.failures_total += 1
+ return None
+
+ return None
+
+ def publish_temperature_sensor(self):
+ """
+ Publish temperature sensor to Home Assistant
+ """
+ if self.temperature_data is None:
+ return
+
+ try:
+ # Extract current temperature
+ current = self.temperature_data.get("current", {})
+ current_temp = current.get("temperature_2m")
+
+ if current_temp is None:
+ self.log("Warn: TemperatureAPI: No current temperature in API response")
+ return
+
+ # Get timezone offset
+ utc_offset_seconds = self.temperature_data.get("utc_offset_seconds", 0)
+ timezone_offset = self.convert_timezone_offset(utc_offset_seconds)
+
+ # Build hourly forecast dictionary
+ hourly = self.temperature_data.get("hourly", {})
+ hourly_times = hourly.get("time", [])
+ hourly_temps = hourly.get("temperature_2m", [])
+
+ forecast = {}
+ if len(hourly_times) == len(hourly_temps):
+ for time_str, temp in zip(hourly_times, hourly_temps):
+ # Convert ISO8601 time to HA format with timezone
+ # Open-Meteo returns: "2026-02-07T00:00"
+ # HA format: "2026-02-07T00:00:00+00:00"
+ ha_timestamp = "{}:00{}".format(time_str, timezone_offset)
+ forecast[ha_timestamp] = temp
+
+ # Build last_updated string
+ last_updated_str = str(self.last_updated_timestamp) if self.last_updated_timestamp else "Never"
+
+ # Publish sensor
+ self.dashboard_item(
+ "sensor." + self.prefix + "_temperature",
+ state=current_temp,
+ attributes={
+ "friendly_name": "External Temperature Forecast",
+ "icon": "mdi:thermometer",
+ "unit_of_measurement": "°C",
+ "last_updated": last_updated_str,
+ "results": forecast,
+ "timezone_offset": timezone_offset,
+ "data_points": len(forecast)
+ },
+ app="temperature"
+ )
+
+ except Exception as e:
+ self.log("Warn: TemperatureAPI: Error publishing sensor: {}".format(e))
diff --git a/apps/predbat/tests/test_load_ml.py b/apps/predbat/tests/test_load_ml.py
new file mode 100644
index 000000000..1bd633ea3
--- /dev/null
+++ b/apps/predbat/tests/test_load_ml.py
@@ -0,0 +1,1646 @@
+# -----------------------------------------------------------------------------
+# Predbat Home Battery System
+# Copyright Trefor Southwell 2025 - All Rights Reserved
+# This application maybe used for personal use only and not for commercial use
+# -----------------------------------------------------------------------------
+# fmt: off
+# pylint: disable=consider-using-f-string
+# pylint: disable=line-too-long
+# pylint: disable=attribute-defined-outside-init
+# fmt: on
+
+import numpy as np
+from datetime import datetime, timezone, timedelta
+import tempfile
+import os
+
+from load_predictor import LoadPredictor, OUTPUT_STEPS, HIDDEN_SIZES, TOTAL_FEATURES, STEP_MINUTES, relu, relu_derivative, huber_loss
+
+
+def test_load_ml(my_predbat=None):
+ """
+ Comprehensive test suite for ML Load Forecaster.
+
+ Tests all major functionality including:
+ - MLP forward/backward pass correctness
+ - Dataset creation with cyclical features
+ - Training convergence on synthetic data
+ - Model save/load with version check
+ - Cold-start and fine-tune scenarios
+ - Validation failure fallback
+ """
+
+ # Registry of all sub-tests
+ sub_tests = [
+ ("relu_functions", _test_relu_functions, "ReLU activation and derivative"),
+ ("huber_loss_functions", _test_huber_loss_functions, "Huber loss computation"),
+ ("forward_pass", _test_forward_pass, "Forward pass computation"),
+ ("backward_pass", _test_backward_pass, "Backward pass gradient computation"),
+ ("cyclical_features", _test_cyclical_features, "Cyclical time feature encoding"),
+ ("load_to_energy", _test_load_to_energy, "Convert cumulative load to energy per step"),
+ ("pv_energy_conversion", _test_pv_energy_conversion, "Convert PV data including future forecasts"),
+ ("dataset_creation", _test_dataset_creation, "Dataset creation from load data"),
+ ("dataset_with_pv", _test_dataset_with_pv, "Dataset creation with PV features"),
+ ("dataset_with_temp", _test_dataset_with_temp, "Dataset creation with temperature features"),
+ ("normalization", _test_normalization, "Z-score normalization correctness"),
+ ("adam_optimizer", _test_adam_optimizer, "Adam optimizer step"),
+ ("training_convergence", _test_training_convergence, "Training convergence on synthetic data"),
+ ("training_with_pv", _test_training_with_pv, "Training with PV input features"),
+ ("training_with_temp", _test_training_with_temp, "Training with temperature input features"),
+ ("model_persistence", _test_model_persistence, "Model save/load with version check"),
+ ("cold_start", _test_cold_start, "Cold start with insufficient data"),
+ ("fine_tune", _test_fine_tune, "Fine-tune on recent data"),
+ ("prediction", _test_prediction, "End-to-end prediction"),
+ ("prediction_with_pv", _test_prediction_with_pv, "Prediction with PV forecast data"),
+ ("prediction_with_temp", _test_prediction_with_temp, "Prediction with temperature forecast data"),
+ # ("real_data_training", _test_real_data_training, "Train on real load_minutes_debug.json data with chart"),
+ ("component_fetch_load_data", _test_component_fetch_load_data, "LoadMLComponent _fetch_load_data method"),
+ ("component_publish_entity", _test_component_publish_entity, "LoadMLComponent _publish_entity method"),
+ ]
+
+ failed_tests = []
+ passed_count = 0
+
+ for name, test_func, description in sub_tests:
+ try:
+ print(f" Running {name}: {description}...", end=" ")
+ test_func()
+ print("PASS")
+ passed_count += 1
+ except Exception as e:
+ print(f"FAIL: {e}")
+ import traceback
+
+ traceback.print_exc()
+ failed_tests.append((name, str(e)))
+
+ print(f"\nML Load Forecaster Tests: {passed_count}/{len(sub_tests)} passed")
+ if failed_tests:
+ print("Failed tests:")
+ for name, error in failed_tests:
+ print(f" - {name}: {error}")
+ assert False, f"ML Load Forecaster: {len(failed_tests)} tests failed"
+
+
+def _test_relu_functions():
+ """Test ReLU activation and derivative"""
+ # Test ReLU
+ x = np.array([-2, -1, 0, 1, 2])
+ expected = np.array([0, 0, 0, 1, 2])
+ result = relu(x)
+ assert np.allclose(result, expected), f"ReLU output mismatch: {result} vs {expected}"
+
+ # Test ReLU derivative
+ expected_deriv = np.array([0, 0, 0, 1, 1])
+ result_deriv = relu_derivative(x)
+ assert np.allclose(result_deriv, expected_deriv), f"ReLU derivative mismatch: {result_deriv} vs {expected_deriv}"
+
+
+def _test_huber_loss_functions():
+ """Test Huber loss computation"""
+ # Test with small error (L2 region)
+ y_true = np.array([[1.0, 2.0, 3.0]])
+ y_pred = np.array([[1.1, 2.1, 3.1]]) # Error = 0.1
+ loss = huber_loss(y_true, y_pred, delta=1.0)
+ # For small errors, Huber is 0.5 * error^2
+ expected = 0.5 * (0.1**2)
+ assert abs(loss - expected) < 0.01, f"Huber loss for small error: expected {expected}, got {loss}"
+
+ # Test with large error (L1 region)
+ y_pred_large = np.array([[3.0, 4.0, 5.0]]) # Error = 2.0
+ loss_large = huber_loss(y_true, y_pred_large, delta=1.0)
+ # For large errors, Huber is delta * (|error| - 0.5 * delta)
+ expected_large = 1.0 * (2.0 - 0.5)
+ assert abs(loss_large - expected_large) < 0.1, f"Huber loss for large error: expected {expected_large}, got {loss_large}"
+
+
+def _test_forward_pass():
+ """Test that forward pass produces expected output shape and values"""
+ predictor = LoadPredictor(learning_rate=0.001)
+
+ # Initialize weights
+ predictor._initialize_weights()
+
+ # Create test input: batch of 2, with TOTAL_FEATURES features
+ X = np.random.randn(2, TOTAL_FEATURES).astype(np.float32)
+
+ # Forward pass
+ output, activations, pre_activations = predictor._forward(X)
+
+ # Check output shape: should be (batch_size, OUTPUT_STEPS)
+ assert output.shape == (2, OUTPUT_STEPS), f"Expected output shape (2, {OUTPUT_STEPS}), got {output.shape}"
+
+ # Check that output is finite
+ assert np.all(np.isfinite(output)), "Forward pass produced non-finite values"
+
+ # Check activations structure
+ assert len(activations) == len(HIDDEN_SIZES) + 2, "Wrong number of activations"
+ assert len(pre_activations) == len(HIDDEN_SIZES) + 1, "Wrong number of pre-activations"
+
+
+def _test_backward_pass():
+ """Test that backward pass produces gradients with correct shapes"""
+ predictor = LoadPredictor(learning_rate=0.001)
+ predictor._initialize_weights()
+
+ # Forward pass
+ np.random.seed(42)
+ X = np.random.randn(4, TOTAL_FEATURES).astype(np.float32)
+ y_true = np.random.randn(4, OUTPUT_STEPS).astype(np.float32)
+
+ output, activations, pre_activations = predictor._forward(X)
+
+ # Backward pass
+ weight_grads, bias_grads = predictor._backward(y_true, activations, pre_activations)
+
+ # Check that gradients exist for all weight layers
+ assert len(weight_grads) == len(HIDDEN_SIZES) + 1, "Wrong number of weight gradients"
+ assert len(bias_grads) == len(HIDDEN_SIZES) + 1, "Wrong number of bias gradients"
+
+ # Check gradient shapes match weight shapes
+ for i, (w_grad, w) in enumerate(zip(weight_grads, predictor.weights)):
+ assert w_grad.shape == w.shape, f"Weight gradient {i} shape mismatch: {w_grad.shape} vs {w.shape}"
+
+ for i, (b_grad, b) in enumerate(zip(bias_grads, predictor.biases)):
+ assert b_grad.shape == b.shape, f"Bias gradient {i} shape mismatch: {b_grad.shape} vs {b.shape}"
+
+
+def _test_cyclical_features():
+ """Test cyclical time feature encoding"""
+ predictor = LoadPredictor()
+
+ # Test midnight (minute 0)
+ features = predictor._create_time_features(0, 0)
+ assert len(features) == 4, "Should have 4 time features"
+ assert abs(features[0] - 0.0) < 1e-6, "Midnight sin should be 0"
+ assert abs(features[1] - 1.0) < 1e-6, "Midnight cos should be 1"
+
+ # Test noon (minute 720)
+ features = predictor._create_time_features(720, 0)
+ assert abs(features[0] - 0.0) < 1e-6, "Noon sin should be 0"
+ assert abs(features[1] - (-1.0)) < 1e-6, "Noon cos should be -1"
+
+ # Test 6 AM (minute 360) - sin should be 1, cos should be 0
+ features = predictor._create_time_features(360, 0)
+ assert abs(features[0] - 1.0) < 1e-6, "6 AM sin should be 1"
+ assert abs(features[1] - 0.0) < 1e-6, "6 AM cos should be 0"
+
+ # Test Monday (dow 0) vs Thursday (dow 3)
+ features_mon = predictor._create_time_features(0, 0)
+ features_thu = predictor._create_time_features(0, 3)
+ assert features_mon[2] != features_thu[2], "Different days should have different encodings"
+
+
+def _test_load_to_energy():
+ """Test conversion of cumulative load to energy per step"""
+ predictor = LoadPredictor()
+
+ # Create synthetic cumulative load data
+ # Cumulative: minute 0 = 10, minute 5 = 9, minute 10 = 8, etc.
+ load_minutes = {0: 10.0, 5: 9.0, 10: 8.0, 15: 7.5, 20: 7.0}
+
+ energy_per_step = predictor._load_to_energy_per_step(load_minutes)
+
+ # Energy from 0-5: 10 - 9 = 1
+ assert abs(energy_per_step.get(0, -1) - 1.0) < 1e-6, "Energy 0-5 should be 1.0"
+ # Energy from 5-10: 9 - 8 = 1
+ assert abs(energy_per_step.get(5, -1) - 1.0) < 1e-6, "Energy 5-10 should be 1.0"
+ # Energy from 10-15: 8 - 7.5 = 0.5
+ assert abs(energy_per_step.get(10, -1) - 0.5) < 1e-6, "Energy 10-15 should be 0.5"
+ # Energy from 15-20: 7.5 - 7 = 0.5
+ assert abs(energy_per_step.get(15, -1) - 0.5) < 1e-6, "Energy 15-20 should be 0.5"
+
+
+def _test_pv_energy_conversion():
+ """Test conversion of PV data including future forecasts (negative minutes)"""
+ predictor = LoadPredictor()
+
+ # Create PV data with both historical (positive) and future (negative) minutes
+ # Historical: minute 0-20 (backwards in time)
+ # Future: minute -5 to -20 (forward in time)
+ pv_minutes = {
+ # Historical (cumulative decreasing as we go back in time)
+ 0: 10.0,
+ 5: 9.0,
+ 10: 8.0,
+ 15: 7.0,
+ 20: 6.0,
+ # Future forecasts (cumulative increasing as we go forward)
+ -5: 11.0,
+ -10: 12.5,
+ -15: 14.0,
+ -20: 15.0,
+ }
+
+ pv_energy_per_step = predictor._load_to_energy_per_step(pv_minutes)
+
+ # Historical energy (positive minutes, going backwards)
+ # Energy from 0-5: 10 - 9 = 1
+ assert abs(pv_energy_per_step.get(0, -1) - 1.0) < 1e-6, "PV energy 0-5 should be 1.0"
+ # Energy from 5-10: 9 - 8 = 1
+ assert abs(pv_energy_per_step.get(5, -1) - 1.0) < 1e-6, "PV energy 5-10 should be 1.0"
+
+ # Future energy (negative minutes, going forward)
+ # Energy from -20 to -15: 15.0 - 14.0 = 1.0
+ assert abs(pv_energy_per_step.get(-20, -1) - 1.0) < 1e-6, f"PV future energy -20 to -15 should be 1.0, got {pv_energy_per_step.get(-20, -1)}"
+ # Energy from -15 to -10: 14.0 - 12.5 = 1.5
+ assert abs(pv_energy_per_step.get(-15, -1) - 1.5) < 1e-6, f"PV future energy -15 to -10 should be 1.5, got {pv_energy_per_step.get(-15, -1)}"
+ # Energy from -10 to -5: 12.5 - 11.0 = 1.5
+ assert abs(pv_energy_per_step.get(-10, -1) - 1.5) < 1e-6, f"PV future energy -10 to -5 should be 1.5, got {pv_energy_per_step.get(-10, -1)}"
+ # Energy from -5 to 0: 11.0 - 10.0 = 1.0
+ assert abs(pv_energy_per_step.get(-5, -1) - 1.0) < 1e-6, f"PV future energy -5 to 0 should be 1.0, got {pv_energy_per_step.get(-5, -1)}"
+
+
+def _create_synthetic_pv_data(n_days=7, now_utc=None, forecast_hours=48):
+ """Create synthetic PV data for testing (historical + forecast)"""
+ if now_utc is None:
+ now_utc = datetime.now(timezone.utc)
+
+ pv_minutes = {}
+ cumulative = 0.0
+
+ # Historical PV (positive minutes, backwards from now)
+ n_minutes = n_days * 24 * 60
+ # Start from a multiple of STEP_MINUTES and go down to 0
+ start_minute = (n_minutes // STEP_MINUTES) * STEP_MINUTES
+ for minute in range(start_minute, -STEP_MINUTES, -STEP_MINUTES):
+ dt = now_utc - timedelta(minutes=minute)
+ hour = dt.hour
+
+ # PV generation pattern: 0 at night, peak at midday
+ if 6 <= hour < 18:
+ # Peak around noon (hour 12)
+ hour_offset = abs(hour - 12)
+ energy = max(0, 0.5 - hour_offset * 0.08 + 0.05 * np.random.randn())
+ else:
+ energy = 0.0
+
+ energy = max(0, energy)
+ cumulative += energy
+ pv_minutes[minute] = cumulative
+
+ # Future PV forecast (negative minutes, forward from now)
+ forecast_cumulative = pv_minutes[0] # Start from current cumulative
+ for step in range(1, (forecast_hours * 60 // STEP_MINUTES) + 1):
+ minute = -step * STEP_MINUTES
+ dt = now_utc + timedelta(minutes=step * STEP_MINUTES)
+ hour = dt.hour
+
+ # Same pattern for forecast
+ if 6 <= hour < 18:
+ hour_offset = abs(hour - 12)
+ energy = max(0, 0.5 - hour_offset * 0.08 + 0.05 * np.random.randn())
+ else:
+ energy = 0.0
+
+ energy = max(0, energy)
+ forecast_cumulative += energy
+ pv_minutes[minute] = forecast_cumulative
+
+ return pv_minutes
+
+
+def _create_synthetic_temp_data(n_days=7, now_utc=None, forecast_hours=48):
+ """Create synthetic temperature data for testing (historical + forecast)"""
+ if now_utc is None:
+ now_utc = datetime.now(timezone.utc)
+
+ temp_minutes = {}
+
+ # Historical temperature (positive minutes, backwards from now)
+ n_minutes = n_days * 24 * 60
+ start_minute = (n_minutes // STEP_MINUTES) * STEP_MINUTES
+ for minute in range(start_minute, -STEP_MINUTES, -STEP_MINUTES):
+ dt = now_utc - timedelta(minutes=minute)
+ hour = dt.hour + dt.minute / 60.0 # Fractional hour for smooth variation
+
+ # Smooth sinusoidal daily temperature pattern
+ # Temperature peaks around 1pm (hour 13) and minimum around 1am (hour 1)
+ # Using cosine wave shifted so maximum is at hour 13
+ hours_since_peak = (hour - 13.0) % 24.0
+ daily_cycle = np.cos(2 * np.pi * hours_since_peak / 24.0)
+
+ # Base temp 6°C, amplitude 4°C, so range is 2°C to 10°C
+ # Add small multi-day variation (0.5°C amplitude over 3-day cycle)
+ day_num = minute / (24 * 60)
+ multi_day_variation = 0.5 * np.sin(2 * np.pi * day_num / 3.0)
+
+ temp = 6.0 + 4.0 * daily_cycle + multi_day_variation
+
+ temp = max(-10.0, min(40.0, temp)) # Reasonable bounds
+ temp_minutes[minute] = temp
+
+ # Future temperature forecast (negative minutes, forward from now)
+ for step in range(1, (forecast_hours * 60 // STEP_MINUTES) + 1):
+ minute = -step * STEP_MINUTES
+ dt = now_utc + timedelta(minutes=step * STEP_MINUTES)
+ hour = dt.hour + dt.minute / 60.0 # Fractional hour for smooth variation
+
+ # Same smooth pattern for forecast
+ hours_since_peak = (hour - 13.0) % 24.0
+ daily_cycle = np.cos(2 * np.pi * hours_since_peak / 24.0)
+
+ # Continue the multi-day variation into the future
+ day_num = -minute / (24 * 60) # Negative minute means future
+ multi_day_variation = 0.5 * np.sin(2 * np.pi * day_num / 3.0)
+
+ temp = 6.0 + 4.0 * daily_cycle + multi_day_variation
+
+ temp = max(-10.0, min(40.0, temp))
+ temp_minutes[minute] = temp
+
+ return temp_minutes
+
+
+def _create_synthetic_load_data(n_days=7, now_utc=None):
+ """Create synthetic load data for testing"""
+ if now_utc is None:
+ now_utc = datetime.now(timezone.utc)
+
+ n_minutes = n_days * 24 * 60
+ load_minutes = {}
+ cumulative = 0.0
+
+ # Build backwards from now (minute 0 = now)
+ # Start from a multiple of STEP_MINUTES and go down to 0
+ start_minute = (n_minutes // STEP_MINUTES) * STEP_MINUTES
+ for minute in range(start_minute, -STEP_MINUTES, -STEP_MINUTES):
+ # Time for this minute
+ dt = now_utc - timedelta(minutes=minute)
+ hour = dt.hour
+
+ # Simple daily pattern: higher during day
+ if 6 <= hour < 22:
+ energy = 0.2 + 0.1 * np.random.randn() # ~0.2 kWh per 5 min during day
+ else:
+ energy = 0.05 + 0.02 * np.random.randn() # ~0.05 kWh at night
+
+ energy = max(0, energy)
+ cumulative += energy
+ load_minutes[minute] = cumulative
+
+ return load_minutes
+
+
+def _test_dataset_creation():
+ """Test dataset creation from load minute data with train/val split"""
+ predictor = LoadPredictor()
+ now_utc = datetime.now(timezone.utc)
+
+ # Create synthetic load data: 7 days
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+
+ # Create dataset - now returns 5 values (train + val split)
+ X_train, y_train, train_weights, X_val, y_val = predictor._create_dataset(load_data, now_utc, time_decay_days=7)
+
+ # Should have valid training samples
+ assert X_train is not None, "Training X should not be None"
+ assert X_train.shape[0] > 0, "Training should have samples"
+ assert X_train.shape[0] == y_train.shape[0], "X_train and y_train should have same number of samples"
+ assert train_weights.shape[0] == X_train.shape[0], "Train weights should match training samples"
+
+ # Should have validation samples
+ assert X_val is not None, "Validation X should not be None"
+ assert X_val.shape[0] > 0, "Validation should have samples"
+ assert X_val.shape[0] == y_val.shape[0], "X_val and y_val should have same number of samples"
+
+ # Feature dimension: TOTAL_FEATURES
+ assert X_train.shape[1] == TOTAL_FEATURES, f"Expected {TOTAL_FEATURES} features, got {X_train.shape[1]}"
+
+ # Output dimension: OUTPUT_STEPS (1 for autoregressive)
+ assert y_train.shape[1] == OUTPUT_STEPS, f"Expected {OUTPUT_STEPS} outputs, got {y_train.shape[1]}"
+
+ # Validation should be approximately 24h worth of samples (288 at 5-min intervals)
+ expected_val_samples = 24 * 60 // STEP_MINUTES
+ assert abs(X_val.shape[0] - expected_val_samples) < 10, f"Expected ~{expected_val_samples} val samples, got {X_val.shape[0]}"
+
+
+def _test_dataset_with_pv():
+ """Test dataset creation includes PV features correctly"""
+ predictor = LoadPredictor()
+ # Use a fixed daytime hour to ensure PV generation
+ now_utc = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc) # Noon on summer day
+
+ # Create synthetic load and PV data
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ pv_data = _create_synthetic_pv_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only for training
+
+ # Create dataset with PV data
+ X_train, y_train, train_weights, X_val, y_val = predictor._create_dataset(load_data, now_utc, pv_minutes=pv_data, time_decay_days=7)
+
+ # Should have valid samples
+ assert X_train is not None, "Training X should not be None"
+ assert X_train.shape[0] > 0, "Training should have samples"
+
+ # Feature dimension should include PV features: LOOKBACK_STEPS (load) + LOOKBACK_STEPS (PV) + LOOKBACK_STEPS (temp) + 4 (time) = TOTAL_FEATURES
+ from load_predictor import NUM_LOAD_FEATURES, NUM_PV_FEATURES, NUM_TEMP_FEATURES, NUM_TIME_FEATURES
+
+ expected_features = NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES + NUM_TIME_FEATURES
+ assert X_train.shape[1] == expected_features, f"Expected {expected_features} features with PV, got {X_train.shape[1]}"
+ assert X_train.shape[1] == TOTAL_FEATURES, f"TOTAL_FEATURES should be {expected_features}, is {TOTAL_FEATURES}"
+
+ # Verify PV features are not all zeros (unless no PV data provided)
+ # PV features are in the middle section: indices NUM_LOAD_FEATURES to NUM_LOAD_FEATURES+NUM_PV_FEATURES
+ pv_feature_section = X_train[:, NUM_LOAD_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES]
+ # At least some PV values should be non-zero (during daylight hours)
+ assert np.any(pv_feature_section > 0), "PV features should contain some non-zero values"
+
+ # Temperature features should be all zeros since we didn't provide temp_minutes
+ temp_feature_section = X_train[:, NUM_LOAD_FEATURES + NUM_PV_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES]
+ assert np.all(temp_feature_section == 0), "Temperature features should be zero when no temp data provided"
+
+
+def _test_dataset_with_temp():
+ """Test dataset creation includes temperature features correctly"""
+ predictor = LoadPredictor()
+ now_utc = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc)
+
+ # Create synthetic load and temperature data
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ temp_data = _create_synthetic_temp_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only
+
+ # Create dataset with temperature data
+ X_train, y_train, train_weights, X_val, y_val = predictor._create_dataset(load_data, now_utc, temp_minutes=temp_data, time_decay_days=7)
+
+ # Should have valid samples
+ assert X_train is not None, "Training X should not be None"
+ assert X_train.shape[0] > 0, "Training should have samples"
+
+ # Feature dimension should include temperature features
+ from load_predictor import NUM_LOAD_FEATURES, NUM_PV_FEATURES, NUM_TEMP_FEATURES, NUM_TIME_FEATURES
+
+ expected_features = NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES + NUM_TIME_FEATURES
+ assert X_train.shape[1] == expected_features, f"Expected {expected_features} features with temp, got {X_train.shape[1]}"
+ assert X_train.shape[1] == TOTAL_FEATURES, f"TOTAL_FEATURES should be {expected_features}, is {TOTAL_FEATURES}"
+
+ # Verify temperature features are not all zeros
+ # Temperature features are after load and PV: indices NUM_LOAD_FEATURES+NUM_PV_FEATURES to NUM_LOAD_FEATURES+NUM_PV_FEATURES+NUM_TEMP_FEATURES
+ temp_feature_section = X_train[:, NUM_LOAD_FEATURES + NUM_PV_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES + NUM_TEMP_FEATURES]
+ # At least some temperature values should be non-zero
+ assert np.any(temp_feature_section != 0), "Temperature features should contain non-zero values"
+ # Check temperature values are in reasonable range (after normalization they won't be in Celsius range)
+ assert np.min(temp_feature_section) > -50, "Temperature features should be reasonable"
+ assert np.max(temp_feature_section) < 50, "Temperature features should be reasonable"
+
+ # PV features should be all zeros since we didn't provide pv_minutes
+ pv_feature_section = X_train[:, NUM_LOAD_FEATURES : NUM_LOAD_FEATURES + NUM_PV_FEATURES]
+ assert np.all(pv_feature_section == 0), "PV features should be zero when no PV data provided"
+
+
+def _test_normalization():
+ """Test Z-score normalization correctness"""
+ predictor = LoadPredictor()
+
+ # Create test data
+ np.random.seed(42)
+ X = np.random.randn(100, TOTAL_FEATURES).astype(np.float32) * 10 + 5 # Mean ~5, std ~10
+
+ # Normalize with fit
+ X_norm = predictor._normalize_features(X, fit=True)
+
+ # Check mean ~0 and std ~1 along each feature
+ assert np.allclose(np.mean(X_norm, axis=0), 0, atol=0.1), "Normalized mean should be ~0"
+ assert np.allclose(np.std(X_norm, axis=0), 1, atol=0.1), "Normalized std should be ~1"
+
+ # Test target normalization
+ y = np.random.randn(100, OUTPUT_STEPS).astype(np.float32) * 2 + 3
+ y_norm = predictor._normalize_targets(y, fit=True)
+
+ # Check denormalization
+ y_denorm = predictor._denormalize_predictions(y_norm)
+ assert np.allclose(y, y_denorm, atol=1e-5), "Denormalization should recover original"
+
+
+def _test_adam_optimizer():
+ """Test Adam optimizer update step"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ predictor._initialize_weights()
+
+ # Store original weights
+ orig_weight = predictor.weights[0].copy()
+
+ # Create dummy gradients
+ weight_grads = [np.ones_like(w) * 0.1 for w in predictor.weights]
+ bias_grads = [np.ones_like(b) * 0.1 for b in predictor.biases]
+
+ # Perform Adam update
+ predictor._adam_update(weight_grads, bias_grads)
+
+ # Weight should have changed
+ assert not np.allclose(orig_weight, predictor.weights[0]), "Adam update should change weights"
+
+ # adam_t should have incremented
+ assert predictor.adam_t == 1, "Adam timestep should be 1"
+
+
+def _test_training_convergence():
+ """Test that training converges on simple synthetic data"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ now_utc = datetime.now(timezone.utc)
+
+ # Create simple repeating daily pattern
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+
+ # Train with few epochs
+ val_mae = predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=10, time_decay_days=7)
+
+ # Training should complete and return a validation MAE
+ assert val_mae is not None, "Training should return validation MAE"
+ assert predictor.model_initialized, "Model should be initialized after training"
+ assert predictor.epochs_trained > 0, "Should have trained some epochs"
+
+
+def _test_training_with_pv():
+ """Test that training works correctly with PV input features"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ now_utc = datetime.now(timezone.utc)
+
+ # Create load and PV data
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ pv_data = _create_synthetic_pv_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only for training
+
+ # Train with PV data
+ val_mae = predictor.train(load_data, now_utc, pv_minutes=pv_data, is_initial=True, epochs=10, time_decay_days=7)
+
+ # Training should complete successfully
+ assert val_mae is not None, "Training with PV should return validation MAE"
+ assert predictor.model_initialized, "Model should be initialized after training with PV"
+ assert predictor.epochs_trained > 0, "Should have trained some epochs with PV data"
+
+ # Verify the model can accept correct input size (with PV features)
+ test_input = np.random.randn(1, TOTAL_FEATURES).astype(np.float32)
+ output, _, _ = predictor._forward(test_input)
+ assert output.shape == (1, OUTPUT_STEPS), "Model should produce correct output shape with PV features"
+
+
+def _test_training_with_temp():
+ """Test that training works correctly with temperature input features"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ now_utc = datetime.now(timezone.utc)
+
+ # Create load and temperature data
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ temp_data = _create_synthetic_temp_data(n_days=7, now_utc=now_utc, forecast_hours=0) # Historical only for training
+
+ # Train with temperature data
+ val_mae = predictor.train(load_data, now_utc, temp_minutes=temp_data, is_initial=True, epochs=10, time_decay_days=7)
+
+ # Training should complete successfully
+ assert val_mae is not None, "Training with temperature should return validation MAE"
+ assert predictor.model_initialized, "Model should be initialized after training with temperature"
+ assert predictor.epochs_trained > 0, "Should have trained some epochs with temperature data"
+
+ # Verify the model can accept correct input size (with temperature features)
+ test_input = np.random.randn(1, TOTAL_FEATURES).astype(np.float32)
+ output, _, _ = predictor._forward(test_input)
+ assert output.shape == (1, OUTPUT_STEPS), "Model should produce correct output shape with temperature features"
+
+
+def _test_model_persistence():
+ """Test model save/load with version check"""
+ predictor = LoadPredictor(learning_rate=0.005)
+ now_utc = datetime.now(timezone.utc)
+
+ # Train briefly
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=5, now_utc=now_utc)
+ predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=5, time_decay_days=7)
+
+ # Save to temp file
+ with tempfile.NamedTemporaryFile(suffix=".npz", delete=False) as f:
+ temp_path = f.name
+
+ try:
+ predictor.save(temp_path)
+
+ # Load into new predictor
+ predictor2 = LoadPredictor(learning_rate=0.005)
+ success = predictor2.load(temp_path)
+
+ assert success, "Model load should succeed"
+ assert predictor2.model_initialized, "Loaded model should be marked as initialized"
+
+ # Compare weights
+ for w1, w2 in zip(predictor.weights, predictor2.weights):
+ assert np.allclose(w1, w2), "Weights should match after load"
+
+ # Test prediction produces same result
+ np.random.seed(123)
+ test_input = np.random.randn(1, TOTAL_FEATURES).astype(np.float32)
+ out1, _, _ = predictor._forward(test_input)
+ out2, _, _ = predictor2._forward(test_input)
+ assert np.allclose(out1, out2), "Predictions should match after load"
+
+ finally:
+ if os.path.exists(temp_path):
+ os.unlink(temp_path)
+
+
+def _test_cold_start():
+ """Test cold start with insufficient data returns None"""
+ predictor = LoadPredictor()
+ now_utc = datetime.now(timezone.utc)
+
+ # Only 1 day of data (insufficient for 48h horizon + lookback)
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=1, now_utc=now_utc)
+
+ # Training should fail or return None
+ val_mae = predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=5, time_decay_days=7)
+
+ # With only 1 day of data, we can't create a valid dataset for 48h prediction
+ # The result depends on actual data coverage
+ # Just verify it doesn't crash
+ assert True, "Cold start should not crash"
+
+
+def _test_fine_tune():
+ """Test fine-tuning on recent data only"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ now_utc = datetime.now(timezone.utc)
+
+ # Initial training on 7 days
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=5, time_decay_days=7)
+
+ # Store original weights
+ orig_weights = [w.copy() for w in predictor.weights]
+
+ # Fine-tune with same data but as fine-tune mode
+ # Note: Fine-tune uses is_finetune=True which only looks at last 24h
+ # For the test to work, we need enough data for the full training
+ predictor.train(load_data, now_utc, pv_minutes=None, is_initial=False, epochs=3, time_decay_days=7)
+
+ # Even if fine-tune has insufficient data, initial training should have worked
+ # The test validates that fine-tune doesn't crash and model is still valid
+ assert predictor.model_initialized, "Model should still be initialized after fine-tune attempt"
+
+
+def _test_prediction():
+ """Test end-to-end prediction"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ now_utc = datetime.now(timezone.utc)
+ midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0)
+
+ # Train on synthetic data
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ predictor.train(load_data, now_utc, pv_minutes=None, is_initial=True, epochs=10, time_decay_days=7)
+
+ # Make prediction
+ predictions = predictor.predict(load_data, now_utc, midnight_utc, pv_minutes=None)
+
+ # Should return dict with minute keys
+ if predictions: # May return empty dict if validation fails
+ assert isinstance(predictions, dict), "Predictions should be a dict"
+ # Check some predictions exist
+ assert len(predictions) > 0, "Should have some predictions"
+ # All values should be non-negative
+ for minute, val in predictions.items():
+ assert val >= 0, f"Prediction at minute {minute} should be non-negative"
+
+
+def _test_prediction_with_pv():
+ """Test end-to-end prediction with PV forecast data"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ now_utc = datetime.now(timezone.utc)
+ midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0)
+
+ # Create load and PV data (with 48h forecast)
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ pv_data = _create_synthetic_pv_data(n_days=7, now_utc=now_utc, forecast_hours=48) # Include forecast
+
+ # Train with PV data
+ predictor.train(load_data, now_utc, pv_minutes=pv_data, is_initial=True, epochs=10, time_decay_days=7)
+
+ # Make prediction with PV forecast
+ predictions = predictor.predict(load_data, now_utc, midnight_utc, pv_minutes=pv_data)
+
+ # Should return predictions
+ if predictions:
+ assert isinstance(predictions, dict), "Predictions should be a dict"
+ assert len(predictions) > 0, "Should have predictions with PV data"
+
+ # Verify all values are non-negative
+ for minute, val in predictions.items():
+ assert val >= 0, f"Prediction at minute {minute} should be non-negative"
+
+ # Verify predictions span 48 hours (576 steps at 5-min intervals)
+ max_minute = max(predictions.keys())
+ assert max_minute >= 2800, f"Predictions should span ~48h (2880 min), got {max_minute} min"
+
+
+def _test_prediction_with_temp():
+ """Test end-to-end prediction with temperature forecast data"""
+ predictor = LoadPredictor(learning_rate=0.01)
+ now_utc = datetime.now(timezone.utc)
+ midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0)
+
+ # Create load and temperature data (with 48h forecast)
+ np.random.seed(42)
+ load_data = _create_synthetic_load_data(n_days=7, now_utc=now_utc)
+ temp_data = _create_synthetic_temp_data(n_days=7, now_utc=now_utc, forecast_hours=48) # Include forecast
+
+ # Train with temperature data
+ predictor.train(load_data, now_utc, temp_minutes=temp_data, is_initial=True, epochs=10, time_decay_days=7)
+
+ # Make prediction with temperature forecast
+ predictions = predictor.predict(load_data, now_utc, midnight_utc, temp_minutes=temp_data)
+
+ # Should return predictions
+ if predictions:
+ assert isinstance(predictions, dict), "Predictions should be a dict"
+ assert len(predictions) > 0, "Should have predictions with temperature data"
+
+ # Verify all values are non-negative
+ for minute, val in predictions.items():
+ assert val >= 0, f"Prediction at minute {minute} should be non-negative"
+
+ # Verify predictions span 48 hours (576 steps at 5-min intervals)
+ max_minute = max(predictions.keys())
+ assert max_minute >= 2800, f"Predictions should span ~48h (2880 min), got {max_minute} min"
+
+
+def _test_real_data_training():
+ """
+ Test training on real load_minutes_debug.json data and generate comparison chart
+ """
+ import json
+ import os
+
+ # Try to load the input_train_data.json which has real PV and temperature
+ input_train_paths = ["../coverage/input_train_data.json", "coverage/input_train_data.json", "input_train_data.json"]
+
+ load_data = None
+ pv_data = None
+ temp_data = None
+
+ for json_path in input_train_paths:
+ if os.path.exists(json_path):
+ with open(json_path, "r") as f:
+ train_data = json.load(f)
+ # Format: [load_minutes_new, age_days, load_minutes_now, pv_data, temperature_data]
+ if len(train_data) >= 5:
+ # Convert string keys to integers
+ load_data = {int(k): float(v) for k, v in train_data[0].items()}
+ pv_data = {int(k): float(v) for k, v in train_data[3].items()} if train_data[3] else {}
+ temp_data = {int(k): float(v) for k, v in train_data[4].items()} if train_data[4] else {}
+ print(f" Loaded training data from {json_path}")
+ print(f" Load: {len(load_data)} datapoints")
+ print(f" PV: {len(pv_data)} datapoints")
+ print(f" Temperature: {len(temp_data)} datapoints")
+ break
+
+ if load_data is None:
+ print(" WARNING: No training data found, skipping real data test")
+ return
+
+ # Initialize predictor with lower learning rate for better convergence
+ predictor = LoadPredictor(learning_rate=0.0005, max_load_kw=20.0)
+ now_utc = datetime.now(timezone.utc)
+ midnight_utc = now_utc.replace(hour=0, minute=0, second=0, microsecond=0)
+
+ # Calculate how many days of data we have
+ max_minute = max(load_data.keys())
+ n_days = max_minute / (24 * 60)
+ print(f" Data spans {n_days:.1f} days ({max_minute} minutes)")
+
+ # Generate synthetic data only if real data wasn't loaded
+ if pv_data is None or len(pv_data) == 0:
+ print(f" Generating synthetic PV data for {n_days:.1f} days...")
+ pv_data = _create_synthetic_pv_data(n_days=int(n_days) + 1, now_utc=now_utc, forecast_hours=48)
+ print(f" Generated {len(pv_data)} PV datapoints")
+
+ if temp_data is None or len(temp_data) == 0:
+ print(f" Generating synthetic temperature data for {n_days:.1f} days...")
+ temp_data = _create_synthetic_temp_data(n_days=int(n_days) + 1, now_utc=now_utc, forecast_hours=48)
+ print(f" Generated {len(temp_data)} temperature datapoints")
+
+ # Train on full dataset with more epochs for larger network
+ data_source = "real" if (pv_data and len(pv_data) > 100 and temp_data and len(temp_data) > 100) else "synthetic"
+ print(f" Training on real load + {data_source} PV/temperature with {len(load_data)} points...")
+ success = predictor.train(load_data, now_utc, pv_minutes=pv_data, temp_minutes=temp_data, is_initial=True, epochs=50, time_decay_days=7)
+
+ assert success, "Training on real data should succeed"
+ assert predictor.model_initialized, "Model should be initialized after training"
+
+ # Make predictions
+ print(" Generating predictions with PV + temperature forecasts...")
+ predictions = predictor.predict(load_data, now_utc, midnight_utc, pv_minutes=pv_data, temp_minutes=temp_data)
+
+ assert isinstance(predictions, dict), "Predictions should be a dict"
+ assert len(predictions) > 0, "Should have predictions"
+
+ print(f" Generated {len(predictions)} predictions")
+
+ # Create comparison chart using matplotlib
+ try:
+ import matplotlib
+
+ matplotlib.use("Agg") # Non-interactive backend
+ import matplotlib.pyplot as plt
+
+ # Chart layout: 7 days of history (negative hours) + 2 days of predictions (positive hours)
+ # X-axis: -168 to +48 hours (0 = now)
+ history_hours = 7 * 24 # 7 days back
+ prediction_hours = 48 # 2 days forward
+
+ # Convert historical load_data (cumulative kWh) to energy per 5-min step (kWh)
+ # Going backwards in time: minute 0 is now, higher minutes are past
+ historical_minutes = []
+ historical_energy = []
+ max_history_minutes = min(history_hours * 60, max_minute)
+
+ for minute in range(0, max_history_minutes, STEP_MINUTES):
+ if minute in load_data and (minute + STEP_MINUTES) in load_data:
+ energy_kwh = max(0, load_data[minute] - load_data.get(minute + STEP_MINUTES, load_data[minute]))
+ historical_minutes.append(minute)
+ historical_energy.append(energy_kwh)
+
+ # Extract validation period actual data (most recent 24h = day 7)
+ # This is the data the model was validated against
+ val_actual_minutes = []
+ val_actual_energy = []
+ val_period_hours = 24 # Most recent 24h
+ for minute in range(0, val_period_hours * 60, STEP_MINUTES):
+ if minute in load_data and (minute + STEP_MINUTES) in load_data:
+ energy_kwh = max(0, load_data[minute] - load_data.get(minute + STEP_MINUTES, load_data[minute]))
+ val_actual_minutes.append(minute)
+ val_actual_energy.append(energy_kwh)
+
+ # Generate validation predictions: what would the model predict for day 7
+ # using only data from day 2-7 (excluding most recent 24h)?
+ # Simulate predicting from 24h ago
+ val_pred_minutes = []
+ val_pred_energy = []
+
+ # Create a modified load_data that excludes the most recent 24h
+ # This simulates predicting "yesterday" from "2 days ago"
+ val_holdout_minutes = val_period_hours * 60
+ shifted_load_data = {}
+ for minute, cum_kwh in load_data.items():
+ if minute >= val_holdout_minutes:
+ # Shift back by 24h so model predicts into "held out" period
+ shifted_load_data[minute - val_holdout_minutes] = cum_kwh
+
+ # Make validation prediction (predict next 24h from shifted data)
+ if shifted_load_data:
+ shifted_now = now_utc - timedelta(hours=val_period_hours)
+ shifted_midnight = shifted_now.replace(hour=0, minute=0, second=0, microsecond=0)
+
+ # Create shifted PV data for validation prediction
+ shifted_pv_data = {}
+ for minute, cum_kwh in pv_data.items():
+ if minute >= val_holdout_minutes:
+ shifted_pv_data[minute - val_holdout_minutes] = cum_kwh
+
+ # Create shifted temperature data for validation prediction
+ shifted_temp_data = {}
+ for minute, temp in temp_data.items():
+ if minute >= val_holdout_minutes:
+ shifted_temp_data[minute - val_holdout_minutes] = temp
+
+ val_predictions = predictor.predict(shifted_load_data, shifted_now, shifted_midnight, pv_minutes=shifted_pv_data, temp_minutes=shifted_temp_data)
+
+ # Extract first 24h of validation predictions
+ val_pred_keys = sorted(val_predictions.keys())
+ for i, minute in enumerate(val_pred_keys):
+ if minute >= val_period_hours * 60:
+ break
+ if i == 0:
+ energy_kwh = val_predictions[minute]
+ else:
+ prev_minute = val_pred_keys[i - 1]
+ energy_kwh = max(0, val_predictions[minute] - val_predictions[prev_minute])
+ val_pred_minutes.append(minute)
+ val_pred_energy.append(energy_kwh)
+
+ # Convert predictions (cumulative kWh) to energy per step (kWh)
+ # predictions dict is: {0: cum0, 5: cum5, 10: cum10, ...} representing FUTURE
+ pred_minutes = []
+ pred_energy = []
+ pred_keys = sorted(predictions.keys())
+ for i, minute in enumerate(pred_keys):
+ if minute >= prediction_hours * 60:
+ break
+ if i == 0:
+ # First step - use the value directly as energy
+ energy_kwh = predictions[minute]
+ else:
+ # Subsequent steps - calculate difference from previous
+ prev_minute = pred_keys[i - 1]
+ energy_kwh = max(0, predictions[minute] - predictions[prev_minute])
+ pred_minutes.append(minute)
+ pred_energy.append(energy_kwh)
+
+ # Convert PV data to energy per step for plotting
+ # Historical PV (positive minutes, going back in time)
+ pv_historical_minutes = []
+ pv_historical_energy = []
+ for minute in range(0, max_history_minutes, STEP_MINUTES):
+ if minute in pv_data and (minute + STEP_MINUTES) in pv_data:
+ energy_kwh = max(0, pv_data[minute] - pv_data.get(minute + STEP_MINUTES, pv_data[minute]))
+ pv_historical_minutes.append(minute)
+ pv_historical_energy.append(energy_kwh)
+
+ # Future PV forecasts (negative minutes in pv_data dict, representing future)
+ pv_forecast_minutes = []
+ pv_forecast_energy = []
+ for minute in range(-prediction_hours * 60, 0, STEP_MINUTES):
+ if minute in pv_data and (minute + STEP_MINUTES) in pv_data:
+ energy_kwh = max(0, pv_data[minute] - pv_data.get(minute + STEP_MINUTES, pv_data[minute]))
+ pv_forecast_minutes.append(minute)
+ pv_forecast_energy.append(energy_kwh)
+
+ # Extract temperature data (non-cumulative, so we use raw values)
+ # Historical temperature (positive minutes in temp_data dict, going back in time)
+ temp_historical_minutes = []
+ temp_historical_celsius = []
+ for minute in range(0, max_history_minutes, STEP_MINUTES):
+ if minute in temp_data:
+ temp_celsius = temp_data[minute]
+ temp_historical_minutes.append(minute)
+ temp_historical_celsius.append(temp_celsius)
+
+ # Future temperature forecasts (negative minutes in temp_data dict, representing future)
+ temp_forecast_minutes = []
+ temp_forecast_celsius = []
+ for minute in range(-prediction_hours * 60, 0, STEP_MINUTES):
+ if minute in temp_data:
+ temp_celsius = temp_data[minute]
+ temp_forecast_minutes.append(minute)
+ temp_forecast_celsius.append(temp_celsius)
+
+ # Create figure with single plot showing timeline
+ fig, ax = plt.subplots(1, 1, figsize=(16, 6))
+
+ # Create secondary y-axis for temperature
+ ax2 = ax.twinx()
+
+ # Plot PV data first (in background)
+ # Historical PV (negative hours, going back in time)
+ if pv_historical_minutes:
+ pv_hist_hours = [-m / 60 for m in pv_historical_minutes] # Negative for past
+ ax.plot(pv_hist_hours, pv_historical_energy, "orange", linewidth=0.8, label="Historical PV (7 days)", alpha=0.3, linestyle="--")
+
+ # Future PV forecasts (positive hours, going forward)
+ if pv_forecast_minutes:
+ # Convert negative minutes to positive hours for future
+ pv_forecast_hours = [-m / 60 for m in pv_forecast_minutes] # Negative minutes become positive hours
+ ax.plot(pv_forecast_hours, pv_forecast_energy, "orange", linewidth=1.2, label="PV Forecast (48h)", alpha=0.5, linestyle="--")
+
+ # Plot temperature data on secondary y-axis
+ # Historical temperature (negative hours, going back in time)
+ if temp_historical_minutes:
+ temp_hist_hours = [-m / 60 for m in temp_historical_minutes] # Negative for past
+ ax2.plot(temp_hist_hours, temp_historical_celsius, "purple", linewidth=0.8, label="Historical Temp (7 days)", alpha=0.4, linestyle="-.")
+
+ # Future temperature forecasts (positive hours, going forward)
+ if temp_forecast_minutes:
+ # Convert negative minutes to positive hours for future
+ temp_forecast_hours = [-m / 60 for m in temp_forecast_minutes] # Negative minutes become positive hours
+ ax2.plot(temp_forecast_hours, temp_forecast_celsius, "purple", linewidth=1.2, label="Temp Forecast (48h)", alpha=0.6, linestyle="-.")
+
+ # Plot historical data (negative hours, going back in time)
+ # minute 0 = now (hour 0), minute 60 = 1 hour ago (hour -1)
+ if historical_minutes:
+ hist_hours = [-m / 60 for m in historical_minutes] # Negative for past
+ ax.plot(hist_hours, historical_energy, "b-", linewidth=0.8, label="Historical Load (7 days)", alpha=0.5)
+
+ # Highlight validation period actual data (most recent 24h) with thicker line
+ if val_actual_minutes:
+ val_actual_hours = [-m / 60 for m in val_actual_minutes] # Negative for past
+ ax.plot(val_actual_hours, val_actual_energy, "b-", linewidth=1.5, label="Actual Day 7 (validation)", alpha=0.9)
+
+ # Plot validation predictions (what model predicted for day 7)
+ if val_pred_minutes:
+ # These predictions map to the validation period (most recent 24h)
+ # val_pred minute 0 -> actual minute 0 -> hour 0, etc.
+ val_pred_hours = [-m / 60 for m in val_pred_minutes] # Same position as actual
+ ax.plot(val_pred_hours, val_pred_energy, "g-", linewidth=1.5, label="ML Prediction (day 7)", alpha=0.9)
+
+ # Plot future predictions (positive hours, going forward)
+ if pred_minutes:
+ pred_hours = [m / 60 for m in pred_minutes] # Positive for future
+ ax.plot(pred_hours, pred_energy, "r-", linewidth=1.5, label="ML Prediction (48h future)", alpha=0.9)
+
+ # Add vertical line at "now"
+ ax.axvline(x=0, color="black", linestyle="--", linewidth=2, label="Now", alpha=0.8)
+
+ # Shade the validation region (most recent 24h)
+ ax.axvspan(-24, 0, alpha=0.1, color="green", label="Validation Period")
+
+ # Formatting
+ ax.set_xlabel("Hours (negative = past, positive = future)", fontsize=12)
+ ax.set_ylabel("Load (kWh per 5 min)", fontsize=12)
+ ax2.set_ylabel("Temperature (°C)", fontsize=12, color="purple")
+ ax2.tick_params(axis="y", labelcolor="purple")
+ ax.set_title("ML Load Predictor with PV + Temperature Input: Validation (Day 7) + 48h Forecast", fontsize=14, fontweight="bold")
+
+ # Combine legends from both axes
+ lines1, labels1 = ax.get_legend_handles_labels()
+ lines2, labels2 = ax2.get_legend_handles_labels()
+ ax.legend(lines1 + lines2, labels1 + labels2, loc="upper right", fontsize=10)
+ ax.grid(True, alpha=0.3)
+ ax.set_xlim(-history_hours, prediction_hours)
+
+ # Add day markers
+ for day in range(-7, 3):
+ hour = day * 24
+ if -history_hours <= hour <= prediction_hours:
+ ax.axvline(x=hour, color="gray", linestyle=":", linewidth=0.5, alpha=0.5)
+
+ plt.tight_layout()
+
+ # Save to coverage directory
+ chart_paths = ["../coverage/ml_prediction_chart.png", "coverage/ml_prediction_chart.png", "ml_prediction_chart.png"]
+ for chart_path in chart_paths:
+ try:
+ plt.savefig(chart_path, dpi=150, bbox_inches="tight")
+ print(f" Chart saved to {chart_path}")
+ break
+ except:
+ continue
+
+ plt.close()
+
+ except ImportError:
+ print(" WARNING: matplotlib not available, skipping chart generation")
+
+
+def _test_component_fetch_load_data():
+ """Test LoadMLComponent._fetch_load_data method"""
+ import asyncio
+ from datetime import datetime, timezone
+ from load_ml_component import LoadMLComponent
+ from unittest.mock import MagicMock
+
+ # Helper to run async tests
+ def run_async(coro):
+ try:
+ loop = asyncio.get_event_loop()
+ except RuntimeError:
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ return loop.run_until_complete(coro)
+
+ # Create mock base object with all necessary properties
+ class MockBase:
+ def __init__(self):
+ self.prefix = "predbat"
+ self.config_root = None
+ self.now_utc = datetime.now(timezone.utc)
+ self.midnight_utc = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
+ self.minutes_now = (self.now_utc - self.midnight_utc).seconds // 60
+ self.local_tz = timezone.utc
+ self.args = {}
+ self.log_messages = []
+
+ def log(self, msg):
+ self.log_messages.append(msg)
+
+ def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None):
+ return {
+ "load_today": ["sensor.load_today"],
+ "load_power": None, # Disable load_power to simplify test
+ "car_charging_energy": None, # Disable car charging to simplify test
+ "load_scaling": 1.0,
+ "car_charging_energy_scale": 1.0,
+ }.get(key, default)
+
+ def get_state_wrapper(self, entity_id, default=None, attribute=None, refresh=False, required_unit=None, raw=False):
+ """Mock get_state_wrapper - returns None for temperature by default"""
+ return default
+
+ def fetch_pv_forecast(self):
+ """Mock fetch_pv_forecast - returns empty forecasts"""
+ return {}, {}
+
+ # Create synthetic load data (28 days worth)
+ def create_load_minutes(days=28, all_minutes=False):
+ """
+ Create cumulative load data going backwards from minute 0
+
+ Args:
+ days: Number of days of data to create
+ all_minutes: If True, create entries for every minute (not just 5-min intervals)
+ """
+ load_data = {}
+ cumulative = 0.0
+
+ if all_minutes:
+ # Create entry for every minute (for car charging test)
+ for minute in range(days * 24 * 60, -1, -1):
+ energy_step = 0.1 / 5 # Scale down since we have 5x more entries
+ cumulative += energy_step
+ load_data[minute] = cumulative
+ else:
+ # Create entries at 5-minute intervals (normal case)
+ for minute in range(days * 24 * 60, -1, -5):
+ energy_step = 0.1 # 0.1 kWh per 5 min
+ cumulative += energy_step
+ load_data[minute] = cumulative
+
+ return load_data, days
+
+ # Test 1: Successful fetch with minimal config
+ async def test_basic_fetch():
+ mock_base = MockBase()
+ load_data, age = create_load_minutes(28)
+ mock_base.minute_data_load = MagicMock(return_value=(load_data, age))
+ mock_base.minute_data_import_export = MagicMock(return_value=None)
+ # Mock the fill_load_from_power method - it should just return the load_minutes unchanged
+ mock_base.fill_load_from_power = MagicMock(side_effect=lambda x, y: x)
+
+ component = LoadMLComponent(mock_base, load_ml_enable=True)
+ # Override default values for testing
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is not None, "Should return load data"
+ assert result_age == 28, f"Expected 28 days, got {result_age}"
+ assert len(result_data) > 0, "Load data should not be empty"
+ assert result_now >= 0, f"Current load should be non-negative, got {result_now}"
+ print(" ✓ Basic fetch successful")
+
+ # Test 2: Missing sensor (should return None)
+ async def test_missing_sensor():
+ class MockBaseNoSensor:
+ def __init__(self):
+ self.prefix = "predbat"
+ self.config_root = None
+ self.now_utc = datetime.now(timezone.utc)
+ self.local_tz = timezone.utc
+ self.args = {}
+
+ def log(self, msg):
+ pass
+
+ def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None):
+ return default
+
+ mock_base_no_sensor = MockBaseNoSensor()
+
+ component = LoadMLComponent(mock_base_no_sensor, load_ml_enable=True)
+ # Override default values for testing
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is None, "Should return None when sensor missing"
+ assert result_age == 0, "Age should be 0 when sensor missing"
+ assert result_now == 0, "Current load should be 0 when sensor missing"
+ print(" ✓ Missing sensor handled correctly")
+
+ # Test 3: Car charging subtraction
+ async def test_car_charging_subtraction():
+ mock_base_with_car = MockBase()
+
+ # Create load data with entries for EVERY minute (not just 5-min intervals)
+ # This is required because the component's car charging subtraction loop
+ # iterates over every minute from 1 to max_minute
+ original_load_data, age = create_load_minutes(7, all_minutes=True)
+ car_charging_data = {i: i * 0.001 for i in range(0, 7 * 24 * 60 + 1)} # Small cumulative car charging (0.001 kWh/min)
+
+ # Override get_arg to enable car_charging_energy
+ def mock_get_arg_with_car(key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None):
+ return {
+ "load_today": ["sensor.load_today"],
+ "load_power": None,
+ "car_charging_energy": ["sensor.car_charging"], # Enable car charging
+ "load_scaling": 1.0,
+ "car_charging_energy_scale": 1.0,
+ }.get(key, default)
+
+ mock_base_with_car.get_arg = mock_get_arg_with_car
+
+ # Return a copy of the data so the original isn't modified
+ mock_base_with_car.minute_data_load = MagicMock(return_value=(dict(original_load_data), age))
+ mock_base_with_car.minute_data_import_export = MagicMock(return_value=car_charging_data)
+
+ component = LoadMLComponent(mock_base_with_car, load_ml_enable=True)
+ # Override default values for testing
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is not None, f"Should return load data"
+ assert result_age > 0, f"Should have valid age (got {result_age})"
+ assert len(result_data) > 0, "Result data should not be empty"
+ assert result_now >= 0, f"Current load should be non-negative, got {result_now}"
+
+ # Verify car charging was called
+ assert mock_base_with_car.minute_data_import_export.called, "minute_data_import_export should be called"
+
+ # Verify all values are non-negative after subtraction
+ for minute, value in result_data.items():
+ assert value >= 0, f"Load at minute {minute} should be non-negative, got {value}"
+
+ print(" ✓ Car charging subtraction works")
+
+ # Test 4: Load power fill
+ async def test_load_power_fill():
+ mock_base_with_power = MockBase()
+
+ # Override get_arg to enable load_power
+ def mock_get_arg_with_power(key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None):
+ return {
+ "load_today": ["sensor.load_today"],
+ "load_power": ["sensor.load_power"], # Enable load_power
+ "car_charging_energy": None,
+ "load_scaling": 1.0,
+ "car_charging_energy_scale": 1.0,
+ }.get(key, default)
+
+ mock_base_with_power.get_arg = mock_get_arg_with_power
+
+ load_data, age = create_load_minutes(7)
+ load_power_data, _ = create_load_minutes(7)
+
+ mock_base_with_power.minute_data_load = MagicMock(side_effect=[(load_data, age), (load_power_data, age)]) # First call for load_today # Second call for load_power
+ mock_base_with_power.minute_data_import_export = MagicMock(return_value=None)
+ mock_base_with_power.fill_load_from_power = MagicMock(return_value=load_data)
+
+ component = LoadMLComponent(mock_base_with_power, load_ml_enable=True)
+ # Override default values for testing
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is not None, "Should return load data"
+ assert mock_base_with_power.fill_load_from_power.called, "fill_load_from_power should be called"
+ assert result_now >= 0, f"Current load should be non-negative, got {result_now}"
+ print(" ✓ Load power fill invoked")
+
+ # Test 5: Exception handling
+ async def test_exception_handling():
+ mock_base = MockBase()
+ mock_base.minute_data_load = MagicMock(side_effect=Exception("Test exception"))
+
+ component = LoadMLComponent(mock_base, load_ml_enable=True)
+ # Override default values for testing
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is None, "Should return None on exception"
+ assert result_age == 0, "Age should be 0 on exception"
+ assert result_now == 0, "Current load should be 0 on exception"
+ print(" ✓ Exception handling works")
+
+ # Test 6: Empty load data
+ async def test_empty_load_data():
+ mock_base = MockBase()
+ mock_base.minute_data_load = MagicMock(return_value=(None, 0))
+ mock_base.minute_data_import_export = MagicMock(return_value=None)
+
+ component = LoadMLComponent(mock_base, load_ml_enable=True)
+ # Override default values for testing
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is None, "Should return None when load data is empty"
+ assert result_age == 0, "Age should be 0 when load data is empty"
+ assert result_now == 0, "Current load should be 0 when load data is empty"
+ print(" ✓ Empty load data handled correctly")
+
+ # Test 7: Temperature data fetch with future predictions only
+ async def test_temperature_data_fetch():
+ from datetime import timedelta
+
+ mock_base_with_temp = MockBase()
+
+ # Create mock temperature data (dict with timestamp strings as keys)
+ # This simulates future temperature predictions from sensor.predbat_temperature attribute "results"
+ base_time = mock_base_with_temp.now_utc
+ temp_predictions = {}
+ for hours_ahead in range(1, 49): # 48 hours of predictions
+ timestamp = base_time + timedelta(hours=hours_ahead)
+ timestamp_str = timestamp.strftime("%Y-%m-%dT%H:%M:%S%z")
+ temp_predictions[timestamp_str] = 15.0 + (hours_ahead % 12) # Simulated temperature pattern
+
+ # Override get_state_wrapper using MagicMock to return temperature predictions
+ def mock_get_state_wrapper_side_effect(entity_id, default=None, attribute=None, refresh=False, required_unit=None, raw=False):
+ if entity_id == "sensor.predbat_temperature" and attribute == "results":
+ return temp_predictions
+ return default
+
+ mock_base_with_temp.get_state_wrapper = MagicMock(side_effect=mock_get_state_wrapper_side_effect)
+
+ load_data, age = create_load_minutes(7)
+
+ # Mock minute_data_load to return load data
+ mock_base_with_temp.minute_data_load = MagicMock(return_value=(load_data, age))
+ mock_base_with_temp.minute_data_import_export = MagicMock(return_value={})
+ mock_base_with_temp.fill_load_from_power = MagicMock(side_effect=lambda x, y: x)
+
+ component = LoadMLComponent(mock_base_with_temp, load_ml_enable=True)
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is not None, "Should return load data"
+ assert result_temp is not None, "Should return temperature data"
+ assert isinstance(result_temp, dict), "Temperature data should be a dict"
+ assert len(result_temp) > 0, "Temperature data should not be empty"
+
+ # Verify we have future temperature data (positive minutes from midnight)
+ # Note: minute_data with backwards=False returns positive minute keys
+ # These represent minutes from midnight forward (future predictions)
+ assert len(result_temp) > 0, "Should have future temperature predictions"
+
+ # Verify get_state_wrapper was called correctly
+ assert mock_base_with_temp.get_state_wrapper.called, "get_state_wrapper should be called"
+
+ print(" ✓ Temperature data fetch (future predictions) works")
+
+ # Test 8: Temperature data with no predictions (None return)
+ async def test_temperature_no_data():
+ mock_base_no_temp = MockBase()
+
+ load_data, age = create_load_minutes(7)
+ mock_base_no_temp.minute_data_load = MagicMock(return_value=(load_data, age))
+ mock_base_no_temp.minute_data_import_export = MagicMock(return_value={})
+ mock_base_no_temp.fill_load_from_power = MagicMock(side_effect=lambda x, y: x)
+
+ # get_state_wrapper returns None (default behavior)
+
+ component = LoadMLComponent(mock_base_no_temp, load_ml_enable=True)
+ component.ml_learning_rate = 0.001
+ component.ml_epochs_initial = 10
+ component.ml_epochs_update = 2
+ component.ml_min_days = 1
+ component.ml_validation_threshold = 2.0
+ component.ml_time_decay_days = 7
+ component.ml_max_load_kw = 23.0
+ component.ml_max_model_age_hours = 48
+
+ result_data, result_age, result_now, result_pv, result_temp = await component._fetch_load_data()
+
+ assert result_data is not None, "Should return load data"
+ assert result_temp is not None, "Should return temperature data (empty dict)"
+ assert isinstance(result_temp, dict), "Temperature data should be a dict"
+ assert len(result_temp) == 0, "Temperature data should be empty when no predictions available"
+
+ print(" ✓ Temperature data with no predictions handled correctly")
+
+ # Run all sub-tests
+ print(" Running LoadMLComponent._fetch_load_data tests:")
+ run_async(test_basic_fetch())
+ run_async(test_missing_sensor())
+ run_async(test_car_charging_subtraction())
+ run_async(test_load_power_fill())
+ run_async(test_exception_handling())
+ run_async(test_empty_load_data())
+ run_async(test_temperature_data_fetch())
+ run_async(test_temperature_no_data())
+ print(" All _fetch_load_data tests passed!")
+
+
+def _test_component_publish_entity():
+ """Test LoadMLComponent._publish_entity method"""
+ from datetime import datetime, timezone, timedelta
+ from load_ml_component import LoadMLComponent
+ from unittest.mock import MagicMock
+ from const import TIME_FORMAT
+
+ # Create mock base object
+ class MockBase:
+ def __init__(self):
+ self.prefix = "predbat"
+ self.config_root = None
+ self.now_utc = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
+ self.midnight_utc = datetime(2026, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+ self.minutes_now = 720 # 12:00 = 720 minutes since midnight
+ self.local_tz = timezone.utc
+ self.args = {}
+ self.log_messages = []
+ self.dashboard_calls = []
+
+ def log(self, msg):
+ self.log_messages.append(msg)
+
+ def get_arg(self, key, default=None, indirect=True, combine=False, attribute=None, index=None, domain=None, can_override=True, required_unit=None):
+ return {
+ "load_today": ["sensor.load_today"],
+ "load_power": None,
+ "car_charging_energy": None,
+ "load_scaling": 1.0,
+ "car_charging_energy_scale": 1.0,
+ }.get(key, default)
+
+ # Test 1: Basic entity publishing with predictions
+ print(" Testing _publish_entity:")
+ mock_base = MockBase()
+
+ component = LoadMLComponent(mock_base, load_ml_enable=True)
+
+ # Mock dashboard_item to capture calls
+ def mock_dashboard_item(entity_id, state, attributes, app):
+ mock_base.dashboard_calls.append({"entity_id": entity_id, "state": state, "attributes": attributes, "app": app})
+
+ component.dashboard_item = mock_dashboard_item
+
+ # Set up test data
+ component.load_minutes_now = 10.5 # Current load today
+ component.current_predictions = {
+ 0: 0.1, # Now (delta from "before predictions" to now = 0.1)
+ 5: 0.2, # 5 minutes from now
+ 60: 1.3, # 1 hour from now (load_today_h1)
+ 480: 9.7, # 8 hours from now (load_today_h8)
+ 1440: 28.9, # 24 hours from now
+ }
+
+ # Set up predictor state
+ component.predictor.validation_mae = 0.5
+ component.predictor.get_model_age_hours = MagicMock(return_value=2.0) # Mock model age calculation
+ component.last_train_time = datetime(2026, 1, 1, 10, 0, 0, tzinfo=timezone.utc)
+ component.load_data_age_days = 7.0
+ component.model_status = "active"
+ component.predictor.epochs_trained = 50
+
+ # Call _publish_entity
+ component._publish_entity()
+
+ # Verify dashboard_item was called (now twice - for main entity and accuracy entity)
+ assert len(mock_base.dashboard_calls) == 2, "dashboard_item should be called twice"
+
+ call = mock_base.dashboard_calls[0]
+ call2 = mock_base.dashboard_calls[1]
+
+ # Verify entity_id
+ assert call["entity_id"] == "sensor.predbat_load_ml_forecast", f"Expected sensor.predbat_load_ml_forecast, got {call['entity_id']}"
+ assert call2["entity_id"] == "sensor.predbat_load_ml_stats", f"Expected sensor.predbat_load_ml_stats, got {call2['entity_id']}"
+ # Verify state (max prediction value)
+ assert call2["state"] == 28.9, f"Expected state 28.9, got {call2['state']}"
+
+ # Verify app
+ assert call2["app"] == "load_ml", f"Expected app 'load_ml', got {call2['app']}"
+
+ # Verify attributes
+ attrs = call["attributes"]
+ attrs2 = call2["attributes"]
+
+ # Check results format
+ assert "results" in attrs, "results should be in attributes"
+ results = attrs["results"]
+ assert isinstance(results, dict), "results should be a dict"
+
+ # Verify results are timestamp-formatted and include load_minutes_now offset
+ # predictions are relative to now, so minute 60 = 1 hour from now = 13:00
+ expected_timestamp_60 = (mock_base.midnight_utc + timedelta(minutes=60 + 720)).strftime(TIME_FORMAT)
+ assert expected_timestamp_60 in results, f"Expected timestamp {expected_timestamp_60} in results"
+ # Value should be prediction (1.3) + load_minutes_now (10.5) = 11.8
+ assert abs(results[expected_timestamp_60] - 11.8) < 0.01, f"Expected value 11.8 at {expected_timestamp_60}, got {results[expected_timestamp_60]}"
+
+ # Check load_today (current load)
+ assert "load_today" in attrs2, "load_today should be in attributes"
+ assert attrs2["load_today"] == 10.5, f"Expected load_today 10.5, got {attrs2['load_today']}"
+
+ # Check load_today_h1 (1 hour ahead)
+ assert "load_today_h1" in attrs2, "load_today_h1 should be in attributes"
+ assert abs(attrs2["load_today_h1"] - 11.8) < 0.01, f"Expected load_today_h1 11.8, got {attrs2['load_today_h1']}"
+
+ # Check load_today_h8 (8 hours ahead)
+ assert "load_today_h8" in attrs2, "load_today_h8 should be in attributes"
+ assert abs(attrs2["load_today_h8"] - 20.2) < 0.01, f"Expected load_today_h8 20.2 (9.7+10.5), got {attrs2['load_today_h8']}"
+ # Check MAE
+ assert "mae_kwh" in attrs2, "mae_kwh should be in attributes"
+ assert attrs2["mae_kwh"] == 0.5, f"Expected mae_kwh 0.5, got {attrs2['mae_kwh']}"
+
+ # Check last_trained
+ assert "last_trained" in attrs2, "last_trained should be in attributes"
+ assert attrs2["last_trained"] == "2026-01-01T10:00:00+00:00", f"Expected last_trained 2026-01-01T10:00:00+00:00, got {attrs2['last_trained']}"
+
+ # Check model_age_hours (12:00 - 10:00 = 2 hours)
+ assert "model_age_hours" in attrs2, "model_age_hours should be in attributes"
+ assert attrs2["model_age_hours"] == 2.0, f"Expected model_age_hours 2.0, got {attrs2['model_age_hours']}"
+
+ # Check training_days
+ assert "training_days" in attrs2, "training_days should be in attributes"
+ assert attrs2["training_days"] == 7.0, f"Expected training_days 7.0, got {attrs2['training_days']}"
+
+ # Check status
+ assert "status" in attrs2, "status should be in attributes"
+ assert attrs2["status"] == "active", f"Expected status 'active', got {attrs2['status']}"
+
+ # Check model_version
+ assert "model_version" in attrs2, "model_version should be in attributes"
+ from load_predictor import MODEL_VERSION
+
+ assert attrs2["model_version"] == MODEL_VERSION, f"Expected model_version {MODEL_VERSION}, got {attrs2['model_version']}"
+
+ # Check epochs_trained
+ assert "epochs_trained" in attrs2, "epochs_trained should be in attributes"
+ assert attrs2["epochs_trained"] == 50, f"Expected epochs_trained 50, got {attrs2['epochs_trained']}"
+
+ # Check power_today values (instantaneous power in kW)
+ assert "power_today_now" in attrs2, "power_today_now should be in attributes"
+ assert "power_today_h1" in attrs2, "power_today_h1 should be in attributes"
+ assert "power_today_h8" in attrs2, "power_today_h8 should be in attributes"
+
+ # power_today_now: delta from start (prev_value=0) to minute 0 (0.1 kWh) / 5 min * 60 = 1.2 kW
+ expected_power_now = (0.1 - 0.0) / 5 * 60
+ assert abs(attrs2["power_today_now"] - expected_power_now) < 0.01, f"Expected power_today_now {expected_power_now:.2f}, got {attrs2['power_today_now']}"
+
+ # power_today_h1: delta from minute 55 to minute 60
+ # We need to interpolate - predictions are sparse, so the actual delta will depend on what's in the dict
+ # For minute 60, prev_value in the loop would be the value at minute 55 (or closest)
+ # Since we don't have minute 55 in our test data, prev_value when reaching minute 60 will be from minute 5
+ # So delta = (1.3 - 0.2) / 5 * 60 = 13.2 kW
+ expected_power_h1 = (1.3 - 0.2) / 5 * 60
+ assert abs(attrs2["power_today_h1"] - expected_power_h1) < 0.01, f"Expected power_today_h1 {expected_power_h1:.2f}, got {attrs2['power_today_h1']}"
+
+ # power_today_h8: delta from minute 475 to minute 480
+ # prev_value would be from minute 60, so delta = (9.7 - 1.3) / 5 * 60 = 100.8 kW
+ expected_power_h8 = (9.7 - 1.3) / 5 * 60
+ assert abs(attrs2["power_today_h8"] - expected_power_h8) < 0.01, f"Expected power_today_h8 {expected_power_h8:.2f}, got {attrs2['power_today_h8']}"
+
+ # Check friendly_name
+ assert attrs["friendly_name"] == "ML Load Forecast", "friendly_name should be 'ML Load Forecast'"
+ assert attrs2["friendly_name"] == "ML Load Stats", "friendly_name should be 'ML Load Stats'"
+ # Check state_class
+ assert attrs2["state_class"] == "measurement", "state_class should be 'measurement'"
+
+ # Check unit_of_measurement
+ assert attrs2["unit_of_measurement"] == "kWh", "unit_of_measurement should be 'kWh'"
+
+ # Check icon
+ assert attrs["icon"] == "mdi:chart-line", "icon should be 'mdi:chart-line'"
+ assert attrs2["icon"] == "mdi:chart-line", "icon should be 'mdi:chart-line'"
+
+ print(" ✓ Entity published with correct attributes")
+
+ # Test 2: Empty predictions
+ mock_base.dashboard_calls = []
+ component.current_predictions = {}
+ component._publish_entity()
+
+ assert len(mock_base.dashboard_calls) == 2, "dashboard_item should be called even with empty predictions"
+ call = mock_base.dashboard_calls[0]
+ call2 = mock_base.dashboard_calls[1]
+ assert call2["state"] == 0, "State should be 0 with empty predictions"
+ assert call["attributes"]["results"] == {}, "results should be empty dict"
+
+ print(" ✓ Empty predictions handled correctly")
+
+ print(" All _publish_entity tests passed!")
diff --git a/apps/predbat/tests/test_minute_data_import_export.py b/apps/predbat/tests/test_minute_data_import_export.py
index 5850e9ff1..abe65cdfa 100644
--- a/apps/predbat/tests/test_minute_data_import_export.py
+++ b/apps/predbat/tests/test_minute_data_import_export.py
@@ -66,7 +66,7 @@ def mock_get_history_wrapper(entity_id, days):
# Test with array containing real entities and '0' fixed value
entity_ids = ["sensor.import_1", "0", "sensor.import_2"]
- result = my_predbat.minute_data_import_export(now_utc=now, key=entity_ids[0], scale=1.0, required_unit="kWh") # Pass first entity directly
+ result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key=entity_ids[0], scale=1.0, required_unit="kWh") # Pass first entity directly
# Verify we got data from entity1
if len(result) == 0:
@@ -76,7 +76,7 @@ def mock_get_history_wrapper(entity_id, days):
# Now test with the config approach using an array
my_predbat.args["import_today_test"] = entity_ids
- result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh")
+ result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh")
# Verify data was accumulated from both real entities
if len(result) == 0:
@@ -99,7 +99,7 @@ def mock_get_history_wrapper(entity_id, days):
my_predbat.args["import_today_test2"] = ["0", "1", "5"]
- result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test2", scale=1.0, required_unit="kWh")
+ result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test2", scale=1.0, required_unit="kWh")
if len(result) != 0:
print("ERROR: Test 2 failed - should return empty dict for fixed values only, got {} entries".format(len(result)))
@@ -110,7 +110,7 @@ def mock_get_history_wrapper(entity_id, days):
my_predbat.args["import_today_test3"] = [None, "", "sensor.import_1"]
- result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test3", scale=1.0, required_unit="kWh")
+ result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test3", scale=1.0, required_unit="kWh")
# Should only get data from sensor.import_1
if len(result) == 0:
@@ -120,9 +120,9 @@ def mock_get_history_wrapper(entity_id, days):
# Test 4: Verify scaling works with accumulated data
print("Test 4: Scaling with accumulated data")
- result_scaled = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test", scale=2.0, required_unit="kWh")
+ result_scaled = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test", scale=2.0, required_unit="kWh")
- result_unscaled = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh")
+ result_unscaled = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test", scale=1.0, required_unit="kWh")
if 0 in result_scaled and 0 in result_unscaled:
expected_scaled = result_unscaled[0] * 2.0
@@ -136,7 +136,7 @@ def mock_get_history_wrapper(entity_id, days):
# Test 5: Single entity passed directly (not from config)
print("Test 5: Single entity passed directly")
- result = my_predbat.minute_data_import_export(now_utc=now, key="sensor.import_1", scale=1.0, required_unit="kWh")
+ result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="sensor.import_1", scale=1.0, required_unit="kWh")
if len(result) == 0:
print("ERROR: Test 5 failed - no data returned for direct entity")
@@ -147,7 +147,7 @@ def mock_get_history_wrapper(entity_id, days):
my_predbat.args["import_today_test6"] = ["sensor.nonexistent", "sensor.import_1"]
- result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test6", scale=1.0, required_unit="kWh")
+ result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test6", scale=1.0, required_unit="kWh")
# Should still get data from sensor.import_1
if len(result) == 0:
@@ -159,7 +159,7 @@ def mock_get_history_wrapper(entity_id, days):
my_predbat.args["import_today_test7"] = "sensor.import_1"
- result = my_predbat.minute_data_import_export(now_utc=now, key="import_today_test7", scale=1.0, required_unit="kWh")
+ result = my_predbat.minute_data_import_export(max_days_previous=2, now_utc=now, key="import_today_test7", scale=1.0, required_unit="kWh")
if len(result) == 0:
print("ERROR: Test 7 failed - no data returned for single string entity")
diff --git a/apps/predbat/tests/test_temperature.py b/apps/predbat/tests/test_temperature.py
new file mode 100644
index 000000000..aa26b3ab4
--- /dev/null
+++ b/apps/predbat/tests/test_temperature.py
@@ -0,0 +1,419 @@
+# -----------------------------------------------------------------------------
+# Predbat Home Battery System
+# Copyright Trefor Southwell 2025 - All Rights Reserved
+# This application maybe used for personal use only and not for commercial use
+# -----------------------------------------------------------------------------
+# fmt: off
+# pylint: disable=consider-using-f-string
+# pylint: disable=line-too-long
+# pylint: disable=attribute-defined-outside-init
+
+"""
+Temperature API Component Tests
+
+Comprehensive test suite for the External Temperature API component.
+Tests all major functionality including:
+- Initialization and configuration with zone.home fallback
+- URL placeholder replacement for latitude/longitude
+- API data fetching with retry logic and error handling
+- Timezone offset conversion (positive and negative)
+- Sensor creation with current temperature and forecast data
+- Cache persistence on API failures
+- HA timestamp format conversion
+"""
+
+from temperature import TemperatureAPI
+from datetime import datetime, timezone
+
+
+class MockTemperatureAPI(TemperatureAPI):
+ """Mock TemperatureAPI class for testing without ComponentBase dependencies"""
+
+ def __init__(self, temperature_latitude, temperature_longitude, temperature_url):
+ # Don't call parent __init__ to avoid ComponentBase
+ self.last_updated_timestamp = None
+ self.failures_total = 0
+ self.dashboard_items = {}
+ self.log_messages = []
+ self.prefix = "predbat"
+ self._last_updated_time = None
+ self.state_storage = {}
+ self.initialize(
+ temperature_enable=True,
+ temperature_latitude=temperature_latitude,
+ temperature_longitude=temperature_longitude,
+ temperature_url=temperature_url
+ )
+
+ def log(self, message):
+ self.log_messages.append(message)
+
+ def dashboard_item(self, entity_id, state, attributes, app=None):
+ self.dashboard_items[entity_id] = {"state": state, "attributes": attributes, "app": app}
+
+ def update_success_timestamp(self):
+ self._last_updated_time = datetime.now(timezone.utc)
+
+ def last_updated_time(self):
+ return self._last_updated_time
+
+ def get_state_wrapper(self, entity_id, default=None, attribute=None):
+ """Mock get_state_wrapper"""
+ if entity_id in self.state_storage:
+ if attribute:
+ return self.state_storage[entity_id].get("attributes", {}).get(attribute, default)
+ return self.state_storage[entity_id].get("state", default)
+ return default
+
+ def set_state(self, entity_id, state, attributes=None):
+ """Mock set_state"""
+ self.state_storage[entity_id] = {"state": state, "attributes": attributes or {}}
+
+
+def _test_temperature_initialization(my_predbat):
+ """Test TemperatureAPI initialization with various configurations"""
+ print(" Testing TemperatureAPI initialization...")
+
+ # Test with explicit coordinates
+ temp_component = MockTemperatureAPI(
+ temperature_latitude=51.5074,
+ temperature_longitude=-0.1278,
+ temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m"
+ )
+
+ if temp_component.temperature_latitude != 51.5074:
+ print(" ERROR: Incorrect latitude: {}".format(temp_component.temperature_latitude))
+ return 1
+
+ if temp_component.temperature_longitude != -0.1278:
+ print(" ERROR: Incorrect longitude: {}".format(temp_component.temperature_longitude))
+ return 1
+
+ print(" PASS: Initialization with explicit coordinates")
+ return 0
+
+
+def _test_temperature_zone_home_fallback(my_predbat):
+ """Test zone.home coordinate fallback"""
+ print(" Testing zone.home coordinate fallback...")
+
+ # Initialize without explicit coordinates
+ temp_component = MockTemperatureAPI(
+ temperature_latitude=None,
+ temperature_longitude=None,
+ temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m"
+ )
+
+ # Set zone.home with coordinates
+ temp_component.set_state("zone.home", state="home", attributes={"latitude": 52.52, "longitude": 13.41})
+
+ # Test coordinate resolution
+ lat, lon = temp_component.get_coordinates()
+
+ if lat != 52.52 or lon != 13.41:
+ print(" ERROR: Failed to fallback to zone.home coordinates: lat={}, lon={}".format(lat, lon))
+ return 1
+
+ print(" PASS: zone.home fallback works correctly")
+ return 0
+
+
+def _test_temperature_url_placeholder_replacement(my_predbat):
+ """Test URL placeholder replacement with coordinates"""
+ print(" Testing URL placeholder replacement...")
+
+ temp_component = MockTemperatureAPI(
+ temperature_latitude=51.5074,
+ temperature_longitude=-0.1278,
+ temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m"
+ )
+
+ url = temp_component.build_api_url(51.5074, -0.1278)
+ expected_url = "https://api.open-meteo.com/v1/forecast?latitude=51.5074&longitude=-0.1278&hourly=temperature_2m¤t=temperature_2m"
+ if url != expected_url:
+ print(" ERROR: URL placeholder replacement failed")
+ print(" Expected: {}".format(expected_url))
+ print(" Got: {}".format(url))
+ return 1
+
+ print(" PASS: URL placeholders replaced correctly")
+ return 0
+
+
+def _test_temperature_timezone_offset_conversion(my_predbat):
+ """Test timezone offset conversion from seconds to ±HH:MM format"""
+ print(" Testing timezone offset conversion...")
+
+ my_predbat.args["temperature_latitude"] = 51.5074
+ my_predbat.args["temperature_longitude"] = -0.1278
+
+ temp_component = MockTemperatureAPI(
+ temperature_latitude=51.5074,
+ temperature_longitude=-0.1278,
+ temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m"
+ )
+
+ # Test UTC (0 offset)
+ offset_str = temp_component.convert_timezone_offset(0)
+ if offset_str != "+00:00":
+ print(" ERROR: Failed to convert 0 seconds to +00:00, got: {}".format(offset_str))
+ return 1
+
+ # Test positive offset (CET)
+ offset_str = temp_component.convert_timezone_offset(3600)
+ if offset_str != "+01:00":
+ print(" ERROR: Failed to convert 3600 seconds to +01:00, got: {}".format(offset_str))
+ return 1
+
+ # Test negative offset (EST)
+ offset_str = temp_component.convert_timezone_offset(-18000)
+ if offset_str != "-05:00":
+ print(" ERROR: Failed to convert -18000 seconds to -05:00, got: {}".format(offset_str))
+ return 1
+
+ # Test offset with minutes (IST)
+ offset_str = temp_component.convert_timezone_offset(19800) # +05:30
+ if offset_str != "+05:30":
+ print(" ERROR: Failed to convert 19800 seconds to +05:30, got: {}".format(offset_str))
+ return 1
+
+ print(" PASS: Timezone offset conversion works correctly")
+ return 0
+
+
+def _test_temperature_sensor_creation(my_predbat):
+ """Test sensor creation with current temperature and forecast"""
+ print(" Testing sensor creation with temperature data...")
+
+ temp_component = MockTemperatureAPI(
+ temperature_latitude=51.5074,
+ temperature_longitude=-0.1278,
+ temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m"
+ )
+
+ # Mock API response data
+ mock_data = {
+ "latitude": 51.5,
+ "longitude": -0.12,
+ "utc_offset_seconds": 0,
+ "timezone": "GMT",
+ "current": {
+ "time": "2026-02-07T10:30",
+ "temperature_2m": 9.5
+ },
+ "hourly": {
+ "time": [
+ "2026-02-07T00:00",
+ "2026-02-07T01:00",
+ "2026-02-07T02:00",
+ "2026-02-07T03:00"
+ ],
+ "temperature_2m": [8.2, 8.5, 8.8, 9.1]
+ }
+ }
+
+ # Set the data and publish sensor
+ temp_component.temperature_data = mock_data
+ temp_component.last_updated_timestamp = datetime.now()
+ temp_component.publish_temperature_sensor()
+
+ # Verify sensor was created
+ sensor_entity = "sensor.predbat_temperature"
+ if sensor_entity not in temp_component.dashboard_items:
+ print(" ERROR: Temperature sensor was not created")
+ return 1
+
+ sensor_state = temp_component.dashboard_items[sensor_entity]["state"]
+ if sensor_state != 9.5:
+ print(" ERROR: Incorrect sensor state: {} (expected 9.5)".format(sensor_state))
+ return 1
+
+ # Verify attributes
+ sensor_attrs = temp_component.dashboard_items[sensor_entity]["attributes"]
+ results = sensor_attrs.get("results")
+ if results is None:
+ print(" ERROR: results attribute not set")
+ return 1
+
+ # Check forecast has correct HA timestamp format
+ expected_keys = [
+ "2026-02-07T00:00:00+00:00",
+ "2026-02-07T01:00:00+00:00",
+ "2026-02-07T02:00:00+00:00",
+ "2026-02-07T03:00:00+00:00"
+ ]
+
+ for key in expected_keys:
+ if key not in results:
+ print(" ERROR: Missing results key: {}".format(key))
+ print(" Available keys: {}".format(list(results.keys())))
+ return 1
+
+ # Verify temperature values
+ if results["2026-02-07T00:00:00+00:00"] != 8.2:
+ print(" ERROR: Incorrect results value for first hour")
+ return 1
+
+ print(" PASS: Sensor created with correct state and forecast")
+ return 0
+
+
+def _test_temperature_cache_persistence(my_predbat):
+ """Test that cached data persists on API failure"""
+ print(" Testing cache persistence on API failure...")
+
+ my_predbat.args["temperature_latitude"] = 51.5074
+ my_predbat.args["temperature_longitude"] = -0.1278
+ temp_component = MockTemperatureAPI(
+ temperature_latitude=51.5074,
+ temperature_longitude=-0.1278,
+ temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m"
+ )
+
+ # Set initial cached data
+ initial_data = {
+ "utc_offset_seconds": 0,
+ "current": {"temperature_2m": 10.0},
+ "hourly": {"time": ["2026-02-07T00:00"], "temperature_2m": [9.5]}
+ }
+
+ temp_component.temperature_data = initial_data
+ temp_component.last_updated_timestamp = datetime(2026, 2, 7, 10, 0)
+ initial_time = temp_component.last_updated_timestamp
+
+ # Publish sensor with initial data
+ temp_component.publish_temperature_sensor()
+
+ # Verify initial sensor state
+ sensor_entity = "sensor.predbat_temperature"
+ if sensor_entity not in temp_component.dashboard_items:
+ print(" ERROR: Sensor not created")
+ return 1
+
+ sensor_state = temp_component.dashboard_items[sensor_entity]["state"]
+ if sensor_state != 10.0:
+ print(" ERROR: Initial sensor state incorrect: {}".format(sensor_state))
+ return 1
+
+ # Simulate API failure by keeping old data
+ temp_component.temperature_data = initial_data # Keep old data
+ temp_component.publish_temperature_sensor()
+
+ # Verify sensor still has old data (10.0)
+ sensor_state = temp_component.dashboard_items[sensor_entity]["state"]
+ if sensor_state != 10.0:
+ print(f" ERROR: Sensor state changed when it shouldn't - got {sensor_state}")
+ return 1
+
+ # Verify last_updated timestamp hasn't changed
+ if temp_component.last_updated_timestamp != initial_time:
+ print(" ERROR: last_updated timestamp changed when it shouldn't")
+ return 1
+
+ print(" PASS: Cached data persists on API failure")
+ return 0
+
+
+def _test_temperature_negative_timezone_offset(my_predbat):
+ """Test negative timezone offset handling (e.g., US timezones)"""
+ print(" Testing negative timezone offset handling...")
+
+ my_predbat.args["temperature_latitude"] = 40.7128
+ my_predbat.args["temperature_longitude"] = -74.0060
+ temp_component = MockTemperatureAPI(
+ temperature_latitude=40.7128,
+ temperature_longitude=-74.0060,
+ temperature_url="https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m¤t=temperature_2m"
+ )
+
+ # Mock API response with negative timezone offset (EST)
+ mock_data = {
+ "utc_offset_seconds": -18000, # -05:00
+ "current": {"temperature_2m": 5.5},
+ "hourly": {
+ "time": ["2026-02-07T00:00"],
+ "temperature_2m": [4.8]
+ }
+ }
+
+ temp_component.temperature_data = mock_data
+ temp_component.last_updated_timestamp = datetime.now()
+ temp_component.publish_temperature_sensor()
+
+ # Verify sensor attributes have correct timezone
+ sensor_entity = "sensor.predbat_temperature"
+ if sensor_entity not in temp_component.dashboard_items:
+ print(" ERROR: Sensor not created")
+ return 1
+
+ forecast = temp_component.dashboard_items[sensor_entity]["attributes"].get("results", {})
+ if not forecast:
+ print(" ERROR: Forecast not found in sensor attributes")
+ return 1
+
+ # Check for negative timezone offset in timestamp
+ expected_key = "2026-02-07T00:00:00-05:00"
+ if expected_key not in forecast:
+ print(" ERROR: Expected key {} not found in forecast".format(expected_key))
+ print(" Available keys: {}".format(list(forecast.keys())))
+ return 1
+
+ print(" PASS: Negative timezone offset handled correctly")
+ return 0
+
+
+def test_temperature(my_predbat=None):
+ """
+ Comprehensive test suite for External Temperature API.
+
+ Tests all major functionality including:
+ - Initialization and configuration
+ - zone.home coordinate fallback
+ - URL placeholder replacement
+ - Timezone offset conversion (positive and negative)
+ - Sensor creation with current temperature and forecast
+ - Cache persistence on API failures
+ - HA timestamp format conversion
+ """
+
+ # Registry of all sub-tests
+ sub_tests = [
+ ("initialization", _test_temperature_initialization, "Temperature API initialization"),
+ ("zone_home_fallback", _test_temperature_zone_home_fallback, "zone.home coordinate fallback"),
+ ("url_placeholder", _test_temperature_url_placeholder_replacement, "URL placeholder replacement"),
+ ("timezone_offset", _test_temperature_timezone_offset_conversion, "Timezone offset conversion"),
+ ("sensor_creation", _test_temperature_sensor_creation, "Sensor creation with forecast data"),
+ ("cache_persistence", _test_temperature_cache_persistence, "Cache persistence on failure"),
+ ("negative_timezone", _test_temperature_negative_timezone_offset, "Negative timezone offset handling"),
+ ]
+
+ print("\n" + "=" * 70)
+ print("EXTERNAL TEMPERATURE API TEST SUITE")
+ print("=" * 70)
+
+ failed = 0
+ passed = 0
+
+ for test_name, test_func, test_desc in sub_tests:
+ print("\n[{}] {}".format(test_name, test_desc))
+ try:
+ test_result = test_func(my_predbat)
+ if test_result:
+ failed += 1
+ print(" ❌ FAILED")
+ else:
+ passed += 1
+ print(" ✅ PASSED")
+ except Exception as e:
+ print(" ❌ EXCEPTION: {}".format(e))
+ import traceback
+ traceback.print_exc()
+ failed += 1
+
+ print("\n" + "=" * 70)
+ print("TEMPERATURE API TEST RESULTS")
+ print(" Passed: {}".format(passed))
+ print(" Failed: {}".format(failed))
+ print("=" * 70)
+
+ return failed
diff --git a/apps/predbat/unit_test.py b/apps/predbat/unit_test.py
index a005b3157..a12f9649f 100644
--- a/apps/predbat/unit_test.py
+++ b/apps/predbat/unit_test.py
@@ -96,6 +96,8 @@
from tests.test_ohme import test_ohme
from tests.test_component_base import test_component_base_all
from tests.test_solis import run_solis_tests
+from tests.test_load_ml import test_load_ml
+from tests.test_temperature import test_temperature
# Mock the components and plugin system
@@ -244,6 +246,10 @@ def main():
("component_base", test_component_base_all, "ComponentBase tests (all)", False),
# Solis Cloud API unit tests
("solis", run_solis_tests, "Solis Cloud API tests (V1/V2 time window writes, change detection)", False),
+ # ML Load Forecaster tests
+ ("load_ml", test_load_ml, "ML Load Forecaster tests (MLP, training, persistence, validation)", False),
+ # External Temperature API tests
+ ("temperature", test_temperature, "External Temperature API tests (initialization, zone.home fallback, timezone conversion, caching)", False),
("optimise_levels", run_optimise_levels_tests, "Optimise levels tests", False),
("optimise_windows", run_optimise_all_windows_tests, "Optimise all windows tests", True),
("debug_cases", run_debug_cases, "Debug case file tests", True),
diff --git a/apps/predbat/utils.py b/apps/predbat/utils.py
index 9125c2704..f2c8b718a 100644
--- a/apps/predbat/utils.py
+++ b/apps/predbat/utils.py
@@ -41,7 +41,7 @@ def get_now_from_cumulative(data, minutes_now, backwards):
return max(value, 0)
-def prune_today(data, now_utc, midnight_utc, prune=True, group=15, prune_future=False, intermediate=False):
+def prune_today(data, now_utc, midnight_utc, prune=True, group=15, prune_future=False, prune_future_days=0, prune_past_days=0, intermediate=False, offset_minutes=0):
"""
Remove data from before today
"""
@@ -54,18 +54,19 @@ def prune_today(data, now_utc, midnight_utc, prune=True, group=15, prune_future=
timekey = datetime.strptime(key, TIME_FORMAT_SECONDS)
else:
timekey = datetime.strptime(key, TIME_FORMAT)
- if last_time and (timekey - last_time).seconds < group * 60:
+ if last_time and (timekey - last_time).total_seconds() < group * 60:
continue
- if intermediate and last_time and ((timekey - last_time).seconds > group * 60):
+ if intermediate and last_time and ((timekey - last_time).total_seconds() > group * 60):
# Large gap, introduce intermediate data point
seconds_gap = int((timekey - last_time).total_seconds())
for i in range(1, seconds_gap // int(group * 60)):
- new_time = last_time + timedelta(seconds=i * group * 60)
- results[new_time.strftime(TIME_FORMAT)] = prev_value
- if not prune or (timekey > midnight_utc):
- if prune_future and (timekey > now_utc):
+ new_time = last_time + timedelta(seconds=i * group * 60) + timedelta(minutes=offset_minutes)
+ results[new_time.isoformat()] = prev_value
+ if not prune or (timekey > (midnight_utc - timedelta(days=prune_past_days))):
+ if prune_future and (timekey > (now_utc + timedelta(days=prune_future_days))):
continue
- results[key] = data[key]
+ new_time = timekey + timedelta(minutes=offset_minutes)
+ results[new_time.isoformat()] = data[key]
last_time = timekey
prev_value = data[key]
return results
diff --git a/apps/predbat/web.py b/apps/predbat/web.py
index 2a9efd2ee..2ab56472e 100644
--- a/apps/predbat/web.py
+++ b/apps/predbat/web.py
@@ -57,7 +57,7 @@
get_dashboard_collapsible_js,
)
-from utils import calc_percent_limit, str2time, dp0, dp2, format_time_ago, get_override_time_from_string, history_attribute, prune_today
+from utils import calc_percent_limit, str2time, dp0, dp2, dp4, format_time_ago, get_override_time_from_string, history_attribute, prune_today
from const import TIME_FORMAT, TIME_FORMAT_DAILY, TIME_FORMAT_HA
from predbat import THIS_VERSION
from component_base import ComponentBase
@@ -142,6 +142,7 @@ async def start(self):
app.router.add_get("/internals", self.html_internals)
app.router.add_get("/api/internals", self.html_api_internals)
app.router.add_get("/api/internals/download", self.html_api_internals_download)
+ app.router.add_get("/api/status", self.html_api_get_status)
# Notify plugin system that web interface is ready
if hasattr(self.base, "plugin_system") and self.base.plugin_system:
@@ -1794,6 +1795,24 @@ async def html_api_post_state(self, request):
else:
return web.Response(content_type="application/json", text='{"result": "error"}')
+ async def html_api_get_status(self, request):
+ """
+ Get current Predbat status (calculating state and battery info)
+ """
+ try:
+ calculating = self.get_arg("active", False)
+ if self.base.update_pending:
+ calculating = True
+
+ battery_icon = self.get_battery_status_icon()
+
+ status_data = {"calculating": calculating, "battery_html": battery_icon}
+
+ return web.Response(content_type="application/json", text=json.dumps(status_data))
+ except Exception as e:
+ self.log("Error getting status: {}".format(e))
+ return web.Response(status=500, content_type="application/json", text=json.dumps({"error": str(e)}))
+
async def html_api_ping(self, request):
"""
Check if Predbat is running
@@ -2570,6 +2589,96 @@ def get_chart(self, chart):
{"name": "Forecast CL", "data": pv_today_forecastCL, "opacity": "0.3", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#e90a0a"},
]
text += self.render_chart(series_data, "kW", "Solar Forecast", now_str)
+ elif chart == "LoadML":
+ # Get historical load data for last 24 hours
+ load_today = prune_today(history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 1, required=False), attributes=True, state_key="load_today"), self.now_utc, self.midnight_utc, prune=False)
+ load_today_h1 = prune_today(
+ history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 1, required=False), attributes=True, state_key="load_today_h1"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 1
+ )
+ load_today_h8 = prune_today(
+ history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 1, required=False), attributes=True, state_key="load_today_h8"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 8
+ )
+
+ # Get ML forecast from load_forecast_ml entity results
+ load_ml_forecast = self.get_entity_results("sensor." + self.prefix + "_load_ml_forecast")
+
+ series_data = [
+ {"name": "Load (Actual)", "data": load_today, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#3291a8"},
+ {"name": "Forecast (+1h)", "data": load_today_h1, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "color": "#f5a442"},
+ {"name": "Forecast (+8h)", "data": load_today_h8, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "color": "#9b59b6"},
+ {"name": "Load (ML Forecast)", "data": load_ml_forecast, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#eb2323"},
+ ]
+ text += self.render_chart(series_data, "kWh", "ML Load Forecast", now_str)
+ elif chart == "LoadMLPower":
+ # Get historical load power
+ load_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".load_power", 7, required=False))
+ load_power = prune_today(load_power_hist, self.now_utc, self.midnight_utc, prune=False)
+
+ # Get ML predicted load energy (cumulative) and convert to power (kW)
+ load_ml_forecast_energy = self.get_entity_results("sensor." + self.prefix + "_load_ml_forecast")
+ load_ml_forecast_power = {}
+
+ power_today = prune_today(history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 7, required=False), attributes=True, state_key="power_today"), self.now_utc, self.midnight_utc, prune=False)
+ power_today_h1 = prune_today(
+ history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 7, required=False), attributes=True, state_key="power_today_h1"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 1
+ )
+ power_today_h8 = prune_today(
+ history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_load_ml_stats", 7, required=False), attributes=True, state_key="power_today_h8"), self.now_utc, self.midnight_utc, prune=False, offset_minutes=60 * 8
+ )
+
+ # Sort timestamps and calculate deltas to get energy per interval
+ if load_ml_forecast_energy:
+ from datetime import datetime
+
+ sorted_timestamps = sorted(load_ml_forecast_energy.keys())
+ prev_energy = 0
+ prev_timestamp = None
+ for timestamp in sorted_timestamps:
+ energy = load_ml_forecast_energy[timestamp]
+ energy_delta = max(energy - prev_energy, 0)
+
+ # Calculate actual interval in hours between this and previous timestamp
+ if prev_timestamp:
+ # Parse timestamps and calculate difference in hours
+ curr_dt = datetime.strptime(timestamp, TIME_FORMAT)
+ prev_dt = datetime.strptime(prev_timestamp, TIME_FORMAT)
+ interval_hours = (curr_dt - prev_dt).total_seconds() / 3600.0
+ load_ml_forecast_power[timestamp] = dp4(energy_delta / interval_hours)
+
+ prev_energy = energy
+ prev_timestamp = timestamp
+
+ # Get historical PV power
+ pv_power_hist = history_attribute(self.get_history_wrapper(self.prefix + ".pv_power", 1, required=False))
+ pv_power = prune_today(pv_power_hist, self.now_utc, self.midnight_utc, prune=False)
+
+ # Get temperature prediction data and limit to 48 hours forward
+ temperature_forecast = prune_today(self.get_entity_results("sensor." + self.prefix + "_temperature"), self.now_utc, self.midnight_utc, prune_future=True, prune_future_days=2, prune=True, prune_past_days=7)
+
+ series_data = [
+ {"name": "Load Power (Actual)", "data": load_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#3291a8", "unit": "kW"},
+ {"name": "Load Power (ML Predicted Future)", "data": load_ml_forecast_power, "opacity": "0.5", "stroke_width": "3", "chart_type": "area", "stroke_curve": "smooth", "color": "#eb2323", "unit": "kW"},
+ {"name": "Load Power (Used)", "data": load_power_best, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"},
+ {"name": "Load Power ML History", "data": power_today, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"},
+ {"name": "Load Power ML History +1h", "data": power_today_h1, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"},
+ {"name": "Load Power ML History +8h", "data": power_today_h8, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "unit": "kW"},
+ {"name": "PV Power (Actual)", "data": pv_power, "opacity": "1.0", "stroke_width": "3", "stroke_curve": "smooth", "color": "#f5c43d", "unit": "kW"},
+ {"name": "PV Power (Predicted)", "data": pv_power_best, "opacity": "0.7", "stroke_width": "2", "stroke_curve": "smooth", "chart_type": "area", "color": "#ffa500", "unit": "kW"},
+ {"name": "Temperature", "data": temperature_forecast, "opacity": "1.0", "stroke_width": "2", "stroke_curve": "smooth", "color": "#ff6b6b", "unit": "°C"},
+ ]
+
+ # Configure secondary axis for temperature
+ secondary_axis = [
+ {
+ "title": "°C",
+ "series_name": "Temperature",
+ "decimals": 1,
+ "opposite": True,
+ "labels_formatter": "return val.toFixed(1) + '°C';",
+ }
+ ]
+
+ text += self.render_chart(series_data, "kW", "ML Load & PV Power with Temperature", now_str, extra_yaxis=secondary_axis)
else:
text += "