ML Load predictor, external temperature components

.cspell/custom-dictionary-workspace.txt

@@ -7,6 +7,7 @@ AIO
 AIO's
 aiohttp
 Alertfeed
+allclose
 Anson
 apexcharts
 appdaemon
@@ -19,6 +20,11 @@ autoflake
 automations
 autopep
 autoupdate
+axvline
+axvspan
+backprop
+Backpropagate
+backpropagation
 Basepath
 Batpred
 battemperature
@@ -62,6 +68,7 @@ dayname
 daynumber
 daysymbol
 dend
+denorm
 devcontainer
 devcontainers
 dexport
@@ -91,6 +98,7 @@ energythroughput
 epod
 euids
 evse
+exog
 exportlimit
 fdpwr
 fdsoc
@@ -163,12 +171,17 @@ kvar
 kvarh
 kwargs
 kwhb
+labelcolor
 linebreak
+linestyle
+loadml
+loadmlpower
 loadspower
 localfolder
 lockstep
 logdata
 loglines
+lookback
 luxpower
 markdownlint
 matplotlib
@@ -254,6 +267,7 @@ pylint
 pyproject
 pytest
 pytz
+randn
 rarr
 recp
 Redownload
@@ -271,6 +285,7 @@ rstart
 rtype
 ruamel
 saverestore
+savez
 scalarstring
 searr
 securetoken
@@ -327,11 +342,13 @@ timekey
 timelapse
 timenow
 timeobj
+timestep
 timestr
 timezone
 tojson
 Trefor
 treforsiphone
+twinx
 unsmoothed
 unstaged
 useid
@@ -349,6 +366,7 @@ wrongsha
 xaxis
 xaxistooltip
 xlabel
+xlim
 xload
 xticks
 yaxis

apps/predbat/components.py

@@ -14,6 +14,7 @@
 from ohme import OhmeAPI
 from octopus import OctopusAPI
 from carbon import CarbonAPI
+from temperature import TemperatureAPI
 from axle import AxleAPI
 from solax import SolaxAPI
 from solis import SolisAPI
@@ -23,6 +24,7 @@
 from db_manager import DatabaseManager
 from fox import FoxAPI
 from web_mcp import PredbatMCPServer
+from load_ml_component import LoadMLComponent
 from datetime import datetime, timezone, timedelta
 import asyncio
 import os
@@ -220,6 +222,17 @@
         },
         "phase": 1,
     },
+    "temperature": {
+        "class": TemperatureAPI,
+        "name": "External Temperature API",
+        "args": {
+            "temperature_enable": {"required_true": True, "config": "temperature_enable", "default": False},
+            "temperature_latitude": {"required": False, "config": "temperature_latitude", "default": None},
+            "temperature_longitude": {"required": False, "config": "temperature_longitude", "default": None},
+            "temperature_url": {"required": False, "config": "temperature_url", "default": "https://api.open-meteo.com/v1/forecast?latitude=LATITUDE&longitude=LONGITUDE&hourly=temperature_2m&current=temperature_2m&past_days=7"},
+        },
+        "phase": 1,
+    },
     "axle": {
         "class": AxleAPI,
         "name": "Axle Energy",
@@ -265,6 +278,17 @@
         "phase": 1,
         "can_restart": True,
     },
+    "load_ml": {
+        "class": LoadMLComponent,
+        "name": "ML Load Forecaster",
+        "event_filter": "predbat_load_ml_",
+        "args": {
+            "load_ml_enable": {"required_true": True, "config": "load_ml_enable", "default": False},
+            "load_ml_source": {"required": False, "config": "load_ml_source", "default": False},
+        },
+        "phase": 1,
+        "can_restart": True,
+    },
 }
 
 

apps/predbat/config.py

@@ -2101,4 +2101,5 @@
     "forecast_solar_max_age": {"type": "float"},
     "enable_coarse_fine_levels": {"type": "boolean"},
     "load_power_fill_enable": {"type": "boolean"},
+    "load_ml_enable": {"type": "boolean"},
 }

apps/predbat/fetch.py

@@ -507,7 +507,7 @@ def get_from_incrementing(self, data, index, backwards=True):
         else:
             return max(data.get(index + 1, 0) - data.get(index, 0), 0)
 
-    def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True):
+    def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True):
         """
         Download one or more entities for import/export data
         """
@@ -529,15 +529,15 @@ def minute_data_import_export(self, now_utc, key, scale=1.0, required_unit=None,
                 continue
 
             try:
-                history = self.get_history_wrapper(entity_id=entity_id, days=self.max_days_previous)
+                history = self.get_history_wrapper(entity_id=entity_id, days=max_days_previous)
             except (ValueError, TypeError) as exc:
                 self.log("Warn: No history data found for {} : {}".format(entity_id, exc))
                 history = []
 
             if history and len(history) > 0:
                 import_today, _ = minute_data(
                     history[0],
-                    self.max_days_previous,
+                    max_days_previous,
                     now_utc,
                     "state",
                     "last_updated",
@@ -674,8 +674,15 @@ def fetch_sensor_data(self, save=True):
                 self.iboost_today = dp2(abs(self.iboost_energy_today[0] - self.iboost_energy_today[self.minutes_now]))
                 self.log("iBoost energy today from sensor reads {} kWh".format(self.iboost_today))
 
+        # Fetch ML forecast if enabled
+        load_ml_forecast = {}
+        if self.get_arg("load_ml_enable", False) and self.get_arg("load_ml_source", False):
+            load_ml_forecast = self.fetch_ml_load_forecast(self.now_utc)
+            if load_ml_forecast:
+                self.load_forecast_only = True  # Use only ML forecast for load if enabled and we have data
+
         # Fetch extra load forecast
-        self.load_forecast, self.load_forecast_array = self.fetch_extra_load_forecast(self.now_utc)
+        self.load_forecast, self.load_forecast_array = self.fetch_extra_load_forecast(self.now_utc, load_ml_forecast)
 
         # Load previous load data
         if self.get_arg("ge_cloud_data", False):
@@ -712,28 +719,28 @@ def fetch_sensor_data(self, save=True):
 
             # Load import today data
             if "import_today" in self.args:
-                self.import_today = self.minute_data_import_export(self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
+                self.import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
                 self.import_today_now = get_now_from_cumulative(self.import_today, self.minutes_now, backwards=True)
             else:
                 self.log("Warn: You have not set import_today in apps.yaml, you will have no previous import data")
 
             # Load export today data
             if "export_today" in self.args:
-                self.export_today = self.minute_data_import_export(self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
+                self.export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
                 self.export_today_now = get_now_from_cumulative(self.export_today, self.minutes_now, backwards=True)
             else:
                 self.log("Warn: You have not set export_today in apps.yaml, you will have no previous export data")
 
             # PV today data
             if "pv_today" in self.args:
-                self.pv_today = self.minute_data_import_export(self.now_utc, "pv_today", required_unit="kWh")
+                self.pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh")
                 self.pv_today_now = get_now_from_cumulative(self.pv_today, self.minutes_now, backwards=True)
             else:
                 self.log("Warn: You have not set pv_today in apps.yaml, you will have no previous PV data")
 
         # Battery temperature
         if "battery_temperature_history" in self.args:
-            self.battery_temperature_history = self.minute_data_import_export(self.now_utc, "battery_temperature_history", scale=1.0, increment=False, smoothing=False)
+            self.battery_temperature_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, "battery_temperature_history", scale=1.0, increment=False, smoothing=False)
             data = []
             for minute in range(0, 24 * 60, 5):
                 data.append({minute: self.battery_temperature_history.get(minute, 0)})
@@ -1059,8 +1066,8 @@ def fetch_sensor_data(self, save=True):
         # Fetch PV forecast if enabled, today must be enabled, other days are optional
         self.pv_forecast_minute, self.pv_forecast_minute10 = self.fetch_pv_forecast()
 
-        # Apply modal filter to historical data
         if self.load_minutes and not self.load_forecast_only:
+            # Apply modal filter to historical data
             self.previous_days_modal_filter(self.load_minutes)
             self.log("Historical days now {} weight {}".format(self.days_previous, self.days_previous_weight))
 
@@ -1203,17 +1210,17 @@ def download_ge_data(self, now_utc):
             self.log("GECloudData load_last_period from immediate sensor is {} kW".format(dp2(self.load_last_period)))
 
         if "import_today" in self.args:
-            import_today = self.minute_data_import_export(self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
+            import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
             self.import_today_now = get_now_from_cumulative(import_today, self.minutes_now, backwards=True)
 
         # Load export today data
         if "export_today" in self.args:
-            export_today = self.minute_data_import_export(self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
+            export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
             self.export_today_now = get_now_from_cumulative(export_today, self.minutes_now, backwards=True)
 
         # PV today data
         if "pv_today" in self.args:
-            pv_today = self.minute_data_import_export(self.now_utc, "pv_today", required_unit="kWh")
+            pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh")
             self.pv_today_now = get_now_from_cumulative(pv_today, self.minutes_now, backwards=True)
 
         self.log("Downloaded {} datapoints from GECloudData going back {} days".format(len(self.load_minutes), self.load_minutes_age))
@@ -1769,13 +1776,51 @@ def get_car_charging_planned(self):
                 )
             )
 
-    def fetch_extra_load_forecast(self, now_utc):
+    def fetch_ml_load_forecast(self, now_utc):
+        """
+        Fetches ML load forecast from sensor
+        and returns it as a minute_data dictionary
+        """
+        # Use ML Model for load prediction
+        load_ml_forecast = self.get_state_wrapper("sensor." + self.prefix + "_load_ml_forecast", attribute="results")
+        if load_ml_forecast:
+            self.log("Loading ML load forecast from sensor.sensor.{}_load_ml_forecast".format(self.prefix))
+            # Convert format from dict to array
+            if isinstance(load_ml_forecast, dict):
+                data_array = []
+                for key, value in load_ml_forecast.items():
+                    data_array.append({"energy": value, "last_updated": key})
+
+                # Load data
+                load_forecast, _ = minute_data(
+                    data_array,
+                    self.forecast_days + 1,
+                    self.midnight_utc,
+                    "energy",
+                    "last_updated",
+                    backwards=False,
+                    clean_increment=False,
+                    smoothing=True,
+                    divide_by=1.0,
+                    scale=self.load_scaling,
+                )
+
+                if load_forecast:
+                    self.log("Loaded the ML load forecast; now {}kWh to midnight {}kwh".format(load_forecast.get(self.minutes_now, 0), load_forecast.get(24 * 60 - PREDICT_STEP, 0)))
+                    return load_forecast
+        return {}
+
+    def fetch_extra_load_forecast(self, now_utc, ml_forecast=None):
         """
         Fetch extra load forecast, this is future load data
         """
         load_forecast_final = {}
         load_forecast_array = []
 
+        # Add ML forecast if available
+        if ml_forecast:
+            load_forecast_array.append(ml_forecast)
+
         if "load_forecast" in self.args:
             entity_ids = self.get_arg("load_forecast", indirect=False)
             if isinstance(entity_ids, str):
@@ -1855,7 +1900,7 @@ def fetch_carbon_intensity(self, entity_id):
         state = self.get_state_wrapper(entity_id=entity_id)
         if state is not None:
             try:
-                carbon_history = self.minute_data_import_export(self.now_utc, entity_id, required_unit="g/kWh", increment=False, smoothing=False)
+                carbon_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, entity_id, required_unit="g/kWh", increment=False, smoothing=False)
             except (ValueError, TypeError):
                 self.log("Warn: No carbon intensity history in sensor {}".format(entity_id))
         else:
@@ -2166,7 +2211,7 @@ def load_car_energy(self, now_utc):
         """
         self.car_charging_energy = {}
         if "car_charging_energy" in self.args:
-            self.car_charging_energy = self.minute_data_import_export(now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh")
+            self.car_charging_energy = self.minute_data_import_export(self.max_days_previous, now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh")
         else:
             self.log("Car charging hold {}, threshold {}kWh".format(self.car_charging_hold, self.car_charging_threshold * 60.0))
         return self.car_charging_energy