fix(fetch): exclude saving-session/Axle boosted minutes from automatic rate thresholds (#5050)

apps/predbat/annual.py

@@ -788,6 +788,12 @@ def reset_sample_state(predbat):
     predbat.rate_min = 0
     predbat.rate_max = 0
     predbat.rate_average = 0
+    # set_rate_thresholds() reads these alongside the rates. _apply_rates() clears them too, since
+    # _run_scenarios() installs a second tariff without coming back through here.
+    predbat.rate_import_saving_minutes = set()
+    predbat.rate_export_saving_minutes = set()
+    predbat.rate_import_pre_saving = {}
+    predbat.rate_export_pre_saving = {}
 
     predbat.load_inday_adjustment = 1.0
     predbat.load_scaling_dynamic = None
@@ -1188,6 +1194,13 @@ def _apply_rates(predbat, rate_import, rate_export):
     predbat.rate_export = rate_export
     predbat.rate_low_threshold = 0
     predbat.rate_high_threshold = 0
+    # A prior live cycle's saving-session minutes are absolute offsets into that cycle's own rate
+    # tables - stale and meaningless against this simulated tariff, and set_rate_thresholds() would
+    # otherwise exclude whatever unrelated minutes happen to sit at the same positions here.
+    predbat.rate_import_saving_minutes = set()
+    predbat.rate_export_saving_minutes = set()
+    predbat.rate_import_pre_saving = {}
+    predbat.rate_export_pre_saving = {}
 
     if predbat.rate_import:
         predbat.rate_scan(predbat.rate_import, print=False)

apps/predbat/compare.py

@@ -38,6 +38,8 @@ class Compare:
     def __init__(self, my_predbat):
         self.pb = my_predbat
         self.log = self.pb.log
+        # Live cycle's saving-minute sets and pre-saving snapshots, captured by run_all() with the base rates
+        self.live_saving_state = None
         self.config_root = self.pb.config_root
         self.dashboard_item = self.pb.dashboard_item
         self.currency_symbols = self.pb.currency_symbols
@@ -71,28 +73,44 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base, io_adjusted_ba
         # Reset rates to base
         pb.rate_import = copy.deepcopy(rate_import_base)
         pb.rate_export = copy.deepcopy(rate_export_base)
+        # Both sides start as the live cycle's rates. Each branch below that installs this tariff's own
+        # rates for a side records it here, so the checks further down don't depend on object identity
+        # (which a copying rate_replicate() or rate source would silently break)
+        import_replaced = False
+        export_replaced = False
 
         # Intelligent Octopus dispatch markers go with the import rates they came from (#5286)
-        live_import = pb.rate_import
         pb.io_adjusted = {}
 
+        # The base rates are the live cycle's, so the live saving-minute sets and pre-saving snapshots
+        # captured by run_all() still describe them (same cycle, same minute offsets). Start from those;
+        # a side whose rates this tariff replaces has them cleared below.
+        live_saving = self.live_saving_state or {}
+        pb.rate_import_saving_minutes = set(live_saving.get("import_minutes", set()))
+        pb.rate_export_saving_minutes = set(live_saving.get("export_minutes", set()))
+        pb.rate_import_pre_saving = dict(live_saving.get("import_pre_saving", {}))
+        pb.rate_export_pre_saving = dict(live_saving.get("export_pre_saving", {}))
+
         # Fetch rates from Octopus Energy API
         if "rates_import_octopus_url" in tariff:
             # Fixed URL for rate import
             import_url = pb.resolve_arg("rates_import_octopus_url", tariff["rates_import_octopus_url"], indirect=False)
             pb.rate_import = pb.download_octopus_rates(import_url)
+            import_replaced = True
         elif "metric_octopus_import" in tariff:
             # Octopus import rates
             entity_id = pb.resolve_arg("metric_octopus_import", tariff["metric_octopus_import"], indirect=False)
             if entity_id:
                 pb.rate_import = pb.fetch_octopus_rates(entity_id, adjust_key="is_intelligent_adjusted")
+                import_replaced = True
             else:
                 self.log("Warn: Compare tariff {} bad Octopus entity id {}".format(tariff.get("id", ""), entity_id))
         elif "metric_energidataservice_import" in tariff:
             # Energi Data Service import rates
             entity_id = pb.resolve_arg("metric_energidataservice_import", tariff["metric_energidataservice_import"], indirect=False)
             if entity_id:
                 pb.rate_import = pb.fetch_energidataservice_rates(entity_id, adjust_key="is_intelligent_adjusted")
+                import_replaced = True
             else:
                 self.log("Warn: Compare tariff {} bad Energidata entity id {}".format(tariff.get("id", ""), entity_id))
         elif "metric_stromligning_import_today" in tariff or "metric_stromligning_import_tomorrow" in tariff:
@@ -101,30 +119,35 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base, io_adjusted_ba
             entity_id_tomorrow = pb.resolve_arg("metric_stromligning_import_tomorrow", tariff.get("metric_stromligning_import_tomorrow"))
             if entity_id_today or entity_id_tomorrow:
                 pb.rate_import = pb.fetch_stromligning_rates(entity_id_today, entity_id_tomorrow, adjust_key="is_intelligent_adjusted")
+                import_replaced = True
             else:
                 self.log("Warn: Compare tariff {} bad Strømligning entity ids".format(tariff.get("id", "")))
         elif "rates_import" in tariff:
             pb.rate_import = pb.basic_rates(tariff["rates_import"], "rates_import", include_manual_api=False)
+            import_replaced = True
         else:
             self.log("Using existing rate import data")
-        if pb.rate_import is live_import:
+        if not import_replaced:
             pb.io_adjusted = copy.deepcopy(io_adjusted_base)
 
         if "rates_export_octopus_url" in tariff:
             # Fixed URL for rate export
             pb.rate_export = pb.download_octopus_rates(pb.resolve_arg("rates_export_octopus_url", tariff["rates_export_octopus_url"], indirect=False))
+            export_replaced = True
         elif "metric_octopus_export" in tariff:
             # Octopus export rates
             entity_id = pb.resolve_arg("metric_octopus_export", tariff["metric_octopus_export"], indirect=False)
             if entity_id:
                 pb.rate_export = pb.fetch_octopus_rates(entity_id)
+                export_replaced = True
             else:
                 self.log("Warn: Compare tariff {} bad Octopus entity id {}".format(tariff.get("id", ""), entity_id))
         elif "metric_energidataservice_export" in tariff:
             # Energi Data Service export rates
             entity_id = pb.resolve_arg("metric_energidataservice_export", tariff["metric_energidataservice_export"], indirect=False)
             if entity_id:
                 pb.rate_export = pb.fetch_energidataservice_rates(entity_id, adjust_key="is_intelligent_adjusted")
+                export_replaced = True
             else:
                 self.log("Warn: Compare tariff {} bad Energidata entity id {}".format(tariff.get("id", ""), entity_id))
         elif "metric_stromligning_export_today" in tariff or "metric_stromligning_export_tomorrow" in tariff:
@@ -133,17 +156,32 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base, io_adjusted_ba
             entity_id_tomorrow = pb.resolve_arg("metric_stromligning_export_tomorrow", tariff.get("metric_stromligning_export_tomorrow"))
             if entity_id_today or entity_id_tomorrow:
                 pb.rate_export = pb.fetch_stromligning_rates(entity_id_today, entity_id_tomorrow)
+                export_replaced = True
             else:
                 self.log("Warn: Compare tariff {} bad Strømligning entity ids".format(tariff.get("id", "")))
         elif "rates_export" in tariff:
             pb.rate_export = pb.basic_rates(tariff["rates_export"], "rates_export", include_manual_api=False)
+            export_replaced = True
         else:
             self.log("Using existing rate export data")
 
+        # A tariff that supplied its own rates for a side makes the live saving minutes for that side
+        # stale: they are offsets into the live tables, and would map unrelated minutes of the new tariff
+        # back to live "base" rates. A side left on the live rates keeps them.
+        if import_replaced:
+            pb.rate_import_saving_minutes = set()
+            pb.rate_import_pre_saving = {}
+        if export_replaced:
+            pb.rate_export_saving_minutes = set()
+            pb.rate_export_pre_saving = {}
+
         if pb.rate_import:
             pb.rate_scan(pb.rate_import, print=False)
             pb.rate_import, pb.rate_import_replicated = pb.rate_replicate(pb.rate_import, pb.io_adjusted, is_import=True)
             if "rates_import_override" in tariff:
+                # A kept pre-saving snapshot must carry the same override, as fetch_sensor_data()'s does
+                if pb.rate_import_saving_minutes and pb.rate_import_pre_saving:
+                    pb.rate_import_pre_saving = pb.override_session_rates(pb.rate_import, pb.rate_import_pre_saving, pb.rate_import_saving_minutes, tariff["rates_import_override"], "rates_import_override", {}, True, include_manual_api=False)
                 pb.rate_import = pb.basic_rates(tariff["rates_import_override"], "rates_import_override", pb.rate_import, pb.rate_import_replicated, include_manual_api=False)
             pb.rate_scan(pb.rate_import, print=True)
 
@@ -152,6 +190,8 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base, io_adjusted_ba
             pb.rate_scan_export(pb.rate_export, print=False)
             pb.rate_export, pb.rate_export_replicated = pb.rate_replicate(pb.rate_export, is_import=False)
             if "rates_export_override" in tariff:
+                if pb.rate_export_saving_minutes and pb.rate_export_pre_saving:
+                    pb.rate_export_pre_saving = pb.override_session_rates(pb.rate_export, pb.rate_export_pre_saving, pb.rate_export_saving_minutes, tariff["rates_export_override"], "rates_export_override", {}, False, include_manual_api=False)
                 pb.rate_export = pb.basic_rates(tariff["rates_export_override"], "rates_export_override", pb.rate_export, pb.rate_export_replicated, include_manual_api=False)
             pb.rate_scan_export(pb.rate_export, print=True)
 
@@ -169,10 +209,7 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base, io_adjusted_ba
         # Find charging windows
         if pb.rate_import:
             # Find charging window
-            pb.low_rates, lowest, highest = pb.rate_scan_window(pb.rate_import, 5, pb.rate_import_cost_threshold, False)
-            # Update threshold automatically
-            if pb.rate_low_threshold == 0 and highest >= pb.rate_min:
-                pb.rate_import_cost_threshold = highest
+            pb.find_low_rate_windows()
 
         # Compare to see if rates changes
         for minute in range(pb.minutes_now, pb.forecast_minutes + pb.minutes_now):
@@ -550,7 +587,7 @@ def recompute_car_charging(self, car_charging_slots):
             if my_predbat.car_charging_planned[car_n] or my_predbat.car_charging_now[car_n]:
                 self.log("Re-plan car {} for charging to {}".format(car_n, my_predbat.car_charging_limit[car_n]))
                 my_predbat.car_charging_plan_smart[car_n] = True
-                my_predbat.car_charging_slots[car_n] = my_predbat.plan_car_charging(car_n, my_predbat.low_rates)
+                my_predbat.car_charging_slots[car_n] = my_predbat.plan_car_charging(car_n, my_predbat.low_rates_tariff)
 
             if my_predbat.car_charging_planned[car_n] and my_predbat.car_charging_exclusive[car_n]:
                 break
@@ -610,6 +647,12 @@ def run_all(self, debug=False, fetch_sensor=True):
         rate_import_base = copy.deepcopy(self.pb.rate_import)
         rate_export_base = copy.deepcopy(self.pb.rate_export)
         io_adjusted_base = copy.deepcopy(self.pb.io_adjusted)
+        self.live_saving_state = {
+            "import_minutes": set(self.pb.rate_import_saving_minutes),
+            "export_minutes": set(self.pb.rate_export_saving_minutes),
+            "import_pre_saving": dict(self.pb.rate_import_pre_saving),
+            "export_pre_saving": dict(self.pb.rate_export_pre_saving),
+        }
 
         # Midnight SOC fallback for tariffs with no prior result
         soc_midnight_fallback = my_predbat.soc_kwh_history.get(my_predbat.minutes_now, my_predbat.soc_kw)