fix(fetch): exclude saving-session/Axle boosted minutes from automatic rate thresholds (#5050)

apps/predbat/annual.py

@@ -1194,6 +1194,13 @@ def _apply_rates(predbat, rate_import, rate_export):
     predbat.rate_export = rate_export
     predbat.rate_low_threshold = 0
     predbat.rate_high_threshold = 0
+    # The saving-minute sets are provenance for the live tariff's rates, captured in
+    # fetch_sensor_data() and frozen there so a later rate_replicate()/basic_rates() overwrite
+    # cannot erase it (#5050). They describe minute offsets in the rates being replaced here, so
+    # they must not outlive them: set_rate_thresholds() below would otherwise exclude whatever
+    # happens to sit at those offsets in the simulated tariff from its min/max/average scan.
+    predbat.rate_import_saving_minutes = set()
+    predbat.rate_export_saving_minutes = set()
 
     if predbat.rate_import:
         predbat.rate_scan(predbat.rate_import, print=False)

apps/predbat/compare.py

@@ -70,6 +70,14 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base):
         pb.rate_import = copy.deepcopy(rate_import_base)
         pb.rate_export = copy.deepcopy(rate_export_base)
 
+        # The saving-minute sets are provenance for the live tariff's rates, captured in
+        # fetch_sensor_data() and frozen there so a later rate_replicate()/basic_rates() overwrite
+        # cannot erase it (#5050). They describe minute offsets in the rates being replaced here, so
+        # they must not outlive them: set_rate_thresholds() below would otherwise exclude whatever
+        # happens to sit at those offsets in this tariff from its min/max/average scan.
+        pb.rate_import_saving_minutes = set()
+        pb.rate_export_saving_minutes = set()
+
         # Fetch rates from Octopus Energy API
         if "rates_import_octopus_url" in tariff:
             # Fixed URL for rate import

apps/predbat/fetch.py

@@ -917,8 +917,10 @@ def fetch_sensor_data(self, save=True):
 
         import_rates = {}
         self.rate_import_replicated = {}
+        self.rate_import_saving_minutes = set()
         export_rates = {}
         self.rate_export_replicated = {}
+        self.rate_export_saving_minutes = set()
         self.rate_slots = []
         self.io_adjusted = {}
         self.low_rates = []
@@ -1191,6 +1193,14 @@ def fetch_sensor_data(self, save=True):
             self.load_saving_slot(self.octopus_saving_slots, import_rates, export=False, rate_replicate=self.rate_import_replicated)
             self.load_free_slot(self.octopus_free_slots, import_rates, export=False, rate_replicate=self.rate_import_replicated)
             load_axle_slot(self, self.axle_sessions, import_rates, export=False, rate_replicate=self.rate_import_replicated)
+            # Snapshot which minutes a saving/free/Axle session tagged "saving" here, before
+            # basic_rates()/apply_manual_rates() below get a chance to overwrite rate_replicate
+            # with their own "increment"/"user" tag on the same minute - a supported combination
+            # (an override active during a saving session). rate_minmax_excluding_saving() reads
+            # this frozen set, not the live rate_replicate dict, precisely so that a later
+            # overwrite here can't erase the "this was a saving minute" provenance it depends on
+            # (#5052 review).
+            self.rate_import_saving_minutes = {minute for minute, tag in self.rate_import_replicated.items() if tag == "saving"}
             import_rates = self.basic_rates(self.get_arg("rates_import_override", [], indirect=False), "rates_import_override", import_rates, self.rate_import_replicated)
             import_rates = self.apply_manual_rates(import_rates, self.manual_import_rates, is_import=True, rate_replicate=self.rate_import_replicated)
             self.rate_scan(import_rates, print=True)
@@ -1213,6 +1223,8 @@ def fetch_sensor_data(self, save=True):
             if self.rate_export_max > 0:
                 self.load_saving_slot(self.octopus_saving_slots, export_rates, export=True, rate_replicate=self.rate_export_replicated)
             load_axle_slot(self, self.axle_sessions, export_rates, export=True, rate_replicate=self.rate_export_replicated)
+            # See the import block's equivalent comment above (#5052 review).
+            self.rate_export_saving_minutes = {minute for minute, tag in self.rate_export_replicated.items() if tag == "saving"}
             export_rates = self.basic_rates(self.get_arg("rates_export_override", [], indirect=False), "rates_export_override", export_rates, self.rate_export_replicated)
             export_rates = self.apply_manual_rates(export_rates, self.manual_export_rates, is_import=False, rate_replicate=self.rate_export_replicated)
             self.rate_scan_export(export_rates, print=True)
@@ -2272,6 +2284,86 @@ def rate_minmax(self, rates):
 
         return dp2(rate_min), dp2(rate_max), dp2(rate_average), rate_min_minute, rate_max_minute
 
+    def rate_minmax_excluding_saving(self, rates, saving_minutes):
+        """
+        Work out min/max/average over the forecast window, excluding only saving-session/Axle
+        minutes that are inflated relative to the tariff's own rates - not every such minute.
+
+        saving_minutes (rate_import_saving_minutes/rate_export_saving_minutes) is a frozen set of
+        minutes load_saving_slot()/load_axle_slot()/load_free_slot() tagged, captured right after
+        those calls and before basic_rates()/apply_manual_rates() run - NOT the live
+        rate_import_replicated/rate_export_replicated dict. A later override active during the
+        same session (a documented, supported combination - an export rate_increment alongside a
+        saving session, say) overwrites that dict's "saving" tag with its own "increment"/"user"
+        tag on the same minute, which would silently defeat a live-dict check here and let the
+        boosted price back into the threshold stats (Copilot review on #5052).
+
+        These minutes mark two economically opposite things: a saving-session/Axle event REWARD,
+        added on top of the tariff rate (GH#5050 - a synthetic one-off high that should not be
+        allowed to raise the automatic threshold above every genuine tariff rate), and a
+        free/discounted import session, which SUBTRACTS from or floors the rate (a genuinely
+        cheap slot the automatic threshold should be free to pick as "low rate"). Excluding every
+        such minute regardless of direction throws the cheap ones away too - with a flat
+        tariff plus one free slot, that leaves only the flat rate, rate_max == rate_min, and
+        set_rate_thresholds() takes its rate_max + 0.1 branch, which sets the threshold above
+        every rate and makes the whole ordinary-price day read as "low" (Copilot review on
+        #5052 - reproduced with exactly that scenario before fixing).
+
+        Prefer a per-minute comparison against the pre-event/base tariff where available
+        (rate_import_base/rate_export_base): for a tagged minute above its own base rate, use the
+        base rate for min/max/average. This catches small rewards on cheap slots too
+        (e.g. 3.49p + 5p = 8.49p), which can stay below the day's global max yet still contaminate
+        averages if not mapped back to their base.
+
+        When no base curve is available, fall back to the earlier two-pass heuristic: first pass
+        over untagged minutes sets a baseline max, second pass excludes tagged minutes above that
+        baseline max.
+
+        Falls back to the plain (unfiltered) min/max/average when every minute in range is a
+        saving minute - an event that covers the whole forecast window leaves no genuine tariff
+        minute to scan, and a 99999/0/0 result would make every downstream comparison in
+        set_rate_thresholds() behave as if there were no data at all, which is worse than the
+        boosted-but-real numbers.
+        """
+        baseline_max = 0
+        baseline_n = 0
+
+        for minute in range(self.minutes_now, self.forecast_minutes + self.minutes_now):
+            if minute in rates and minute not in saving_minutes:
+                baseline_max = max(baseline_max, rates[minute])
+                baseline_n += 1
+
+        if baseline_n == 0:
+            return self.rate_minmax(rates)[:3]
+
+        rate_base = None
+        if rates is self.rate_import and self.rate_import_base:
+            rate_base = self.rate_import_base
+        elif rates is self.rate_export and self.rate_export_base:
+            rate_base = self.rate_export_base
+
+        rate_min = 99999
+        rate_max = 0
+        rate_total = 0
+        rate_n = 0
+
+        for minute in range(self.minutes_now, self.forecast_minutes + self.minutes_now):
+            if minute not in rates:
+                continue
+            rate = rates[minute]
+            if minute in saving_minutes:
+                if rate_base and minute in rate_base:
+                    if rate > rate_base[minute]:
+                        rate = rate_base[minute]
+                elif rate > baseline_max:
+                    continue
+            rate_min = min(rate_min, rate)
+            rate_max = max(rate_max, rate)
+            rate_total += rate
+            rate_n += 1
+
+        return dp2(rate_min), dp2(rate_max), dp2(rate_total / rate_n)
+
     def rate_base_min_max(self, rates):
         """
         Gap-fill `rates` into a "base" import curve (replicated, but without IO-slot/saving-session/
@@ -2357,28 +2449,36 @@ def set_rate_thresholds(self):
         car_planning_on_rates = self.num_cars > 0 and not self.octopus_intelligent_charging
         car_charging_max_price = max(self.car_charging_plan_max_price[: self.num_cars]) if car_planning_on_rates else 0.0
 
+        # Threshold stats are computed over the tariff's own rates, excluding minutes a saving
+        # session / Axle VPP event boosted (GH#5050) - self.rate_min/rate_max/rate_average (and the
+        # export equivalents) include those synthetic minutes and are used elsewhere (dashboard
+        # sensors, graph scaling, plan.py pricing) where the real boosted price is exactly what is
+        # wanted, so this is a separate, narrower scan rather than a change to those.
+        rate_min, rate_max, rate_average = self.rate_minmax_excluding_saving(self.rate_import, self.rate_import_saving_minutes)
+        rate_export_min, rate_export_max, rate_export_average = self.rate_minmax_excluding_saving(self.rate_export, self.rate_export_saving_minutes)
+
         if self.rate_low_threshold > 0:
-            self.rate_import_cost_threshold = dp2(self.rate_average * self.rate_low_threshold)
+            self.rate_import_cost_threshold = dp2(rate_average * self.rate_low_threshold)
         else:
             # In automatic mode select the only rate or everything but the most expensive
-            if (self.rate_max == self.rate_min) or (self.rate_export_max > self.rate_max) or have_alerts or have_manual_soc:
-                self.rate_import_cost_threshold = self.rate_max + 0.1
+            if (rate_max == rate_min) or (rate_export_max > rate_max) or have_alerts or have_manual_soc:
+                self.rate_import_cost_threshold = rate_max + 0.1
             else:
-                self.rate_import_cost_threshold = self.rate_max - 0.5
+                self.rate_import_cost_threshold = rate_max - 0.5
 
         # When we plan car on the rates we need to include all the rates up to the max car price
         if car_planning_on_rates:
             self.rate_import_cost_threshold = max(self.rate_import_cost_threshold, car_charging_max_price + 0.1)
 
         # Compute the export rate threshold
         if self.rate_high_threshold > 0:
-            self.rate_export_cost_threshold = dp2(self.rate_export_average * self.rate_high_threshold)
+            self.rate_export_cost_threshold = dp2(rate_export_average * self.rate_high_threshold)
         else:
             # In automatic mode select the only rate or everything but the most cheapest
-            if (self.rate_export_max == self.rate_export_min) or (self.rate_export_min > self.rate_min) or have_alerts:
-                self.rate_export_cost_threshold = self.rate_export_min - 0.1
+            if (rate_export_max == rate_export_min) or (rate_export_min > rate_min) or have_alerts:
+                self.rate_export_cost_threshold = rate_export_min - 0.1
             else:
-                self.rate_export_cost_threshold = self.rate_export_min + 0.5
+                self.rate_export_cost_threshold = rate_export_min + 0.5
 
         self.log(
             "Rate thresholds (for charge/export) are import {}{} ({}{}), export {}{} ({}{})".format(

apps/predbat/predbat.py

@@ -568,8 +568,10 @@ def reset(self):
         self.carbon_yesterday = 0.0
         self.rate_import = {}
         self.rate_import_replicated = {}
+        self.rate_import_saving_minutes = set()
         self.rate_export = {}
         self.rate_export_replicated = {}
+        self.rate_export_saving_minutes = set()
         self.rate_slots = []
         self.low_rates = []
         self.high_export_rates = []