Skip to content

Commit 1d6272e

Browse files
fix(fetch): keep user overrides on session minutes in the threshold stats (#5163 review)
The cap mapped every saving-tagged minute back to its pre-session rate, including a minute the user had since overridden. A fixed 50p put over a +50p session to discourage charging read as the 25.95p day rate, so the manual threshold dropped from 20.47 to 18.46. A new apply_rate_overrides() helper applies the overrides and manual rates to the live table and, when there are saving minutes, the same to the pre-saving snapshot, so the cap takes a session minute back to "these rates without the session". That covers fixed, increment and manual overrides alike. With no saving minutes nothing reads the snapshot, so the overrides are not applied (or logged) twice. compare.py applies a tariff's own override to a kept snapshot in the same way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent f7e268e commit 1d6272e

4 files changed

Lines changed: 147 additions & 18 deletions

File tree

‎apps/predbat/compare.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,9 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base, io_adjusted_ba
180180
pb.rate_import, pb.rate_import_replicated = pb.rate_replicate(pb.rate_import, pb.io_adjusted, is_import=True)
181181
if "rates_import_override" in tariff:
182182
pb.rate_import = pb.basic_rates(tariff["rates_import_override"], "rates_import_override", pb.rate_import, pb.rate_import_replicated, include_manual_api=False)
183+
# A kept pre-saving snapshot must carry the same override, as fetch_sensor_data()'s does
184+
if pb.rate_import_saving_minutes and pb.rate_import_pre_saving:
185+
pb.rate_import_pre_saving = pb.basic_rates(tariff["rates_import_override"], "rates_import_override", pb.rate_import_pre_saving, {}, include_manual_api=False)
183186
pb.rate_scan(pb.rate_import, print=True)
184187

185188
# Replicate and scan export rates
@@ -188,6 +191,8 @@ def fetch_rates(self, tariff, rate_import_base, rate_export_base, io_adjusted_ba
188191
pb.rate_export, pb.rate_export_replicated = pb.rate_replicate(pb.rate_export, is_import=False)
189192
if "rates_export_override" in tariff:
190193
pb.rate_export = pb.basic_rates(tariff["rates_export_override"], "rates_export_override", pb.rate_export, pb.rate_export_replicated, include_manual_api=False)
194+
if pb.rate_export_saving_minutes and pb.rate_export_pre_saving:
195+
pb.rate_export_pre_saving = pb.basic_rates(tariff["rates_export_override"], "rates_export_override", pb.rate_export_pre_saving, {}, include_manual_api=False)
191196
pb.rate_scan_export(pb.rate_export, print=True)
192197

193198
# Set rate thresholds

‎apps/predbat/fetch.py‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,7 +1251,8 @@ def fetch_sensor_data(self, save=True):
12511251
for car_n in range(self.num_cars):
12521252
import_rates = self.rate_add_io_slots(car_n, import_rates, self.octopus_slots[car_n])
12531253
# Snapshot the tariff as it stands AFTER the IOG/SmartFlex dispatch overlay but BEFORE any
1254-
# saving/free/Axle session or override distorts it. rate_import_base is built further up
1254+
# saving/free/Axle session distorts it (the overrides below are applied to it too).
1255+
# rate_import_base is built further up
12551256
# (ahead of rate_add_io_slots), so capping a saving minute against that would discard a
12561257
# legitimate IOG discount wherever a session and a dispatch slot overlap, and the automatic
12571258
# threshold scan could then miss a genuinely cheap slot (#5163 review).
@@ -1266,8 +1267,9 @@ def fetch_sensor_data(self, save=True):
12661267
# not the live rate_import_replicated dict, so a later overwrite here can't erase the
12671268
# "this was a saving minute" provenance it depends on (GH#5050, #5052 review).
12681269
self.rate_import_saving_minutes = {minute for minute, tag in self.rate_import_replicated.items() if tag == "saving"}
1269-
import_rates = self.basic_rates(self.get_arg("rates_import_override", [], indirect=False), "rates_import_override", import_rates, self.rate_import_replicated)
1270-
import_rates = self.apply_manual_rates(import_rates, self.manual_import_rates, is_import=True, rate_replicate=self.rate_import_replicated)
1270+
import_rates, self.rate_import_pre_saving = self.apply_rate_overrides(
1271+
import_rates, self.rate_import_pre_saving, self.rate_import_saving_minutes, self.get_arg("rates_import_override", [], indirect=False), "rates_import_override", self.manual_import_rates, True, self.rate_import_replicated
1272+
)
12711273
self.rate_scan(import_rates, print=True)
12721274
else:
12731275
self.rate_import_no_io = {}
@@ -1294,8 +1296,9 @@ def fetch_sensor_data(self, save=True):
12941296
load_axle_slot(self, self.axle_sessions, export_rates, export=True, rate_replicate=self.rate_export_replicated)
12951297
# See the import block's equivalent comment above (GH#5050, #5052 review).
12961298
self.rate_export_saving_minutes = {minute for minute, tag in self.rate_export_replicated.items() if tag == "saving"}
1297-
export_rates = self.basic_rates(self.get_arg("rates_export_override", [], indirect=False), "rates_export_override", export_rates, self.rate_export_replicated)
1298-
export_rates = self.apply_manual_rates(export_rates, self.manual_export_rates, is_import=False, rate_replicate=self.rate_export_replicated)
1299+
export_rates, self.rate_export_pre_saving = self.apply_rate_overrides(
1300+
export_rates, self.rate_export_pre_saving, self.rate_export_saving_minutes, self.get_arg("rates_export_override", [], indirect=False), "rates_export_override", self.manual_export_rates, False, self.rate_export_replicated
1301+
)
12991302
self.rate_scan_export(export_rates, print=True)
13001303
else:
13011304
self.log("Warning: No export rate data provided")
@@ -2134,6 +2137,26 @@ def find_charge_window(self, rates, minute, threshold_rate, find_high, alt_rates
21342137
rate_low_average = dp2(rate_low_average / rate_low_count)
21352138
return rate_low_start, rate_low_end, rate_low_average
21362139

2140+
def apply_rate_overrides(self, rates, pre_saving, saving_minutes, override_items, rtype, manual_items, is_import, rate_replicate):
2141+
"""
2142+
Apply the configured rate overrides and then the manual rates on top of rates, tagging each
2143+
overridden minute in rate_replicate. Returns (rates, pre_saving).
2144+
2145+
When there are saving minutes the same overrides also go on pre_saving - the rates without
2146+
any saving/free/Axle session - so it stays "these rates without the session". Otherwise
2147+
rate_minmax_excluding_saving() would cap an overridden session minute (a fixed 50p to
2148+
discourage charging in the session, say) back to the bare tariff and the override would
2149+
drop out of the threshold stats (#5163 review). With no saving minutes nothing reads
2150+
pre_saving, so it is left alone rather than logging every override twice each cycle.
2151+
"""
2152+
rates = self.basic_rates(override_items, rtype, rates, rate_replicate)
2153+
rates = self.apply_manual_rates(rates, manual_items, is_import=is_import, rate_replicate=rate_replicate)
2154+
if saving_minutes and pre_saving:
2155+
self.log("Applying {} to the rates without the saving session too, for the rate thresholds".format(rtype))
2156+
pre_saving = self.basic_rates(override_items, rtype, pre_saving, {})
2157+
pre_saving = self.apply_manual_rates(pre_saving, manual_items, is_import=is_import, rate_replicate={})
2158+
return rates, pre_saving
2159+
21372160
def apply_manual_rates(self, rates, manual_items, is_import=True, rate_replicate=None):
21382161
"""
21392162
Apply manual rates to the rates dictionary
@@ -2433,14 +2456,17 @@ def rate_minmax_excluding_saving(self, rates, saving_minutes, rate_base):
24332456
same session (a documented, supported combination) can overwrite on the very same minute,
24342457
losing the "this was a saving minute" provenance a live-dict check would need (#5052 review).
24352458
2436-
rate_base (rate_import_pre_saving/rate_export_pre_saving) is the tariff curve as it stands
2437-
before any saving/free/Axle/override distortion runs - already computed on every real fetch
2438-
cycle whenever `rates` is non-empty, so this is a straight lookup rather than a second pass
2439-
over `rates` to reconstruct it. On the import side that snapshot is taken AFTER the IOG
2440-
dispatch overlay, not from rate_import_base which predates rate_add_io_slots(): capping a
2441-
saving minute against the pre-IOG curve would discard a legitimate dispatch discount wherever
2442-
a session and a dispatch slot overlap (#5163 review). Export has no IOG overlay, so its
2443-
snapshot is simply rate_export_base.
2459+
rate_base (rate_import_pre_saving/rate_export_pre_saving) is `rates` as it would be with no
2460+
saving/free/Axle session: snapshotted just before the session loaders run, then given the same
2461+
overrides and manual rates as `rates` by apply_rate_overrides() - already computed on every real fetch cycle whenever
2462+
`rates` is non-empty, so this is a straight lookup rather than a second pass over `rates` to
2463+
reconstruct it. Carrying the overrides matters: a user who overrides a session minute (a fixed
2464+
50p to discourage charging, or an increment on top of the event) must see that override in the
2465+
threshold stats, not have it capped back to the bare tariff (#5163 review). On the import side
2466+
the snapshot is taken AFTER the IOG dispatch overlay, not from rate_import_base which predates
2467+
rate_add_io_slots(): capping a saving minute against the pre-IOG curve would discard a
2468+
legitimate dispatch discount wherever a session and a dispatch slot overlap (#5163 review).
2469+
Export has no IOG overlay, so its snapshot starts from rate_export_base.
24442470
24452471
Only ever caps a tagged minute DOWN to its base rate, never raises it - the "saving" tag
24462472
covers two economically opposite things: an event REWARD added on top of the tariff rate (a

‎apps/predbat/tests/test_set_rate_thresholds.py‎

Lines changed: 101 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,52 @@ def test_saving_minute_capped_against_post_io_rates(my_predbat):
415415
return failed
416416

417417

418+
def test_saving_minute_keeps_user_override(my_predbat):
419+
"""An override on a session minute must reach the threshold stats, not be capped back to the tariff.
420+
421+
apply_rate_overrides(), which fetch_sensor_data() runs the overrides through, gives
422+
rate_import_pre_saving the same overrides as rate_import, so the cap takes a session minute back to
423+
"this rate without the session" rather than to the bare tariff. Before, a fixed 50p the user put
424+
over a +50p session to discourage charging read as the 25.95p day rate, and a +5p increment on top
425+
of the event was dropped along with the event (#5163 review).
426+
"""
427+
print("**** test_saving_minute_keeps_user_override ****")
428+
failed = False
429+
430+
session_window = [{"start": "17:00:00", "end": "19:00:00"}]
431+
cases = [
432+
# (label, override items, manual rates, expected average). The window is 14h of 25.95p day
433+
# rate, 8h of 3.49p night rate and the 2h session; a snapshot without the overrides gave 18.46.
434+
("fixed 50p override", [dict(session_window[0], rate=50.0)], {}, (14 * 25.95 + 2 * 50.0 + 8 * 3.49) / 24),
435+
("+5p increment override", [dict(session_window[0], rate_increment=5.0)], {}, (14 * 25.95 + 2 * 30.95 + 8 * 3.49) / 24),
436+
("40p manual rate", [], {minute: 40.0 for minute in range(17 * 60, 19 * 60)}, (14 * 25.95 + 2 * 40.0 + 8 * 3.49) / 24),
437+
]
438+
for label, override, manual, expected in cases:
439+
_setup_two_rate_tariff(my_predbat, event_start=17 * 60, event_end=19 * 60, event_boost=50.0)
440+
my_predbat.rate_import, my_predbat.rate_import_pre_saving = my_predbat.apply_rate_overrides(
441+
my_predbat.rate_import, my_predbat.rate_import_pre_saving, my_predbat.rate_import_saving_minutes, override, "rates_import_override", manual, True, my_predbat.rate_import_replicated
442+
)
443+
my_predbat.rate_low_threshold = 1.0
444+
445+
my_predbat.set_rate_thresholds()
446+
447+
if abs(my_predbat.rate_import_cost_threshold - expected) > 0.01:
448+
print("ERROR: {}: rate_import_cost_threshold should be {:.2f} (average with the override kept), got {}".format(label, expected, my_predbat.rate_import_cost_threshold))
449+
failed = True
450+
451+
# With no session minutes nothing reads the snapshot, so it is left untouched
452+
_setup_two_rate_tariff(my_predbat, event_start=17 * 60, event_end=19 * 60, event_boost=50.0)
453+
pre_saving = my_predbat.rate_import_pre_saving
454+
_, returned = my_predbat.apply_rate_overrides(my_predbat.rate_import, pre_saving, set(), cases[0][1], "rates_import_override", {}, True, my_predbat.rate_import_replicated)
455+
if returned is not pre_saving or returned[17 * 60] != 25.95:
456+
print("ERROR: with no saving minutes the pre-saving snapshot should come back untouched, got {} at 17:00".format(returned.get(17 * 60)))
457+
failed = True
458+
459+
if not failed:
460+
print("PASS")
461+
return failed
462+
463+
418464
def _setup_export_event(my_predbat, export_boost=50.0):
419465
"""Flat 15p export tariff with a boosted, "saving"-tagged export event 10:00-12:00, and a two-rate import tariff (20p night, 30p day)."""
420466
my_predbat.minutes_now = 0
@@ -514,6 +560,8 @@ def test_fetch_snapshots_are_taken_in_the_right_order(my_predbat):
514560
- rate_import_saving_minutes must sit AFTER those three loaders (so every tagged minute is
515561
captured) and BEFORE basic_rates()/apply_manual_rates() (so an override active during a
516562
session cannot overwrite the "saving" tag before it is read - #5052 review).
563+
- The overrides go through apply_rate_overrides(), which also puts them on the pre-saving snapshot
564+
so a saving minute the user overrode is capped to the override, not the bare tariff (#5163 review).
517565
518566
Driving the whole of fetch_sensor_data() would need the entire sensor/Octopus/Axle surface stood
519567
up, so assert on the source order instead. That is weaker than a behavioural test, but it is the
@@ -543,13 +591,13 @@ def find(needle, label):
543591
free_slot = find("self.load_free_slot(self.octopus_free_slots, import_rates", "load_free_slot() for import")
544592
axle_slot = find("load_axle_slot(self, self.axle_sessions, import_rates", "load_axle_slot() for import")
545593
saving_minutes = find("self.rate_import_saving_minutes = {minute for minute, tag", "the rate_import_saving_minutes snapshot")
546-
basic = find('import_rates = self.basic_rates(self.get_arg("rates_import_override"', "basic_rates() for import")
594+
basic = find("import_rates, self.rate_import_pre_saving = self.apply_rate_overrides(", "the import overrides")
547595

548596
export_pre_saving = find("self.rate_export_pre_saving = self.rate_export_base.copy()", "the rate_export_pre_saving snapshot")
549597
export_saving_slot = find("self.load_saving_slot(self.octopus_saving_slots, export_rates", "load_saving_slot() for export")
550598
export_axle_slot = find("load_axle_slot(self, self.axle_sessions, export_rates", "load_axle_slot() for export")
551599
export_saving_minutes = find("self.rate_export_saving_minutes = {minute for minute, tag", "the rate_export_saving_minutes snapshot")
552-
export_basic = find('export_rates = self.basic_rates(self.get_arg("rates_export_override"', "basic_rates() for export")
600+
export_basic = find("export_rates, self.rate_export_pre_saving = self.apply_rate_overrides(", "the export overrides")
553601

554602
if None in (io_loop, pre_saving, saving_slot, free_slot, axle_slot, saving_minutes, basic, export_pre_saving, export_saving_slot, export_axle_slot, export_saving_minutes, export_basic):
555603
return True
@@ -563,11 +611,11 @@ def find(needle, label):
563611
failed = True
564612

565613
if not export_pre_saving < export_saving_slot < export_axle_slot < export_saving_minutes < export_basic:
566-
print("ERROR: export snapshots out of order - pre_saving {}, saving slot {}, axle {}, saving minutes {}, basic_rates {}".format(export_pre_saving, export_saving_slot, export_axle_slot, export_saving_minutes, export_basic))
614+
print("ERROR: export snapshots out of order - pre_saving {}, saving slot {}, axle {}, saving minutes {}, overrides {}".format(export_pre_saving, export_saving_slot, export_axle_slot, export_saving_minutes, export_basic))
567615
failed = True
568616

569617
if not axle_slot < saving_minutes < basic:
570-
print("ERROR: rate_import_saving_minutes must be snapshotted after the session loaders and before basic_rates() - got axle at {}, snapshot at {}, basic_rates at {}".format(axle_slot, saving_minutes, basic))
618+
print("ERROR: rate_import_saving_minutes must be snapshotted after the session loaders and before the overrides - got axle at {}, snapshot at {}, overrides at {}".format(axle_slot, saving_minutes, basic))
571619
failed = True
572620

573621
if not failed:
@@ -746,6 +794,53 @@ def test_compare_reused_live_rates_keep_saving_minutes(my_predbat):
746794
return failed
747795

748796

797+
def test_compare_override_reaches_kept_pre_saving_snapshot(my_predbat):
798+
"""A compare tariff's own override on reused live rates must also go on the kept pre-saving snapshot.
799+
800+
Otherwise the cap takes the overridden session minute back to the live tariff, as it did in
801+
fetch_sensor_data() before the #5163 review.
802+
"""
803+
print("**** test_compare_override_reaches_kept_pre_saving_snapshot ****")
804+
failed = False
805+
806+
from compare import Compare
807+
808+
my_predbat.minutes_now = 0
809+
my_predbat.forecast_minutes = 24 * 60
810+
rate_import_pre_saving = {minute: (30.0 if 6 <= (minute // 60) % 24 < 22 else 7.0) for minute in range(0, 48 * 60)}
811+
rate_import_live = rate_import_pre_saving.copy()
812+
session = set(range(17 * 60, 18 * 60))
813+
for minute in session:
814+
rate_import_live[minute] += 300.0
815+
rate_export_live = {minute: 15.0 for minute in range(0, 48 * 60)}
816+
817+
my_predbat.alert_active_keep = {}
818+
my_predbat.manual_soc_keep = {}
819+
my_predbat.num_cars = 0
820+
821+
compare = Compare(my_predbat)
822+
compare.live_saving_state = {
823+
"import_minutes": set(session),
824+
"export_minutes": set(),
825+
"import_pre_saving": rate_import_pre_saving,
826+
"export_pre_saving": dict(rate_export_live),
827+
}
828+
tariff = {"id": "test", "name": "test", "rates_import_override": [{"start": "17:00:00", "end": "18:00:00", "rate": 40.0}]}
829+
compare.fetch_rates(tariff, rate_import_live, rate_export_live)
830+
831+
rate_min, rate_max, _ = my_predbat.rate_minmax_excluding_saving(my_predbat.rate_import, my_predbat.rate_import_saving_minutes, my_predbat.rate_import_pre_saving)
832+
if rate_max != 40.0:
833+
print("ERROR: the tariff's 40p override on the session hour should be the threshold max, got {} (capped back to the live tariff)".format(rate_max))
834+
failed = True
835+
if rate_import_pre_saving[17 * 60] != 30.0:
836+
print("ERROR: fetch_rates() modified the caller's live pre-saving snapshot in place")
837+
failed = True
838+
839+
if not failed:
840+
print("PASS")
841+
return failed
842+
843+
749844
_SNAPSHOT_FIELDS = (
750845
"minutes_now",
751846
"forecast_minutes",
@@ -811,9 +906,11 @@ def run_set_rate_thresholds_tests(my_predbat):
811906
failed |= test_set_rate_thresholds_keeps_stored_stats_for_an_empty_table(my_predbat)
812907
failed |= test_rate_minmax_excluding_saving_ignores_overwritten_replicate_tag(my_predbat)
813908
failed |= test_saving_minute_capped_against_post_io_rates(my_predbat)
909+
failed |= test_saving_minute_keeps_user_override(my_predbat)
814910
failed |= test_fetch_snapshots_are_taken_in_the_right_order(my_predbat)
815911
failed |= test_compare_and_annual_clear_stale_saving_minutes(my_predbat)
816912
failed |= test_compare_reused_live_rates_keep_saving_minutes(my_predbat)
913+
failed |= test_compare_override_reaches_kept_pre_saving_snapshot(my_predbat)
817914
return failed
818915
finally:
819916
for field, value in snapshot.items():

‎docs/customisation.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,8 @@ If you set this too high you might not get any export slots. If it's too low you
421421

422422
Both thresholds, automatic or set by you, ignore an event price that is above the tariff's own rate (an Octopus saving session
423423
or Axle event reward): those slots count at the tariff's own rate when the thresholds are worked out, so the event does not
424-
stretch the range and pull ordinary-price slots in or out. A free or discounted import session still counts as a cheap slot.
424+
stretch the range and pull ordinary-price slots in or out. A free or discounted import session still counts as a cheap slot,
425+
and a rate override you put on an event slot (rates_import_override, rates_export_override or a manual rate) is kept.
425426
The event rates themselves are still used when planning.
426427

427428
**input_number.predbat_metric_future_rate_offset_import** (_expert mode_) Default 0p/kWh. Sets a pence per kWh offset to apply to future import energy rates that are

0 commit comments

Comments
 (0)