From a769310b0cad510b85628ff9620c3138c13912d0 Mon Sep 17 00:00:00 2001 From: Rik Allen Date: Mon, 5 Oct 2026 23:12:07 +0100 Subject: [PATCH] fix(plan history): rebuild once per completed slot, and judge staleness by each view's own refresh interval The plan page's History view is built by calculate_yesterday() from what actually happened, at most once every 59 minutes. The live Plan view starts at the current slot, so the slots finished since the last rebuild showed in neither view: at 21:35 the History stopped at 20:30 and the Plan started at 21:30. And the page judged every view against the live plan's 15-minute stale limit, so on the History view it warned "Plan data is stale" for about 45 minutes of every hour although nothing was wrong. - calculate_yesterday() now rebuilds once per completed plan slot (plan_interval_minutes), on the first re-plan at least one run (5 minutes) after the slot ends, so the boundary run's own cost_today write is recorded before its history is read back. A rebuild takes 0.4-0.9 s on a live install; at the default 30-minute slots that is 48 a day instead of 24 (before #2913 it ran on every re-plan, about 144). - Each dataset the page shows carries refresh_minutes, the longest Predbat should take to republish it (the live plan: calculate_plan_every; the plan history: a slot, the run it waits and a re-plan interval). The page warns a run after that, never under 15 minutes, against the data the current view shows, and hides the warning when the view has no data. At the defaults: Plan 15 minutes (as before), History 50. Trade-offs, kept: with calculate_plan_every at or above the slot length, re-plans can fall on slot boundaries and the rebuild then happens there; cost_yesterday and savings_yesterday_* roll over at the first re-plan after 00:05 rather than at 00:00; cost_yesterday/savings_yesterday_predbat, with their large html/json attributes, are written twice as often. Co-Authored-By: Claude Opus 5.5 --- apps/predbat/output.py | 38 +++-- .../predbat/tests/test_calculate_yesterday.py | 153 +++++++++++++----- apps/predbat/tests/test_plan_staleness_js.py | 61 +++++++ apps/predbat/unit_test.py | 2 + apps/predbat/web.py | 2 +- apps/predbat/web_helper.py | 72 ++++----- 6 files changed, 242 insertions(+), 86 deletions(-) create mode 100644 apps/predbat/tests/test_plan_staleness_js.py diff --git a/apps/predbat/output.py b/apps/predbat/output.py index 581000e08..a6dbdd3aa 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -1930,6 +1930,8 @@ def import_rate_color(rate): raw_plan["timestamp"] = self.now_utc_real.isoformat() if publish: + # The live plan is republished on every re-plan; the web page's stale check reads this + raw_plan["refresh_minutes"] = self.calculate_plan_every self.dashboard_item(self.prefix + ".plan_html", state="", attributes={"text": self.text_plan, "html": html, "raw": raw_plan, "friendly_name": "Plan in HTML", "icon": "mdi:web-box"}) return html, raw_plan @@ -3184,18 +3186,34 @@ def plan_row_holding_for_car(self, minute_start, minute_end, car_hold_minutes=No held = sum(PREDICT_STEP for minute in range(minute_start, minute_end, PREDICT_STEP) if (minute - self.minutes_now) in self.predict_car_hold_best) return held * 2 >= minute_end - minute_start + def history_slot(self, when): + """Which plan slot the plan history rebuild at when counts towards, as (date, slot number from midnight). + + Slots are plan_interval_minutes long, shifted one run (PREDICT_STEP) later: the run at a slot boundary + writes predbat.cost_today seconds before calculate_yesterday() reads its history back from Home + Assistant, so the slot just finished is rebuilt on the next run, once that value is recorded. This only + helps when that next run re-plans; with calculate_plan_every at or above the slot length, re-plans can + fall on boundaries and the rebuild then happens there. + """ + when = when - timedelta(minutes=PREDICT_STEP) + return when.date(), (when.hour * 60 + when.minute) // self.plan_interval_minutes + + def history_refresh_minutes(self): + """The longest the plan history should go between rebuilds: a slot, the run it waits, and up to one re-plan interval.""" + return self.plan_interval_minutes + PREDICT_STEP + self.calculate_plan_every + def calculate_yesterday(self): """ Calculate the base plan for yesterday """ - # Check savings_last_updated timestamp, we don't need to re-compute this one every iteration, once an hour or when the day rolls over is enough - if self.savings_last_updated: - # savings_last_update is a timestamp object, compare age and date - age = self.now_utc - self.savings_last_updated - if age < timedelta(minutes=59) and self.savings_last_updated.date() == self.now_utc.date(): - # Less than an hour old and already updated today - return + # Recompute once per completed plan slot rather than every iteration: the plan history (the web page's + # History view) shows each slot from what actually happened, so rebuilding as soon as a slot ends keeps the + # last finished slot on show - the live Plan view starts at the current slot, so an hourly rebuild left the + # slots since it in neither view. A day rolling over is a new slot too. Each rebuild takes under a second, + # and it waits one run past the boundary (see history_slot) so the boundary run's own writes are recorded. + if self.savings_last_updated and self.history_slot(self.savings_last_updated) == self.history_slot(self.now_utc): + return # Everything below is anchored on yesterday's recorded cost, so fetch that before doing any of # the expensive work - when Home Assistant isn't recording predbat.cost_today there is nothing @@ -3203,7 +3221,7 @@ def calculate_yesterday(self): cost_today_data = self.get_history_wrapper(entity_id=self.prefix + ".cost_today", days=2, required=False) if not cost_today_data: self.log("Warn: Calculate yesterday: No history for {}.cost_today, so the savings and plan history can not be computed - check that Home Assistant is recording this entity (see the recorder notes in the FAQ)".format(self.prefix)) - # Record the attempt so this is retried on the normal hourly cadence rather than every cycle + # Record the attempt so this is retried in the next slot rather than every cycle self.savings_last_updated = self.now_utc return @@ -3247,7 +3265,7 @@ def calculate_yesterday(self): # Assume user might charge at the lowest rate only, for fixed tariff # Only use yesterday's rate range (k < end_record) for the threshold to prevent today's rates # (which are added progressively as minutes_now increases) from changing the baseline charge - # windows on each hourly recalculation and causing savings_yesterday to fluctuate. + # windows on each recalculation and causing savings_yesterday to fluctuate. charge_window_best = [] rate_low = self.compute_rate_low_for_yesterday(past_rates, end_record) combine_charge = self.combine_charge_slots @@ -3438,6 +3456,7 @@ def calculate_yesterday(self): self.export_limits_best = [] self.export_window_best = [] plan_html_baseline, plan_json_baseline = self.plan_write_debug(True, None, yesterday_pv_step, yesterday_pv_step, yesterday_load_step, yesterday_load_step, end_record, prediction=self.prediction) + plan_json_baseline["refresh_minutes"] = self.history_refresh_minutes() # Now try to show what really happened yesterday self.charge_limit_best = [] @@ -3568,6 +3587,7 @@ def calculate_yesterday(self): plan_html_yesterday, plan_json_yesterday = self.publish_html_plan( yesterday_pv_step, yesterday_pv_step, yesterday_load_step, yesterday_load_step, end_record + minutes_now, publish=False, prediction=self.prediction, car_hold_minutes=car_hold_minutes ) + plan_json_yesterday["refresh_minutes"] = self.history_refresh_minutes() self.forecast_minutes = end_record # Restore state diff --git a/apps/predbat/tests/test_calculate_yesterday.py b/apps/predbat/tests/test_calculate_yesterday.py index 1d6555909..3d100f6ad 100644 --- a/apps/predbat/tests/test_calculate_yesterday.py +++ b/apps/predbat/tests/test_calculate_yesterday.py @@ -13,7 +13,8 @@ Covered scenarios ----------------- -1. Early-exit when savings_last_updated is fresh (< 59 min old, same day). +1. Early-exit when savings_last_updated is in the current plan slot; a run once a slot + has ended (test 1b) or the day has rolled over (test 4) is not skipped. 2. Basic run with no car: function runs, key dashboard entities are published and all state attributes are correctly restored afterwards. 3. Car-slot subtraction: when car_charging_slots has a slot covering some @@ -219,8 +220,8 @@ def _apply_mocks(my_predbat, now_utc, cost_value=100.0, soc_value=5.0): my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _make_history_mock(my_predbat, now_utc, cost_value, soc_value) - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") - my_predbat.publish_html_plan = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) + my_predbat.publish_html_plan = lambda *a, **kw: ("", {}) original_run_pred = my_predbat.run_prediction mock_run_pred = _make_mock_run_prediction(captured_load_steps) @@ -262,12 +263,13 @@ def _restore_methods(my_predbat, original_run_pred, original_step_data=None, ori def _test_early_exit(my_predbat, failed): - """Test 1: Early-exit when savings_last_updated is < 59 min old, same day.""" - print("calculate_yesterday: Test 1 – early-exit when timestamp is fresh") + """Test 1: Early-exit when savings_last_updated is in the current plan slot.""" + print("calculate_yesterday: Test 1 – early-exit when timestamp is in the current slot") now_utc = _setup_base(my_predbat) - # Mark as recently updated (10 minutes ago, same day) - my_predbat.savings_last_updated = now_utc - timedelta(minutes=10) + # 06:25 now, updated at 06:10 - both count towards the 06:05-06:35 history slot (one run past the boundary) + my_predbat.now_utc = now_utc + timedelta(minutes=25) + my_predbat.savings_last_updated = now_utc + timedelta(minutes=10) # Make sure dashboard_item is NOT called by asserting that the savings # entity is absent before the call (it might or might not exist from a @@ -281,11 +283,75 @@ def _test_early_exit(my_predbat, failed): print("ERROR: savings_last_updated was changed despite fresh timestamp") failed = True - # Reset for next test + # Reset for next test, including the clock moved above on the shared fixture + my_predbat.now_utc = now_utc my_predbat.savings_last_updated = None return failed +def _test_rebuild_after_slot_ends(my_predbat, failed): + """Test 1b: A run once the slot of the last update has ended is not skipped, however recent the update. + + The plan history shows each slot from what actually happened, so rebuilding as soon as a slot ends keeps the + last finished slot on show; the hourly rebuild this replaced left up to an hour of slots in no view. + """ + print("calculate_yesterday: Test 1b – rebuilt once the last update's slot has ended") + now_utc = _setup_base(my_predbat) + + # Updated at 05:50, ten minutes ago but towards the 05:35-06:05 history slot; the 06:00 run is still in it + # (it waits a run so its own cost_today write is recorded), so move on to the 06:05 run + my_predbat.savings_last_updated = now_utc - timedelta(minutes=10) + # Mocked first, so a broken early exit fails here rather than running for real on the shared fixture + captured_load, original_run_pred = _apply_mocks(my_predbat, now_utc + timedelta(minutes=5)) + my_predbat.calculate_yesterday() + if my_predbat.savings_last_updated != now_utc - timedelta(minutes=10): + print("ERROR: the run at the slot boundary should wait a run before rebuilding") + failed = True + now_utc = now_utc + timedelta(minutes=5) + my_predbat.now_utc = now_utc + my_predbat.minutes_now = 365 + + my_predbat.calculate_yesterday() + + if my_predbat.savings_last_updated != now_utc: + print("ERROR: calculate_yesterday did not rebuild once the slot had ended: savings_last_updated {}".format(my_predbat.savings_last_updated)) + failed = True + + _restore_methods(my_predbat, original_run_pred) + my_predbat.now_utc = now_utc - timedelta(minutes=5) + my_predbat.minutes_now = 360 + my_predbat.savings_last_updated = None + return failed + + +def _test_history_slot(my_predbat, failed): + """Test 1c: history_slot() follows plan_interval_minutes, one run late, and the published refresh interval matches.""" + print("calculate_yesterday: Test 1c – history slots follow plan_interval_minutes") + _setup_base(my_predbat) + saved = (my_predbat.plan_interval_minutes, my_predbat.calculate_plan_every) + try: + my_predbat.plan_interval_minutes = 15 + my_predbat.calculate_plan_every = 10 + base = datetime(2024, 10, 4, 6, 0, 0, tzinfo=UTC) + # 06:05-06:19 is one 15-minute history slot, 06:20 starts the next; 00:00 still counts towards yesterday + if my_predbat.history_slot(base + timedelta(minutes=5)) != my_predbat.history_slot(base + timedelta(minutes=19)): + print("ERROR: 06:05 and 06:19 should be the same 15-minute history slot") + failed = True + if my_predbat.history_slot(base + timedelta(minutes=19)) == my_predbat.history_slot(base + timedelta(minutes=20)): + print("ERROR: 06:20 should start a new 15-minute history slot") + failed = True + if my_predbat.history_slot(datetime(2024, 10, 5, 0, 0, 0, tzinfo=UTC)) != my_predbat.history_slot(datetime(2024, 10, 4, 23, 50, 0, tzinfo=UTC)): + print("ERROR: the midnight run should still count towards the last slot of the day before") + failed = True + # A 15-minute slot, the 5-minute run it waits and a 10-minute re-plan interval + if my_predbat.history_refresh_minutes() != 30: + print("ERROR: history_refresh_minutes {} should be a slot, a run and a re-plan interval".format(my_predbat.history_refresh_minutes())) + failed = True + finally: + my_predbat.plan_interval_minutes, my_predbat.calculate_plan_every = saved + return failed + + def _test_basic_no_car(my_predbat, failed): """Test 2: Basic run (no car) – entities published, state restored.""" print("calculate_yesterday: Test 2 – basic run, no car") @@ -350,6 +416,13 @@ def _test_basic_no_car(my_predbat, failed): print("ERROR: entity {} was not published".format(entity_id)) failed = True + # --- The plan history views carry how long they may go between rebuilds, for the web page's stale check --- + for entity_suffix in (".cost_yesterday", ".savings_yesterday_predbat"): + history_json = my_predbat.get_state_wrapper(prefix + entity_suffix, attribute="json") + if not isinstance(history_json, dict) or history_json.get("refresh_minutes") != my_predbat.history_refresh_minutes(): + print("ERROR: {} json should carry refresh_minutes {}, got {}".format(entity_suffix, my_predbat.history_refresh_minutes(), history_json.get("refresh_minutes") if isinstance(history_json, dict) else history_json)) + failed = True + # --- run_prediction was called (once for baseline, once for no-pvbat) --- if len(captured_load) < 2: print("ERROR: run_prediction should have been called at least twice, got {} captures".format(len(captured_load))) @@ -384,8 +457,8 @@ def _test_forecast_minutes_widened_before_step_data(my_predbat, failed): forecast_minutes_snapshots = [] my_predbat.step_data_history = _make_recording_step_data(my_predbat.pv_today, my_predbat, forecast_minutes_snapshots) my_predbat.get_history_wrapper = _make_history_mock(my_predbat, now_utc) - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") - my_predbat.publish_html_plan = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) + my_predbat.publish_html_plan = lambda *a, **kw: ("", {}) original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -636,12 +709,12 @@ def _test_car_slot_from_energy_sensor(my_predbat, failed): def _test_early_exit_respects_day_rollover(my_predbat, failed): """Test 4: Early-exit is NOT triggered if savings_last_updated was from - a previous day (even if it is < 59 min old by clock, the date differs).""" + a previous day (the same time of day yesterday is a different slot).""" print("calculate_yesterday: Test 4 – early-exit skipped when date rolls over") now_utc = _setup_base(my_predbat) - # Timestamp from yesterday – even though it is within 59 minutes of now_utc, - # the dates differ so the early-return condition should NOT fire. + # Timestamp from yesterday, five minutes into the same time-of-day slot - a different slot, + # so the early-return condition should NOT fire. my_predbat.savings_last_updated = (now_utc - timedelta(days=1)) + timedelta(minutes=5) captured_load, original_run_pred = _apply_mocks(my_predbat, now_utc) @@ -1333,8 +1406,8 @@ def _history_no_soc(entity_id, days=30, required=True, tracked=True): my_predbat.get_history_wrapper = _history_no_soc my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") - my_predbat.publish_html_plan = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) + my_predbat.publish_html_plan = lambda *a, **kw: ("", {}) original_run_pred = my_predbat.run_prediction ran_count = [0] @@ -1358,8 +1431,8 @@ def _history_no_soc(entity_id, days=30, required=True, tracked=True): my_predbat.get_history_wrapper = _history_no_soc # still returns None for soc_kw_h0 my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") - my_predbat.publish_html_plan = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) + my_predbat.publish_html_plan = lambda *a, **kw: ("", {}) original_run_pred = my_predbat.run_prediction ran_count_b = [0] my_predbat.run_prediction = lambda *a, **kw: _make_counting_run_pred(ran_count_b)(my_predbat, *a, **kw) @@ -1415,11 +1488,11 @@ def _capture_publish_html_plan(*args, **kwargs): captured["charge_window_best"] = copy.deepcopy(my_predbat.charge_window_best) captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) captured["export_limits_best"] = copy.deepcopy(my_predbat.export_limits_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_cross_charging - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -1497,11 +1570,11 @@ def _history_with_one_export(entity_id, days=30, required=True, tracked=True): def _capture_publish_html_plan(*args, **kwargs): captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_one_export - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -1558,11 +1631,11 @@ def _history_with_one_hold(entity_id, days=30, required=True, tracked=True): def _capture_publish_html_plan(*args, **kwargs): """Record the car_hold_minutes the actual-history table is rendered with.""" captured["car_hold_minutes"] = kwargs.get("car_hold_minutes") - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_one_hold - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -1656,11 +1729,11 @@ def _history_with_split_slots(entity_id, days=30, required=True, tracked=True): def _capture_publish_html_plan(*args, **kwargs): captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) captured["export_limits_best"] = copy.deepcopy(my_predbat.export_limits_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_split_slots - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -1752,11 +1825,11 @@ def _history_with_short_export(entity_id, days=30, required=True, tracked=True): def _capture_publish_html_plan(*args, **kwargs): captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) captured["export_limits_best"] = copy.deepcopy(my_predbat.export_limits_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_short_export - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -1827,11 +1900,11 @@ def _history_with_edge_export(entity_id, days=30, required=True, tracked=True): def _capture_publish_html_plan(*args, **kwargs): captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) captured["export_limits_best"] = copy.deepcopy(my_predbat.export_limits_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_edge_export - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -1893,11 +1966,11 @@ def _history_with_edge_blip(entity_id, days=30, required=True, tracked=True): def _capture_publish_html_plan(*args, **kwargs): captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) captured["charge_window_best"] = copy.deepcopy(my_predbat.charge_window_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_edge_blip - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -1968,11 +2041,11 @@ def _history_with_edge_only_states(entity_id, days=30, required=True, tracked=Tr def _capture_publish_html_plan(*args, **kwargs): captured["charge_window_best"] = copy.deepcopy(my_predbat.charge_window_best) captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_edge_only_states - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -2042,11 +2115,11 @@ def _history_with_cross_charging(entity_id, days=30, required=True, tracked=True def _capture_publish_html_plan(*args, **kwargs): captured["charge_window_best"] = copy.deepcopy(my_predbat.charge_window_best) captured["export_window_best"] = copy.deepcopy(my_predbat.export_window_best) - return ("", "{}") + return ("", {}) my_predbat.step_data_history = _make_mock_step_data(my_predbat.pv_today) my_predbat.get_history_wrapper = _history_with_cross_charging - my_predbat.plan_write_debug = lambda *a, **kw: ("", "{}") + my_predbat.plan_write_debug = lambda *a, **kw: ("", {}) my_predbat.publish_html_plan = _capture_publish_html_plan original_run_pred = my_predbat.run_prediction my_predbat.run_prediction = lambda *a, **kw: _make_mock_run_prediction([])(my_predbat, *a, **kw) @@ -2113,6 +2186,8 @@ def _no_history(entity_id, days=30, required=True, tracked=True): my_predbat.step_data_history = _counting_step_data my_predbat.get_history_wrapper = _no_history my_predbat.savings_last_updated = None + # 06:05, the first run counting towards the 06:05-06:35 history slot + my_predbat.now_utc = now_utc + timedelta(minutes=5) my_predbat.calculate_yesterday() @@ -2124,14 +2199,14 @@ def _no_history(entity_id, days=30, required=True, tracked=True): print("ERROR: calculate_yesterday did not record the attempt, so it will redo this work every cycle") failed = True - # A second call ten minutes later must early-exit rather than fetching and warning all over again + # A second call ten minutes later, in the same slot, must early-exit rather than fetching and warning all over again history_calls.clear() - my_predbat.now_utc = now_utc + timedelta(minutes=10) + my_predbat.now_utc = now_utc + timedelta(minutes=15) my_predbat.calculate_yesterday() if history_calls: - print("ERROR: calculate_yesterday retried within the hour, fetching {}".format(history_calls)) + print("ERROR: calculate_yesterday retried within the same slot, fetching {}".format(history_calls)) failed = True my_predbat.now_utc = now_utc @@ -2238,6 +2313,8 @@ def test_calculate_yesterday(my_predbat): print("**** Running calculate_yesterday tests ****") failed = _test_early_exit(my_predbat, failed) + failed = _test_rebuild_after_slot_ends(my_predbat, failed) + failed = _test_history_slot(my_predbat, failed) failed = _test_basic_no_car(my_predbat, failed) failed = _test_forecast_minutes_widened_before_step_data(my_predbat, failed) failed = _test_car_slot_subtraction(my_predbat, failed) diff --git a/apps/predbat/tests/test_plan_staleness_js.py b/apps/predbat/tests/test_plan_staleness_js.py new file mode 100644 index 000000000..9d07dc7d7 --- /dev/null +++ b/apps/predbat/tests/test_plan_staleness_js.py @@ -0,0 +1,61 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +# fmt off +# pylint: disable=consider-using-f-string +# pylint: disable=line-too-long +# pylint: disable=attribute-defined-outside-init + +"""Structural tests for the plan page's stale-data warning in get_plan_renderer_js(). + +Each view (Plan, History, Yesterday without Predbat) is republished on its own cadence, so the warning must be judged +against the data the current view shows and the refresh interval that data carries, and must clear when the view has +no data. Like test_debug_history_client_js.py, this asserts on the JS source (there is no JS engine in this suite). +""" + +from web_helper import get_plan_renderer_js + + +def function_source(renderer_js, signature): + """The source of one function in the renderer, from its signature to the end of its body.""" + start = renderer_js.index(signature) + return renderer_js[start : renderer_js.index("\n }", start)] + + +def test_plan_staleness_js(my_predbat): + """The stale warning uses the current view's data and its refresh_minutes, and is hidden when the view has none.""" + failed = False + print("**** Testing the plan page stale-data warning JS ****") + renderer_js = get_plan_renderer_js() + + check = function_source(renderer_js, "function checkStaleness(data)") + if "Math.max(data.refresh_minutes + 5, 15)" not in check or "STALE_MINUTES_DEFAULT[currentView]" not in check: + print(" ERROR: checkStaleness should allow a run (5 minutes) past the data's refresh_minutes, never under 15, falling back to a per-view default") + failed = True + if "ageMs > limitMinutes * 60000" not in check: + print(" ERROR: checkStaleness should compare the age in milliseconds with the limit in minutes") + failed = True + no_data = check[check.index("if (!data || !data.timestamp) {") :] if "if (!data || !data.timestamp) {" in check else "" + if "staleWarning.style.display = 'none'" not in no_data[: no_data.find("return;")]: + print(" ERROR: checkStaleness should hide the warning, not leave it up, when the view has no data") + failed = True + defaults = next((line for line in renderer_js.splitlines() if "const STALE_MINUTES_DEFAULT" in line), "") + if "plan: 15" not in defaults or "yesterday: 60" not in defaults or "baseline: 60" not in defaults: + print(" ERROR: expected default limits of 15 minutes for the plan and 60 for the history views, got {!r}".format(defaults)) + failed = True + + if "checkStaleness(currentViewData());" not in renderer_js: + print(" ERROR: the unchanged-data poll should check the current view's data, not always the plan's") + failed = True + if "checkStaleness(null);" not in function_source(renderer_js, "function refreshPlan()"): + print(" ERROR: refreshPlan should clear the warning when the current view has no data") + failed = True + if "const data = currentViewData();" not in function_source(renderer_js, "function updateTimestampDisplay()"): + print(" ERROR: updateTimestampDisplay should use currentViewData() rather than its own copy of the view mapping") + failed = True + if "checkStaleness(timestamp)" in renderer_js or "window.planData.timestamp);" in renderer_js: + print(" ERROR: a staleness check still passes a bare timestamp") + failed = True + return failed diff --git a/apps/predbat/unit_test.py b/apps/predbat/unit_test.py index 0f3d5a499..3f3a793a4 100644 --- a/apps/predbat/unit_test.py +++ b/apps/predbat/unit_test.py @@ -122,6 +122,7 @@ from tests.test_web_chat import run_web_chat_tests from tests.test_web_mcp import run_web_mcp_tests from tests.test_debug_history_client_js import test_debug_history_client_js +from tests.test_plan_staleness_js import test_plan_staleness_js from tests.test_metrics_dashboard_soc_refresh import test_soc_chart_center_text_reads_live_data from tests.test_web_functions import run_web_functions_tests, run_web_logo_image_tests, run_web_dark_mode_preference_tests from tests.test_web_power_flow import run_web_power_flow_tests @@ -542,6 +543,7 @@ def main(): ("web_chat", run_web_chat_tests, "Chat tab route, SSE framing and markdown escaping tests", False), ("web_mcp", run_web_mcp_tests, "MCP get_log and apps.yaml redaction tests (issue #4768)", False), ("debug_history_client_js", test_debug_history_client_js, "Debug-history client-side JS structure tests (#4438 review item 22)", False), + ("plan_staleness_js", test_plan_staleness_js, "Plan page stale-data warning JS: per-view data and refresh interval", False), ("metrics_dashboard_soc_refresh", test_soc_chart_center_text_reads_live_data, "Metrics dashboard SoC chart live-refresh tests", False), ("web_functions", run_web_functions_tests, "Web function unit tests", False), ("web_power_flow", run_web_power_flow_tests, "Power flow diagram car charging tests", False), diff --git a/apps/predbat/web.py b/apps/predbat/web.py index 613c33289..168f5f86d 100644 --- a/apps/predbat/web.py +++ b/apps/predbat/web.py @@ -2657,7 +2657,7 @@ async def html_plan(self, request): text += get_plan_css() # Add warning/error divs - text += '' + text += '' text += '' # Add view switcher buttons diff --git a/apps/predbat/web_helper.py b/apps/predbat/web_helper.py index 3e2f41cdb..a9d2b5b3c 100644 --- a/apps/predbat/web_helper.py +++ b/apps/predbat/web_helper.py @@ -7779,14 +7779,8 @@ def get_plan_renderer_js(): // Update timestamp display function updateTimestampDisplay() { - let timestamp = null; - if (currentView === 'plan' && window.planData) { - timestamp = window.planData.timestamp; - } else if (currentView === 'yesterday' && window.yesterdayData) { - timestamp = window.yesterdayData.timestamp; - } else if (currentView === 'baseline' && window.baselineData) { - timestamp = window.baselineData.timestamp; - } + const data = currentViewData(); + const timestamp = data ? data.timestamp : null; const timestampElement = document.getElementById('planTimestamp'); if (timestampElement) { @@ -7854,20 +7848,8 @@ def get_plan_renderer_js(): // Only allow debug mode for the plan view const showDebug = (currentView === 'plan' && debugToggle) ? debugToggle.checked : false; - let data, timestamp, overrides; - if (currentView === 'plan') { - data = window.planData; - timestamp = data ? data.timestamp : null; - overrides = window.overridesData || {}; - } else if (currentView === 'yesterday') { - data = window.yesterdayData; - timestamp = data ? data.timestamp : null; - overrides = {}; - } else { - data = window.baselineData; - timestamp = data ? data.timestamp : null; - overrides = {}; - } + const data = currentViewData(); + const overrides = currentView === 'plan' ? (window.overridesData || {}) : {}; if (!data) { if (currentView === 'plan') { @@ -7877,17 +7859,19 @@ def get_plan_renderer_js(): // which it can't do without the recorded history of predbat.cost_today - say so rather // than sitting on a loading message that will never go away container.innerHTML = '

No data for this view yet

' + - '

This view is computed about once an hour from what actually happened yesterday, ' + - 'so it stays empty for the first hour after Predbat starts.

' + + '

This view is computed from what actually happened, on the first plan update after Predbat starts ' + + 'and again as each plan slot ends, so it should fill in within a few minutes.

' + '

If it never fills in, Predbat could not read the history of predbat.cost_today ' + 'from Home Assistant. Check that the Home Assistant recorder is storing the Predbat entities ' + '(see the recorder notes in the FAQ) and look for Calculate yesterday warnings in the Predbat log.

'; } + // A warning raised for another view does not apply to this one + checkStaleness(null); return; } // Check for stale data - checkStaleness(timestamp); + checkStaleness(data); // Render table const editable = (currentView === 'plan'); @@ -7904,17 +7888,31 @@ def get_plan_renderer_js(): adjustResponsiveSizes(); } - // Check if data is stale (>15 minutes old) - function checkStaleness(timestamp) { - const staleWarning = document.getElementById('staleWarning'); - if (!timestamp || !staleWarning) return; + // How old a view's data may get before it is reported stale, when the data does not say. Each dataset + // carries refresh_minutes, the longest Predbat should take to republish it (the plan once per re-plan, + // the history views once per completed plan slot), and is stale a run (5 minutes) after that. + const STALE_MINUTES_DEFAULT = {plan: 15, yesterday: 60, baseline: 60}; - const dataTime = new Date(timestamp); - const now = new Date(); - const ageMs = now - dataTime; - const isStale = ageMs > 900000; // 15 minutes in milliseconds + // The data the current view shows + function currentViewData() { + return currentView === 'plan' ? window.planData : (currentView === 'yesterday' ? window.yesterdayData : window.baselineData); + } - if (isStale) { + // Show the stale warning when the view's data is older than it should be; with no data, hide it + function checkStaleness(data) { + const staleWarning = document.getElementById('staleWarning'); + if (!staleWarning) return; + if (!data || !data.timestamp) { + staleWarning.style.display = 'none'; + return; + } + + // Never under 15 minutes, the limit before views had their own: a slow run or plan_random_delay can push a + // 5-minute re-plan past 10 minutes without anything being wrong + const limitMinutes = data.refresh_minutes ? Math.max(data.refresh_minutes + 5, 15) : (STALE_MINUTES_DEFAULT[currentView] || 15); + const ageMs = new Date() - new Date(data.timestamp); + if (ageMs > limitMinutes * 60000) { + staleWarning.textContent = '\u26A0\uFE0F Plan data is stale (last updated over ' + limitMinutes + ' minutes ago)'; staleWarning.style.display = 'block'; } else { staleWarning.style.display = 'none'; @@ -7944,10 +7942,8 @@ def get_plan_renderer_js(): // Check if server says data is unchanged if (data.unchanged === true) { // Data hasn't changed, no need to update - // Still check staleness based on plan data timestamp - if (window.planData && window.planData.timestamp) { - checkStaleness(window.planData.timestamp); - } + // Still check staleness, against the data the current view shows + checkStaleness(currentViewData()); // Hide error message const errorDiv = document.getElementById('planError');