diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 92644d205..94302b403 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -605,7 +605,7 @@ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0, return import_today - def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling=1.0, required_unit=None, interpolate=False, pad=True): + def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling=1.0, required_unit=None, interpolate=False, pad=True, clean_increment=True): """ Download one or more entities for load data """ @@ -652,7 +652,7 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling backwards=True, smoothing=True, scale=load_scaling, - clean_increment=True, + clean_increment=clean_increment, accumulate=load_minutes, required_unit=required_unit, interpolate=interpolate, @@ -744,9 +744,8 @@ def fetch_sensor_data(self, save=True): self.download_ge_data(self.now_utc) if ("load_power" in self.args) and self.get_arg("load_power_fill_enable", True): - # Use power data to make load data more accurate self.log("Using load_power data to fill gaps in load_today data") - load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True) + load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True, clean_increment=False) self.load_minutes = self.fill_load_from_power(self.load_minutes, load_power_data) else: # Load data @@ -757,9 +756,12 @@ def fetch_sensor_data(self, save=True): self.load_last_period = (self.load_minutes.get(0, 0) - self.load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP if ("load_power" in self.args) and self.get_arg("load_power_fill_enable", True): - # Use power data to make load data more accurate + # Use power data to make load data more accurate. + # clean_increment=False: power sensors report instantaneous W, not cumulative kWh. + # clean_incrementing_reverse would distort fluctuating power readings into an + # ever-growing cumulative series, inflating fill_load_from_power gap-fills. self.log("Using load_power data to fill gaps in load_today data") - load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True) + load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True, clean_increment=False) self.load_minutes = self.fill_load_from_power(self.load_minutes, load_power_data) else: if self.load_forecast: diff --git a/apps/predbat/load_ml_component.py b/apps/predbat/load_ml_component.py index 84e0df36c..d0bc62358 100644 --- a/apps/predbat/load_ml_component.py +++ b/apps/predbat/load_ml_component.py @@ -315,7 +315,10 @@ async def _fetch_load_data(self): load_power_data = None if self.get_arg("load_power", default=None, indirect=False) and self.get_arg("load_power_fill_enable", True): - load_power_data, load_minutes_age = self.base.minute_data_load(self.now_utc, "load_power", days_to_fetch, required_unit="W", load_scaling=1.0, interpolate=True, pad=False) + # clean_increment=False: power sensors report instantaneous W, not cumulative kWh. + # clean_incrementing_reverse would distort fluctuating power readings into an + # ever-growing cumulative series, inflating fill_load_from_power gap-fills. + load_power_data, load_minutes_age = self.base.minute_data_load(self.now_utc, "load_power", days_to_fetch, required_unit="W", load_scaling=1.0, interpolate=True, pad=False, clean_increment=False) load_minutes = self.base.fill_load_from_power(load_minutes, load_power_data) # Re-read car charging settings dynamically so switch changes are picked up without a restart. diff --git a/apps/predbat/tests/test_fill_load_from_power.py b/apps/predbat/tests/test_fill_load_from_power.py index d2eb14e47..2ab54321d 100644 --- a/apps/predbat/tests/test_fill_load_from_power.py +++ b/apps/predbat/tests/test_fill_load_from_power.py @@ -8,6 +8,7 @@ # Add the apps/predbat directory to the path sys.path.append(os.path.join(os.path.dirname(__file__), "..", "apps", "predbat")) +from datetime import datetime, timezone from fetch import Fetch from utils import dp4 @@ -20,6 +21,7 @@ def __init__(self): self.log_messages = [] self.forecast_minutes = 24 * 60 # 24 hours self.plan_interval_minutes = 30 + self.now_utc = datetime.now(timezone.utc) def log(self, message): """Capture log messages""" @@ -362,6 +364,7 @@ def run_all_tests(my_predbat=None): test_fill_load_from_power_zero_load() test_fill_load_from_power_backwards_time() test_fill_load_from_power_data_extends_beyond_load() + test_fill_load_from_power_distorted_power_inflation() print("\n" + "=" * 60) print("✅ ALL TESTS PASSED") @@ -382,6 +385,88 @@ def run_all_tests(my_predbat=None): return 1 # Return 1 for error +def test_fill_load_from_power_distorted_power_inflation(): + """ + Regression test for issue #3692: ML load actual ~2x real consumption. + + When load_power data is loaded via minute_data_load with clean_increment=True, + clean_incrementing_reverse treats the fluctuating instantaneous power (W) values + as a cumulative sensor and accumulates all positive increments into an ever-growing + series. fill_load_from_power Phase 1 (zero-period filling) then integrates these + inflated values, bumping all more-recent cumulative entries by the total inflated + fill. Phase 2 preserves these bumped endpoints, so the overall energy + (data[0] - data[max]) is inflated even though per-window redistribution is correct. + + This test verifies that: + 1. clean_incrementing_reverse grossly distorts non-cumulative power data + 2. fill_load_from_power with distorted power inflates the total cumulative energy + 3. fill_load_from_power with correct power produces a reasonable total + """ + print("\n=== Test 8: Power data distortion causes load inflation (issue #3692) ===") + + from utils import clean_incrementing_reverse + + fetch = TestFetch() + + # Simulate 240 minutes (4 hours) of load data with a 60-minute flat (zero) period + # from minute 60 to minute 120. Backwards format: minute 0 is now (highest value). + load_minutes = {} + # Minute 0-59: smooth ramp (0.05 kWh/min consumed) + for minute in range(0, 60): + load_minutes[minute] = 15.0 - minute * 0.05 + # Minute 60-120: flat at 12.0 kWh (sensor didn't increment - zero period) + for minute in range(60, 121): + load_minutes[minute] = 12.0 + # Minute 121-240: smooth ramp + for minute in range(121, 241): + load_minutes[minute] = 12.0 - (minute - 120) * 0.05 + + total_original = load_minutes[0] - load_minutes[240] + + # Realistic fluctuating power data (~600W, varying 300-900W) - + # this is what minute_data_load should return when clean_increment=False + correct_power_data = {} + for minute in range(241): + correct_power_data[minute] = 600.0 + 100.0 * ((minute % 7) - 3) + + # Simulate the bug: clean_incrementing_reverse on non-cumulative power data + distorted_power_data = clean_incrementing_reverse(correct_power_data) + + # Verify distortion magnitude + max_correct = max(correct_power_data.values()) + max_distorted = max(distorted_power_data.values()) + print(f" Max correct power: {max_correct:.0f}W, Max distorted: {max_distorted:.0f}W") + assert max_distorted > max_correct * 10, f"Distorted power should be >>10x correct; got {max_distorted:.0f} vs {max_correct:.0f}" + + # Run fill_load_from_power with CORRECT power data + result_correct = fetch.fill_load_from_power(load_minutes.copy(), correct_power_data) + + # Run fill_load_from_power with DISTORTED power data (the bug) + result_distorted = fetch.fill_load_from_power(load_minutes.copy(), distorted_power_data) + + # Check total energy (data[0] - data[max]): the Phase 1 bump inflates data[0] + total_correct = result_correct[0] - result_correct.get(240, result_correct.get(239, 0)) + total_distorted = result_distorted[0] - result_distorted.get(240, result_distorted.get(239, 0)) + + print(f" Total energy original: {total_original:.2f} kWh") + print(f" Total energy (correct power): {total_correct:.2f} kWh") + print(f" Total energy (distorted power): {total_distorted:.2f} kWh") + print(f" Inflation ratio (distorted/orig): {total_distorted / total_original:.2f}x") + + # Correct power fill should add a reasonable amount (60 min at ~600W = ~0.6 kWh) + correct_fill_excess = total_correct - total_original + assert correct_fill_excess < 2.0, f"Correct fill should add <2 kWh over original; added {correct_fill_excess:.2f}" + + # Distorted power fill should cause significant inflation (the bug) + distorted_fill_excess = total_distorted - total_original + assert distorted_fill_excess > 2.0, f"Distorted fill should inflate by >2 kWh; only added {distorted_fill_excess:.2f}. " "If this assertion fails, Phase 2 may be fully correcting Phase 1 and the bug " "mechanism is different than expected." + + # The distorted total should be noticeably larger than the correct total + assert total_distorted > total_correct * 1.2, f"Distorted total ({total_distorted:.2f}) should be >1.2x correct ({total_correct:.2f})" + + print(" PASS: Distorted power data causes measurable cumulative inflation") + + if __name__ == "__main__": success = run_all_tests() sys.exit(0 if success else 1)