Merge pull request #3784 from amasolov/fix/ml-load-power-clean-increment

Fix ML load inflation: skip clean_incrementing_reverse for power sensors

Author: springfall2008

apps/predbat/fetch.py

@@ -605,7 +605,7 @@ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0,
 
         return import_today
 
-    def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling=1.0, required_unit=None, interpolate=False, pad=True):
+    def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling=1.0, required_unit=None, interpolate=False, pad=True, clean_increment=True):
         """
         Download one or more entities for load data
         """
@@ -652,7 +652,7 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling
                     backwards=True,
                     smoothing=True,
                     scale=load_scaling,
-                    clean_increment=True,
+                    clean_increment=clean_increment,
                     accumulate=load_minutes,
                     required_unit=required_unit,
                     interpolate=interpolate,
@@ -744,9 +744,8 @@ def fetch_sensor_data(self, save=True):
             self.download_ge_data(self.now_utc)
 
             if ("load_power" in self.args) and self.get_arg("load_power_fill_enable", True):
-                # Use power data to make load data more accurate
                 self.log("Using load_power data to fill gaps in load_today data")
-                load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True)
+                load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True, clean_increment=False)
                 self.load_minutes = self.fill_load_from_power(self.load_minutes, load_power_data)
         else:
             # Load data
@@ -757,9 +756,12 @@ def fetch_sensor_data(self, save=True):
                 self.load_last_period = (self.load_minutes.get(0, 0) - self.load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP
 
                 if ("load_power" in self.args) and self.get_arg("load_power_fill_enable", True):
-                    # Use power data to make load data more accurate
+                    # Use power data to make load data more accurate.
+                    # clean_increment=False: power sensors report instantaneous W, not cumulative kWh.
+                    # clean_incrementing_reverse would distort fluctuating power readings into an
+                    # ever-growing cumulative series, inflating fill_load_from_power gap-fills.
                     self.log("Using load_power data to fill gaps in load_today data")
-                    load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True)
+                    load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True, clean_increment=False)
                     self.load_minutes = self.fill_load_from_power(self.load_minutes, load_power_data)
             else:
                 if self.load_forecast:

apps/predbat/load_ml_component.py

@@ -315,7 +315,10 @@ async def _fetch_load_data(self):
 
             load_power_data = None
             if self.get_arg("load_power", default=None, indirect=False) and self.get_arg("load_power_fill_enable", True):
-                load_power_data, load_minutes_age = self.base.minute_data_load(self.now_utc, "load_power", days_to_fetch, required_unit="W", load_scaling=1.0, interpolate=True, pad=False)
+                # clean_increment=False: power sensors report instantaneous W, not cumulative kWh.
+                # clean_incrementing_reverse would distort fluctuating power readings into an
+                # ever-growing cumulative series, inflating fill_load_from_power gap-fills.
+                load_power_data, load_minutes_age = self.base.minute_data_load(self.now_utc, "load_power", days_to_fetch, required_unit="W", load_scaling=1.0, interpolate=True, pad=False, clean_increment=False)
                 load_minutes = self.base.fill_load_from_power(load_minutes, load_power_data)
 
             # Re-read car charging settings dynamically so switch changes are picked up without a restart.

apps/predbat/tests/test_fill_load_from_power.py

@@ -8,6 +8,7 @@
 # Add the apps/predbat directory to the path
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "apps", "predbat"))
 
+from datetime import datetime, timezone
 from fetch import Fetch
 from utils import dp4
 
@@ -20,6 +21,7 @@ def __init__(self):
         self.log_messages = []
         self.forecast_minutes = 24 * 60  # 24 hours
         self.plan_interval_minutes = 30
+        self.now_utc = datetime.now(timezone.utc)
 
     def log(self, message):
         """Capture log messages"""
@@ -362,6 +364,7 @@ def run_all_tests(my_predbat=None):
         test_fill_load_from_power_zero_load()
         test_fill_load_from_power_backwards_time()
         test_fill_load_from_power_data_extends_beyond_load()
+        test_fill_load_from_power_distorted_power_inflation()
 
         print("\n" + "=" * 60)
         print("✅ ALL TESTS PASSED")
@@ -382,6 +385,88 @@ def run_all_tests(my_predbat=None):
         return 1  # Return 1 for error
 
 
+def test_fill_load_from_power_distorted_power_inflation():
+    """
+    Regression test for issue #3692: ML load actual ~2x real consumption.
+
+    When load_power data is loaded via minute_data_load with clean_increment=True,
+    clean_incrementing_reverse treats the fluctuating instantaneous power (W) values
+    as a cumulative sensor and accumulates all positive increments into an ever-growing
+    series.  fill_load_from_power Phase 1 (zero-period filling) then integrates these
+    inflated values, bumping all more-recent cumulative entries by the total inflated
+    fill.  Phase 2 preserves these bumped endpoints, so the overall energy
+    (data[0] - data[max]) is inflated even though per-window redistribution is correct.
+
+    This test verifies that:
+    1. clean_incrementing_reverse grossly distorts non-cumulative power data
+    2. fill_load_from_power with distorted power inflates the total cumulative energy
+    3. fill_load_from_power with correct power produces a reasonable total
+    """
+    print("\n=== Test 8: Power data distortion causes load inflation (issue #3692) ===")
+
+    from utils import clean_incrementing_reverse
+
+    fetch = TestFetch()
+
+    # Simulate 240 minutes (4 hours) of load data with a 60-minute flat (zero) period
+    # from minute 60 to minute 120. Backwards format: minute 0 is now (highest value).
+    load_minutes = {}
+    # Minute 0-59: smooth ramp (0.05 kWh/min consumed)
+    for minute in range(0, 60):
+        load_minutes[minute] = 15.0 - minute * 0.05
+    # Minute 60-120: flat at 12.0 kWh (sensor didn't increment - zero period)
+    for minute in range(60, 121):
+        load_minutes[minute] = 12.0
+    # Minute 121-240: smooth ramp
+    for minute in range(121, 241):
+        load_minutes[minute] = 12.0 - (minute - 120) * 0.05
+
+    total_original = load_minutes[0] - load_minutes[240]
+
+    # Realistic fluctuating power data (~600W, varying 300-900W) -
+    # this is what minute_data_load should return when clean_increment=False
+    correct_power_data = {}
+    for minute in range(241):
+        correct_power_data[minute] = 600.0 + 100.0 * ((minute % 7) - 3)
+
+    # Simulate the bug: clean_incrementing_reverse on non-cumulative power data
+    distorted_power_data = clean_incrementing_reverse(correct_power_data)
+
+    # Verify distortion magnitude
+    max_correct = max(correct_power_data.values())
+    max_distorted = max(distorted_power_data.values())
+    print(f"  Max correct power: {max_correct:.0f}W, Max distorted: {max_distorted:.0f}W")
+    assert max_distorted > max_correct * 10, f"Distorted power should be >>10x correct; got {max_distorted:.0f} vs {max_correct:.0f}"
+
+    # Run fill_load_from_power with CORRECT power data
+    result_correct = fetch.fill_load_from_power(load_minutes.copy(), correct_power_data)
+
+    # Run fill_load_from_power with DISTORTED power data (the bug)
+    result_distorted = fetch.fill_load_from_power(load_minutes.copy(), distorted_power_data)
+
+    # Check total energy (data[0] - data[max]): the Phase 1 bump inflates data[0]
+    total_correct = result_correct[0] - result_correct.get(240, result_correct.get(239, 0))
+    total_distorted = result_distorted[0] - result_distorted.get(240, result_distorted.get(239, 0))
+
+    print(f"  Total energy original:           {total_original:.2f} kWh")
+    print(f"  Total energy (correct power):    {total_correct:.2f} kWh")
+    print(f"  Total energy (distorted power):  {total_distorted:.2f} kWh")
+    print(f"  Inflation ratio (distorted/orig): {total_distorted / total_original:.2f}x")
+
+    # Correct power fill should add a reasonable amount (60 min at ~600W = ~0.6 kWh)
+    correct_fill_excess = total_correct - total_original
+    assert correct_fill_excess < 2.0, f"Correct fill should add <2 kWh over original; added {correct_fill_excess:.2f}"
+
+    # Distorted power fill should cause significant inflation (the bug)
+    distorted_fill_excess = total_distorted - total_original
+    assert distorted_fill_excess > 2.0, f"Distorted fill should inflate by >2 kWh; only added {distorted_fill_excess:.2f}. " "If this assertion fails, Phase 2 may be fully correcting Phase 1 and the bug " "mechanism is different than expected."
+
+    # The distorted total should be noticeably larger than the correct total
+    assert total_distorted > total_correct * 1.2, f"Distorted total ({total_distorted:.2f}) should be >1.2x correct ({total_correct:.2f})"
+
+    print("  PASS: Distorted power data causes measurable cumulative inflation")
+
+
 if __name__ == "__main__":
     success = run_all_tests()
     sys.exit(0 if success else 1)