fix(solax): hold a lifetime counter that dips rather than read the recovery as energy

apps/predbat/solax.py

@@ -49,6 +49,9 @@
 SOLAX_DEVICE_INFO_MAX_AGE = 30  # Device info changes rarely, re-poll it once the data is this many minutes old
 SOLAX_CACHE_EXPIRY_HOURS = 24  # Cached plant and device info is discarded by the storage layer after this long
 SOLAX_INFO_RETRY_MINUTES = 5  # Wait this long before retrying a plant or device info read that failed
+SOLAX_COUNTER_DIP_HOLD_HOURS = 24  # A lifetime counter reading below its last published value is held this long before the lower value is taken as a genuine reset
+SOLAX_COUNTER_SAVE_MINUTES = 15  # The last counter values are saved to storage this often, a dip starting or ending is saved at once
+SOLAX_COUNTER_EXPIRY_DAYS = 30  # Saved counter state is discarded by the storage layer after this long
 SOLAX_COMMAND_RETRY_DELAY = 2.0
 SOLAX_COMMAND_MAX_RETRIES = 8
 SOLAX_REGIONS = {
@@ -390,8 +393,19 @@ def initialize(self, client_id, client_secret, region="eu", plant_id=None, autom
         # Plants whose PV energy has been built from their inverters, these never drop back to the plant total
         self.plant_pv_from_devices = set()
 
-        # Last known lifetime yield of each inverter, so that a failed read does not drop it from the plant sum
-        self.inverter_last_yield = {}
+        # Last value published for each lifetime counter and when it first read lower than that, so that a
+        # counter which dips and recovers is held rather than the recovery being read as energy (GH#5388)
+        self.counter_last = {}
+        self.counter_dip_since = {}
+
+        # The same state as restored from storage. The sensors do not survive a Home Assistant restart and the
+        # dip start time is nowhere else, so both are saved (see save_counter_state)
+        self.counter_stored = {}
+        self.counter_save_due = False
+        self.counter_saved = None
+
+        # Plants already warned for having no inverter with PV inputs or a yield counter
+        self.plant_no_pv_warned = set()
 
         # When the plant and device info was last read successfully, seeded from the storage cache on
         # startup so that a restart does not re-read data that is still current, plus when it was last
@@ -622,24 +636,113 @@ def inverter_expected(self, device_sn):
         """
         return self.device_info.get(device_sn, {}).get("onlineStatus", 1) != 0
 
+    def hold_counter_dip(self, entity_id, value):
+        """
+        Value to publish for a lifetime counter in kWh, held at the last published value while it reads lower
+
+        SolaX can return a lower value for a lifetime counter for a while and then the right one again, and
+        published as received the recovery is counted as energy for that day (GH#5388). The last published
+        value is seeded from the sensor itself so that a dip straight after a restart is held too. A lower
+        value that lasts SOLAX_COUNTER_DIP_HOLD_HOURS is taken as a genuine reset. Returns the last published
+        value when there is no reading, or None if there has never been one
+
+        The sensor is the first choice for the last published value as it is exact, the value restored from
+        storage can be a few minutes old and is for when the sensor is gone
+        """
+        try:
+            value = float(value)
+        except (ValueError, TypeError):
+            value = None
+        last = self.counter_last.get(entity_id)
+        if last is None:
+            try:
+                last = float(self.get_state_wrapper(entity_id, default=None))
+            except (ValueError, TypeError):
+                last = self.counter_stored.get(entity_id)
+        if value is None:
+            return last
+        if last is not None and value < last:
+            now = datetime.now(timezone.utc)
+            if entity_id not in self.counter_dip_since:
+                self.counter_dip_since[entity_id] = now
+                self.counter_save_due = True
+                self.log(f"Warn: SolaX API: {entity_id} read {value} which is lower than the last value {last}, keeping the last value")
+            if now - self.counter_dip_since[entity_id] < timedelta(hours=SOLAX_COUNTER_DIP_HOLD_HOURS):
+                self.counter_last[entity_id] = last
+                return last
+            self.log(f"Warn: SolaX API: {entity_id} has read lower than {last} for {SOLAX_COUNTER_DIP_HOLD_HOURS} hours, taking {value} as a reset")
+        if self.counter_dip_since.pop(entity_id, None) is not None:
+            self.counter_save_due = True
+        self.counter_last[entity_id] = value
+        return value
+
+    async def load_counter_state(self):
+        """
+        Restore the lifetime counter state from storage: the last values, and when any dip in progress started
+
+        Without the start time a restart would begin the hold period again, and a system restarted every day
+        would hold the old total for ever after a genuine reset
+        """
+        if not self.storage:
+            return
+        cached = await self.storage.load("solax", "counters")
+        if not isinstance(cached, dict):
+            return
+        for entity_id, value in (cached.get("last") or {}).items():
+            try:
+                self.counter_stored[entity_id] = float(value)
+            except (ValueError, TypeError):
+                pass
+        for entity_id, since in (cached.get("dip_since") or {}).items():
+            try:
+                self.counter_dip_since.setdefault(entity_id, datetime.fromisoformat(since))
+            except (ValueError, TypeError):
+                pass
+        self.log(f"SolaX API: Restored {len(self.counter_stored)} lifetime counter(s) from storage, {len(self.counter_dip_since)} held on a dip")
+
+    async def save_counter_state(self):
+        """
+        Save the lifetime counter state to storage, at once when a dip started or ended and otherwise every
+        SOLAX_COUNTER_SAVE_MINUTES
+
+        Returns:
+            True if the state was saved
+        """
+        if not self.storage or not self.counter_last:
+            return False
+        now = datetime.now(timezone.utc)
+        if not self.counter_save_due and self.counter_saved and now - self.counter_saved < timedelta(minutes=SOLAX_COUNTER_SAVE_MINUTES):
+            return False
+        data = {"last": dict(self.counter_last), "dip_since": {entity_id: since.isoformat() for entity_id, since in self.counter_dip_since.items()}}
+        if not await self.storage.save("solax", "counters", data, format="json", expiry=now + timedelta(days=SOLAX_COUNTER_EXPIRY_DAYS)):
+            return False
+        self.counter_save_due = False
+        self.counter_saved = now
+        return True
+
     def get_inverter_last_yield(self, entity_plant_id, device_sn):
         """
         Last known lifetime yield of an inverter in kWh, or None if it has never given one
 
-        Updated only when a read returns a number. With nothing known yet it is seeded from the inverter's
-        own total yield sensor, which survives a restart, so that an inverter which no longer answers
-        stays in the plant sum rather than stepping it down
+        The inverter's own total yield sensor is the record of it, which survives a restart, so an inverter
+        which no longer answers stays in the plant sum rather than stepping it down. A read that comes back
+        lower, 0.0 included, does not replace a known figure
         """
         device_yield = None
         if device_sn not in self.realtime_device_failed:
             device_yield = self.realtime_device_data.get(device_sn, {}).get("totalYield")
-        elif device_sn not in self.inverter_last_yield:
-            device_yield = self.get_state_wrapper(f"sensor.{self.prefix}_solax_{entity_plant_id}_{device_sn}_total_yield", default=None)
-        try:
-            self.inverter_last_yield[device_sn] = float(device_yield)
-        except (ValueError, TypeError):
-            pass
-        return self.inverter_last_yield.get(device_sn)
+        return self.hold_counter_dip(f"sensor.{self.prefix}_solax_{entity_plant_id}_{device_sn}_total_yield", device_yield)
+
+    def plant_has_pv_source(self, plant_id):
+        """
+        True if any inverter in the plant has PV inputs or a yield counter, a counter reading 0.0 included. An
+        inverter that has not been read counts as having them
+        """
+        for device_sn in self.plant_inverters.get(plant_id, []):
+            realtime = self.realtime_device_data.get(device_sn)
+            if realtime is None or device_sn in self.realtime_device_failed or realtime.get("mpptMap") or realtime.get("pvMap") or realtime.get("totalYield") is not None:
+                return True
+        return False
 
     def get_plant_pv_yield(self, plant_id, plant_yield):
         """
@@ -648,8 +751,9 @@ def get_plant_pv_yield(self, plant_id, plant_yield):
         The plant level totalYield includes the AC output of an AC-coupled battery inverter, so battery
         discharge reads as generation (GH#5356). Each inverter contributes its last known yield, a lifetime
         counter does not go backwards so a failed read cannot make the sum dip. An inverter with no yield of
-        its own counts as 0. Returns None when the value should be held, which is only when an inverter that
-        should be answering failed its read and has never given a yield
+        its own counts as 0, and a plant where no inverter has PV inputs either gives 0 rather than the plant
+        total. Returns None when the value should be held, which is only when an inverter that should be
+        answering failed its read and has never given a yield
 
         plant_id is the ID as SolaX gives it, which is what the inverters are stored under. The entity names
         use a tidied form of it
@@ -669,7 +773,15 @@ def get_plant_pv_yield(self, plant_id, plant_yield):
             return pv_yield
         if plant_id in self.plant_pv_from_devices:
             return None
-        # No inverter reports a yield of its own, so the plant total is the only PV figure there is
+        # A plant with no PV inputs at all has no PV figure. Its plant total is the inverter AC output, which is
+        # battery discharge, so 0 is published rather than that. Any PV on the site is on an inverter SolaX
+        # does not measure, and the load then excludes whatever that PV supplies (GH#5388)
+        if not self.plant_has_pv_source(plant_id):
+            if plant_id not in self.plant_no_pv_warned:
+                self.plant_no_pv_warned.add(plant_id)
+                self.log(f"Warn: SolaX API: Plant {plant_id} has no inverter with PV inputs or a yield counter, PV today is published as 0. If the site has PV on another inverter it is not measured, and load today will be low by whatever that PV supplies")
+            return 0.0
+        # The inverters have PV inputs but report no yield of their own, so the plant total is the only PV figure there is
         return plant_yield
 
     def get_max_power_inverter(self, plant_id):
@@ -2334,7 +2446,8 @@ async def publish_device_realtime_data(self):
                 mpptMap = realtime.get("mpptMap", {})  # cSpell:disable-line
                 totalActivePower = realtime.get("totalActivePower", 0)
                 totalReactivePower = realtime.get("totalReactivePower", 0)
-                totalYield = realtime.get("totalYield", 0)
+                # Held on a dip, this sensor is what the inverter's last known yield is seeded from after a restart
+                totalYield = self.hold_counter_dip(f"sensor.{self.prefix}_solax_{plant_id}_{device_sn}_total_yield", realtime.get("totalYield", 0))
                 deviceStatus = realtime.get("deviceStatus", 0)
                 deviceStatusText = SOLAX_INVERTER_STATUS.get(deviceStatus, "Unknown Status")
 
@@ -2728,10 +2841,16 @@ async def publish_plant_info(self):
             elif realtime_plant_id and realtime_plant_id in self.realtime_data:
                 realtime = self.realtime_data[realtime_plant_id]
 
+                # The lifetime totals are held on a dip, SolaX can return a lower value for a while and the
+                # recovery would otherwise be read as energy. The load below is built from the held values
+                totals = {}
+                for key, name in [("totalYield", "total_yield"), ("totalCharged", "total_charged"), ("totalDischarged", "total_discharged"), ("totalImported", "total_imported"), ("totalExported", "total_exported")]:
+                    totals[key] = self.hold_counter_dip(f"sensor.{self.prefix}_solax_{plant_id}_{name}", realtime.get(key, 0.0)) or 0.0
+
                 # Total Yield sensor
                 self.dashboard_item(
                     f"sensor.{self.prefix}_solax_{plant_id}_total_yield",
-                    state=realtime.get("totalYield", 0.0),
+                    state=totals["totalYield"],
                     attributes={
                         "friendly_name": f"SolaX {plant_name} Total Yield",
                         "unit_of_measurement": "kWh",
@@ -2744,7 +2863,7 @@ async def publish_plant_info(self):
                 # Total Charged sensor
                 self.dashboard_item(
                     f"sensor.{self.prefix}_solax_{plant_id}_total_charged",
-                    state=realtime.get("totalCharged", 0.0),
+                    state=totals["totalCharged"],
                     attributes={
                         "friendly_name": f"SolaX {plant_name} Total Charged",
                         "unit_of_measurement": "kWh",
@@ -2757,7 +2876,7 @@ async def publish_plant_info(self):
                 # Total Discharged sensor
                 self.dashboard_item(
                     f"sensor.{self.prefix}_solax_{plant_id}_total_discharged",
-                    state=realtime.get("totalDischarged", 0.0),
+                    state=totals["totalDischarged"],
                     attributes={
                         "friendly_name": f"SolaX {plant_name} Total Discharged",
                         "unit_of_measurement": "kWh",
@@ -2770,7 +2889,7 @@ async def publish_plant_info(self):
                 # Total Imported sensor
                 self.dashboard_item(
                     f"sensor.{self.prefix}_solax_{plant_id}_total_imported",
-                    state=realtime.get("totalImported", 0.0),
+                    state=totals["totalImported"],
                     attributes={
                         "friendly_name": f"SolaX {plant_name} Total Imported",
                         "unit_of_measurement": "kWh",
@@ -2783,7 +2902,7 @@ async def publish_plant_info(self):
                 # Total Exported sensor
                 self.dashboard_item(
                     f"sensor.{self.prefix}_solax_{plant_id}_total_exported",
-                    state=realtime.get("totalExported", 0.0),
+                    state=totals["totalExported"],
                     attributes={
                         "friendly_name": f"SolaX {plant_name} Total Exported",
                         "unit_of_measurement": "kWh",
@@ -2794,7 +2913,7 @@ async def publish_plant_info(self):
                 )
                 # PV energy is built from the inverters, as the plant total yield above counts the output of an
                 # AC-coupled battery inverter as generation. The load is derived from it so both are held together
-                pv_yield = self.get_plant_pv_yield(realtime_plant_id, realtime.get("totalYield", 0.0))
+                pv_yield = self.get_plant_pv_yield(realtime_plant_id, totals["totalYield"])
                 if pv_yield is None:
                     self.log(f"Warn: SolaX API: No inverter yield read for plant {realtime_plant_id}, keeping the previous PV yield and load values")
                 else:
@@ -2812,7 +2931,7 @@ async def publish_plant_info(self):
 
                     # Work out total load
                     # This is total imported + total discharged - total exported - total charged + PV yield
-                    total_load = realtime.get("totalImported", 0.0) + realtime.get("totalDischarged", 0.0) - realtime.get("totalExported", 0.0) - realtime.get("totalCharged", 0.0) + pv_yield
+                    total_load = totals["totalImported"] + totals["totalDischarged"] - totals["totalExported"] - totals["totalCharged"] + pv_yield
                     self.dashboard_item(
                         f"sensor.{self.prefix}_solax_{plant_id}_total_load",
                         state=total_load,
@@ -2851,6 +2970,7 @@ async def run(self, seconds, first):
         if first:
             # Reuse whatever hardware information is still current, a restart then avoids re-reading it all
             await self.load_static_info()
+            await self.load_counter_state()
 
         # Plant info is static, it is only re-read once the data itself has aged out
         plant_info_refreshed = False
@@ -2928,6 +3048,7 @@ async def run(self, seconds, first):
             await self.publish_device_info()
             await self.publish_device_realtime_data()
             await self.publish_controls()
+            await self.save_counter_state()
 
         # Automatic configuration
         if first and self.automatic: