diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt index 882de297e..fd9caa636 100644 --- a/.cspell/custom-dictionary-workspace.txt +++ b/.cspell/custom-dictionary-workspace.txt @@ -43,6 +43,7 @@ autoflake automations autopep autoupdate +awattar axvline axvspan Backfeed @@ -171,12 +172,14 @@ dstamp dstart dtype dwindow +Dynamisch eauth echargingpile ecosmart Eddi eddis efgh +Elektrizitätswerke elif elinter eload @@ -374,7 +377,11 @@ lstrip luxpower malloc mallopt +malo +malos markdownlint +marketdata +marketprice matplotlib maximise maxrss @@ -473,6 +480,7 @@ openapi openrouter openweathermap Opticaster +ostrom overclaim overfitting oversizing @@ -578,6 +586,8 @@ saverestore savez scalarstring scalings +schoenau +Schönau searr securetoken Segoe @@ -655,6 +665,7 @@ syscmd sysdn syssn tabindex +tado tasmota TCWORLD tdata @@ -679,6 +690,7 @@ timekey timelapse timenow timeobj +timespec timestep timestr timezone @@ -767,3 +779,4 @@ zappi zappis zcat zigbee +Ökostrom diff --git a/apps/predbat/components.py b/apps/predbat/components.py index 611c36477..da32501f1 100644 --- a/apps/predbat/components.py +++ b/apps/predbat/components.py @@ -583,14 +583,23 @@ def load_component_class(component_info): "class": "spotprice.SpotPriceAPI", "name": "Day-ahead Spot Price Tariff", "args": { - # No registry default: when unset, initialize() picks tibber whenever a Tibber token is set - # (with or without a zone), else energycharts + # No registry default: when unset, initialize() picks the one supplier whose credentials are + # set (with or without a zone), else energycharts; credentials for several is a config error "provider": {"required": False, "config": "spotprice_provider"}, "zone": {"required": False, "config": "spotprice_zone"}, "entsoe_token": {"required": False, "secret": True, "config": "spotprice_entsoe_token"}, "tibber_token": {"required": False, "secret": True, "config": "spotprice_tibber_token"}, "tibber_home_id": {"required": False, "secret": True, "config": "spotprice_tibber_home_id"}, + "ostrom_client_id": {"required": False, "secret": True, "config": "spotprice_ostrom_client_id"}, + "ostrom_client_secret": {"required": False, "secret": True, "config": "spotprice_ostrom_client_secret"}, + "ostrom_contract_id": {"required": False, "secret": True, "config": "spotprice_ostrom_contract_id"}, + "octopus_de_api_key": {"required": False, "secret": True, "config": "spotprice_octopus_de_api_key"}, + "octopus_de_account": {"required": False, "secret": True, "config": "spotprice_octopus_de_account"}, + "octopus_de_malo": {"required": False, "secret": True, "config": "spotprice_octopus_de_malo"}, + "ews_api_key": {"required": False, "secret": True, "config": "spotprice_ews_api_key"}, "markup": {"required": False, "config": "spotprice_markup", "default": 0.0}, + # No registry default: when unset, initialize() uses the provider's published percentage (3 for awattar), else 0 + "markup_percent": {"required": False, "config": "spotprice_markup_percent"}, "vat": {"required": False, "config": "spotprice_vat", "default": 0.0}, "charge_zones": {"required": False, "config": "spotprice_charge_zones", "default": []}, "exchange_rate": {"required": False, "config": "spotprice_exchange_rate", "default": 1.0}, @@ -601,9 +610,10 @@ def load_component_class(component_info): "automatic": {"required": False, "config": "spotprice_automatic", "default": True}, }, # The provider defaults to energycharts, so gate on what a real configuration must carry: a - # bidding zone for the spot sources, or a Tibber token. Without this the component would - # start on every install. - "required_or": ["zone", "tibber_token"], + # bidding zone for the spot sources, or a supplier's credentials (either half of a pair is + # enough to start it, so a missing other half is reported rather than silently ignored). + # Without this the component would start on every install. + "required_or": ["zone", "tibber_token", "ostrom_client_id", "ostrom_client_secret", "octopus_de_api_key", "ews_api_key"], "can_restart": True, "phase": 1, }, diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 26712c62c..4d09d53fa 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -2871,12 +2871,20 @@ "iboost_energy_today": {"type": "sensor", "sensor_type": "float"}, "metric_octopus_gas": {"type": "sensor", "sensor_type": "float"}, "rates_gas": {"type": "dict_list"}, - "spotprice_provider": {"type": "string", "empty": False, "allowed": ["entsoe", "energycharts", "tibber"]}, + "spotprice_provider": {"type": "string", "empty": False, "allowed": ["entsoe", "energycharts", "tibber", "ostrom", "octopus_de", "awattar", "ews"]}, "spotprice_zone": {"type": "string", "empty": False}, "spotprice_entsoe_token": {"type": "string", "empty": False}, "spotprice_tibber_token": {"type": "string", "empty": False}, "spotprice_tibber_home_id": {"type": "string", "empty": False}, + "spotprice_ostrom_client_id": {"type": "string", "empty": False}, + "spotprice_ostrom_client_secret": {"type": "string", "empty": False}, + "spotprice_ostrom_contract_id": {"type": "string|integer", "empty": False}, + "spotprice_octopus_de_api_key": {"type": "string", "empty": False}, + "spotprice_octopus_de_account": {"type": "string", "empty": False}, + "spotprice_octopus_de_malo": {"type": "string|integer", "empty": False}, + "spotprice_ews_api_key": {"type": "string", "empty": False}, "spotprice_markup": {"type": "float"}, + "spotprice_markup_percent": {"type": "float"}, "spotprice_vat": {"type": "float"}, "spotprice_charge_zones": {"type": "dict_list"}, "spotprice_exchange_rate": {"type": "float"}, diff --git a/apps/predbat/spotprice.py b/apps/predbat/spotprice.py index 9179bcf2f..196d6a7ca 100644 --- a/apps/predbat/spotprice.py +++ b/apps/predbat/spotprice.py @@ -16,22 +16,35 @@ that price from the spot market itself, so it works for any supplier with a spot-linked tariff rather than needing a per-supplier integration: - import rate = (spot EUR/MWh x exchange_rate / 10 + markup + charge_zone(t)) x (1 + vat) + import rate = (spot + |spot| x markup_percent / 100 + markup + charge_zone(t)) x (1 + vat) -which, with exchange_rate 1, is in euro cents per kWh. Markup and charge zones are entered in the +where spot = spot EUR/MWh x exchange_rate / 10, which, with exchange_rate 1, is in euro cents per kWh. Markup and charge zones are entered in the same minor currency units per kWh and exclude VAT; vat is a fraction (0.19 for 19%). Price sources, chosen by spotprice_provider: - entsoe: ENTSO-E Transparency Platform, document A44 (day-ahead prices). Needs a free security token. Falls back to Energy-Charts when it fails. - energycharts: Fraunhofer ISE Energy-Charts, no key needed. + - awattar: The aWATTar market data API (api.awattar.de / api.awattar.at), no key needed - the + hourly spot prices behind tado Energy's (formerly aWATTar's) HOURLY tariff for + DE-LU and AT. Falls back to ENTSO-E (with a token) and then Energy-Charts. Adds + spotprice_markup_percent (default 3 here) of the absolute spot price on top. - tibber: Tibber's own end-user price (already includes markup, grid fees and VAT), read with a personal access token. No markup or VAT is added on top. + - ostrom: Ostrom's end-user price for the contract's postcode (spot plus taxes, levies and + variable grid fees, VAT included), read with a client ID and secret from Ostrom's + developer portal. Used as-is, like Tibber. + - octopus_de: Octopus Energy Germany's unit rates for the account's active agreement (dynamic + forecast, time-of-use or fixed; VAT included), read from its Kraken GraphQL API + with an API key. Used as-is, like Tibber. + - ews: EWS Schönau's "Ökostrom Dynamisch" quarter-hourly gross prices (VAT, grid fees and + levies included), read from its price API with an API key EWS issues on request. + Used as-is, like Tibber. Export is either a fixed feed-in tariff or spot-linked (spot + export markup, no VAT), with an optional rule that pays nothing in any interval where the spot price is negative (Germany's Solarspitzengesetz). Spot data for the export side is fetched from ENTSO-E/Energy-Charts even when -Tibber supplies the import price. +a supplier (Tibber, Ostrom, Octopus Energy Germany, EWS) supplies the import price. Rates are published in the same shape as the Octopus/Kraken rate sensors (value_inc_vat, valid_from, valid_to) and wired in through metric_octopus_import / metric_octopus_export, so the @@ -40,8 +53,10 @@ import argparse import asyncio +import base64 import functools import hashlib +import json import re import sys import xml.etree.ElementTree as ET @@ -55,12 +70,47 @@ from mock_base import MockBase as SharedMockBase from predbat_metrics import record_api_call -SPOTPRICE_PROVIDERS = ("entsoe", "energycharts", "tibber") +SPOTPRICE_PROVIDERS = ("entsoe", "energycharts", "tibber", "ostrom", "octopus_de", "awattar", "ews") +# Providers whose import price is the market spot price plus the configured markup, charge zones and VAT +SPOT_PROVIDERS = ("entsoe", "energycharts", "awattar") +# Suppliers read through the generic "supplier" source: their API gives the end-user price in minor +# units per kWh with markup, grid fees, levies and VAT already in, so it is used as-is +SUPPLIER_PROVIDERS = ("ostrom", "octopus_de", "ews") +# Every provider whose import price comes from the supplier rather than from spot + markup +ALL_IN_PROVIDERS = ("tibber",) + SUPPLIER_PROVIDERS SPOTPRICE_EXPORT_MODES = ("none", "fixed", "spot") ENTSOE_URL = "https://web-api.tp.entsoe.eu/api" ENERGYCHARTS_URL = "https://api.energy-charts.info/price" +# aWATTar's public market data API, one host per country. tado Energy's HOURLY tariff (the former +# aWATTar tariff) is billed on these hourly prices. +AWATTAR_URLS = {"DE-LU": "https://api.awattar.de/v1/marketdata", "AT": "https://api.awattar.at/v1/marketdata"} +# The HOURLY tariff's published percentage markup, charged on the absolute spot price +AWATTAR_MARKUP_PERCENT = 3.0 +SOURCE_LABELS = {"entsoe": "ENTSO-E", "energycharts": "Energy-Charts", "awattar": "aWATTar"} TIBBER_URL = "https://api.tibber.com/v1-beta/gql" +OSTROM_AUTH_URL = "https://auth.production.ostrom-api.io/oauth2/token" +OSTROM_API_URL = "https://production.ostrom-api.io" +# Ostrom also runs a sandbox (auth.sandbox.ostrom-api.io / sandbox.ostrom-api.io) with test data only +OCTOPUS_DE_URL = "https://api.oeg-kraken.energy/v1/graphql/" +# Octopus Energy Germany's timeslot rules are German local times, whatever the Home Assistant timezone +OCTOPUS_DE_TIMEZONE = pytz.timezone("Europe/Berlin") +EWS_URL = "https://api.ews-schoenau.de/v1/dynamicprices/EWS-OEKO-DYN" +QUARTER_HOUR = timedelta(minutes=15) +# Kraken core (KT-CT) error codes, as documented in the Kraken GraphQL error code reference: a token +# the API no longer accepts - 1111 unauthorized, 1124 JWT signature expired, 1125 token keys rotated, +# 1143 Authorization header not a valid credential - is refreshed and the query retried once; +# credentials obtainKrakenToken rejects outright (1138 check the credentials, 1139 authentication +# failed); and the rate limit (1199 too many requests) +KRAKEN_TOKEN_ERROR_CODES = ("KT-CT-1111", "KT-CT-1124", "KT-CT-1125", "KT-CT-1143") +KRAKEN_CREDENTIAL_ERROR_CODES = ("KT-CT-1138", "KT-CT-1139") +KRAKEN_RATE_LIMIT_CODE = "KT-CT-1199" +# Seconds before an OAuth2 access token's stated expiry at which it is renewed +TOKEN_EXPIRY_MARGIN_SECONDS = 60 +# How often the Ostrom contract (and so the postcode priced) is looked up again when no contract is configured +OSTROM_CONTRACT_RECHECK = timedelta(days=1) +# /spot-prices replies that can mean the postcode no longer fits the account (a move): look the contract up again +OSTROM_CONTRACT_CHANGED_STATUSES = (400, 404, 422) # Bidding zone name -> ENTSO-E EIC area code. Names follow Energy-Charts' bzn codes so one setting # drives both sources. IE-SEM is ENTSO-E only (Energy-Charts does not publish it). @@ -124,7 +174,7 @@ # SEM and the Iberian MIBEL, whose day is 23:00-23:00 in Irish/Portuguese local time - so "tomorrow's # prices are in" and the fetch window are judged on this clock, not the user's. MARKET_TIMEZONE = pytz.timezone("Europe/Brussels") -# Salt for the one-way digest of the Tibber credential in cache filenames +# Salt for the one-way digest of supplier credentials in cache filenames CACHE_DIGEST_SALT = "predbat-spotprice-cache-v1|" # Granularity used to detect overlapping intervals between series of different resolutions. OVERLAP_TICK_MINUTES = 5 @@ -137,7 +187,12 @@ class SpotPriceError(Exception): - """A price source could not supply prices; the message says why.""" + """A price source could not supply prices; the message says why. status is the HTTP status when one caused it.""" + + def __init__(self, message, status=None): + """Store the message and the HTTP status, if any.""" + super().__init__(message) + self.status = status def parse_iso_duration_minutes(text): @@ -419,6 +474,54 @@ def parse_energycharts_json(data): return intervals_from_starts(pairs) +def average_to_hourly(intervals): + """Average finer intervals to whole UTC hours (the zones here are whole hours off UTC, so local hours too). + + An hour is published only when its intervals cover all 60 minutes; a partly covered hour is left + out rather than priced from part of it. Hourly or coarser intervals pass through unchanged. + """ + hours = {} + passthrough = [] + for start, end, value in intervals: + if end - start >= HOUR: + passthrough.append((start, end, value)) + continue + hour = start.replace(minute=0, second=0, microsecond=0) + hours.setdefault(hour, []).append((start, end, value)) + result = list(passthrough) + for hour, parts in hours.items(): + minutes = sum((end - start).total_seconds() / 60 for start, end, _value in parts) + if any(end > hour + HOUR for _start, end, _value in parts) or abs(minutes - 60) > 1e-6: + continue + average = sum(value * (end - start).total_seconds() for start, end, value in parts) / 3600.0 + result.append((hour, hour + HOUR, round(average, 4))) + return sorted(result, key=lambda item: item[0]) + + +@parse_errors_as("aWATTar") +def parse_awattar_json(data): + """Parse an aWATTar /v1/marketdata response into (start, end, price per MWh) UTC intervals. + + Every entry carries its own start and end (milliseconds since the epoch), so intervals keep the + length aWATTar gives them. Only prices in EUR/MWh are accepted. + """ + if not isinstance(data, dict) or not isinstance(data.get("data"), list): + raise SpotPriceError("aWATTar returned an unexpected response") + intervals = [] + for entry in data["data"]: + unit = str(entry.get("unit") or "Eur/MWh").replace(" ", "").lower() + if unit != "eur/mwh": + raise SpotPriceError("aWATTar returned prices in {}, expected Eur/MWh".format(entry.get("unit"))) + if entry.get("marketprice") is None: + continue + start = datetime.fromtimestamp(int(entry["start_timestamp"]) / 1000, tz=timezone.utc) + end = datetime.fromtimestamp(int(entry["end_timestamp"]) / 1000, tz=timezone.utc) + if end <= start: + raise SpotPriceError("aWATTar returned an interval ending before it starts") + intervals.append((start, end, float(entry["marketprice"]))) + return resolve_overlaps([(0, 0, intervals)]) + + @parse_errors_as("Tibber") def parse_tibber_json(data, home_id=None, resolution_minutes=None): """Parse a Tibber priceInfo response into (start, end, total per kWh in major units) UTC intervals. @@ -460,6 +563,356 @@ def parse_tibber_json(data, home_id=None, resolution_minutes=None): return intervals_from_starts(pairs, expected), currency, home.get("id") +@parse_errors_as("EWS") +def parse_ews_json(data): + """Parse an EWS dynamic prices response into (start, end, cents per kWh incl. VAT) UTC intervals. + + The reply has today and tomorrow lists of {startsAt, total}, one per quarter hour, total being + the gross price in cents per kWh. A missing total leaves a gap. As the response format is only + known from third-party use, a reply whose typical price is below 1 is rejected rather than read + as cents: it would mean the unit is euros, and the prices would be a hundred times too low. + """ + if not isinstance(data, dict): + raise SpotPriceError("EWS returned an unexpected response") + pairs = [] + for day in ("today", "tomorrow"): + entries = data.get(day) or [] + if not isinstance(entries, list): + raise SpotPriceError("EWS returned an unexpected {} list".format(day)) + for entry in entries: + starts_at = entry["startsAt"] + if not isinstance(starts_at, str): + raise SpotPriceError("EWS returned a startsAt of type {}".format(type(starts_at).__name__)) + total = entry.get("total") + pairs.append((parse_utc(starts_at), None if total is None else round(float(total), 4))) + values = sorted(value for _start, value in pairs if value is not None) + if values and values[len(values) // 2] < 1.0: + raise SpotPriceError("EWS prices look like euros rather than cents per kWh (typical value {})".format(values[len(values) // 2])) + return intervals_from_starts(pairs, QUARTER_HOUR) + + +@parse_errors_as("Ostrom") +def select_ostrom_contract(data, contract_id=None, warn=None): + """Pick the contract to price from an Ostrom /contracts response. + + Only ACTIVE contracts are priced - a terminated or pending one is never used as a stand-in. + contract_id (spotprice_ostrom_contract_id) chooses one explicitly, and must itself be active; + otherwise there must be exactly one active contract. The status is matched case-insensitively. + Only when no contract carries a status at all is the single dynamic contract used without one, + reported through warn(message). The contract must be a + dynamic product (SIMPLY_DYNAMIC and its successors) with a postcode: the postcode is what makes + /spot-prices include the local taxes, levies and grid fees. Returns the contract dict. + """ + if not isinstance(data, dict) or not isinstance(data.get("data"), list): + raise SpotPriceError("Ostrom returned an unexpected contracts response") + contracts = [contract for contract in data["data"] if isinstance(contract, dict)] + if not contracts: + raise SpotPriceError("Ostrom account has no contracts") + ids = ", ".join(str(contract.get("id")) for contract in contracts) + + def is_active(contract): + """True for a contract whose status is ACTIVE, in any case.""" + return str(contract.get("status") or "").strip().upper() == "ACTIVE" + + def is_dynamic(contract): + """True for a dynamic product.""" + return "DYNAMIC" in str(contract.get("productCode") or "").upper() + + status_known = any("status" in contract for contract in contracts) + if not status_known: + # The status field is missing altogether: fall back to the one dynamic contract, never to a guess among several + dynamic = [contract for contract in contracts if is_dynamic(contract)] + if contract_id not in (None, ""): + dynamic = [contract for contract in dynamic if str(contract.get("id")) == str(contract_id).strip()] + if len(dynamic) != 1: + raise SpotPriceError("Ostrom contracts carry no status, so the active one cannot be told apart ({})".format(ids)) + if warn: + warn("Ostrom contracts carry no status field - using the only dynamic contract {}".format(dynamic[0].get("id"))) + contracts = dynamic + contract_id = None + if contract_id not in (None, ""): + matches = [contract for contract in contracts if str(contract.get("id")) == str(contract_id).strip()] + if not matches: + raise SpotPriceError("Ostrom contract {} not found, the account has {}".format(contract_id, ids)) + contract = matches[0] + if not is_active(contract): + raise SpotPriceError("Ostrom contract {} is {}, not active".format(contract.get("id"), contract.get("status") or "of unknown status")) + else: + active = [contract for contract in contracts if is_active(contract) or not status_known] + if not active: + raise SpotPriceError("no active Ostrom contract, the account has {}".format(", ".join("{} ({})".format(c.get("id"), c.get("status")) for c in contracts))) + if len(active) > 1: + raise SpotPriceError("Ostrom account has several active contracts ({}), set spotprice_ostrom_contract_id".format(", ".join(str(c.get("id")) for c in active))) + contract = active[0] + product = str(contract.get("productCode") or "") + if "DYNAMIC" not in product.upper(): + raise SpotPriceError("Ostrom contract {} is {}, not a dynamic tariff - set its fixed price with rates_import instead".format(contract.get("id"), product or "an unknown product")) + if not str((contract.get("address") or {}).get("zip") or "").strip(): + raise SpotPriceError("Ostrom contract {} has no postcode, so its local taxes and grid fees cannot be priced".format(contract.get("id"))) + return contract + + +@parse_errors_as("Ostrom") +def parse_ostrom_json(data): + """Parse an Ostrom /spot-prices response into (start, end, cents per kWh incl. VAT) UTC intervals. + + Each entry gives an hour from its date. Ostrom documents the end-user's variable price per kWh + as grossKwhPrice (spot, with VAT) + grossKwhTaxAndLevies (taxes, levies and variable grid fees + for the postcode, with VAT); Ostrom's own margin is a monthly fee, not a per-kWh charge. Without + a postcode Ostrom returns the taxes and levies as zero, which would leave only the bare spot + price - so a response where every entry has zero taxes and levies is rejected. + """ + if not isinstance(data, dict) or not isinstance(data.get("data", []), list): + raise SpotPriceError("Ostrom returned an unexpected prices response") + pairs = [] + any_levies = False + for entry in data.get("data") or []: + start = parse_utc(entry["date"]) + price = entry.get("grossKwhPrice") + levies = entry.get("grossKwhTaxAndLevies") + if price is None or levies is None: + pairs.append((start, None)) + continue + if float(levies) != 0: + any_levies = True + pairs.append((start, round(float(price) + float(levies), 4))) + if pairs and not any_levies: + raise SpotPriceError("Ostrom returned prices without taxes, levies or grid fees (no postcode applied)") + return intervals_from_starts(pairs, HOUR) + + +# Octopus Energy Germany (Kraken). The API key is exchanged for a JWT; the account's agreements live +# under properties -> electricityMalos (German market locations). Every query here has been checked +# against the live schema: Kraken validates a query before it looks at the Authorization header. +OCTOPUS_DE_TOKEN_MUTATION = "mutation ObtainToken($input: ObtainJSONWebTokenInput!) { obtainKrakenToken(input: $input) { token } }" +OCTOPUS_DE_ACCOUNTS_QUERY = "query { viewer { accounts { number } } }" +OCTOPUS_DE_RATE_FIELDS = ( + "__typename " "... on SimpleProductUnitRateInformation { latestGrossUnitRateCentsPerKwh } " "... on TimeOfUseProductUnitRateInformation { rates { latestGrossUnitRateCentsPerKwh timeslotActivationRules { activeFromTime activeToTime } } }" +) +OCTOPUS_DE_AGREEMENTS_QUERY = ( + "query Agreements($accountNumber: String!) { account(accountNumber: $accountNumber) { properties { electricityMalos { maloNumber agreements { " + "id isActive isRevoked validFrom validTo product { code displayName fullName isTimeOfUse termsContractType } " + "unitRateInformation { " + OCTOPUS_DE_RATE_FIELDS + " } " + "unitRateForecast { validFrom validTo unitRateInformation { " + OCTOPUS_DE_RATE_FIELDS + " } } " + "} } } } }" +) + + +def kraken_error_codes(body): + """The errorCode of every GraphQL error in a Kraken response.""" + if not isinstance(body, dict): + return [] + return [((error.get("extensions") or {}).get("errorCode") if isinstance(error, dict) else None) for error in body.get("errors") or []] + + +def kraken_error_text(body): + """A short description of the GraphQL errors in a Kraken response.""" + parts = [] + for error in body.get("errors") or [] if isinstance(body, dict) else []: + if isinstance(error, dict): + extensions = error.get("extensions") or {} + parts.append("{}{}".format(extensions.get("errorDescription") or error.get("message") or "error", " ({})".format(extensions["errorCode"]) if extensions.get("errorCode") else "")) + else: + parts.append(str(error)) + return "; ".join(parts) or "unknown error" + + +def jwt_expiry(token): + """The exp claim of a JWT as an aware UTC datetime, or None when it cannot be read. The signature is not checked.""" + try: + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + claims = json.loads(base64.urlsafe_b64decode(payload.encode("ascii"))) + return datetime.fromtimestamp(int(claims["exp"]), tz=timezone.utc) + except Exception: + return None + + +def select_octopus_de_agreements(data, malo=None): + """Pick the market location to price and return (malo number, its agreements in date order). + + malo (spotprice_octopus_de_malo) chooses a market location explicitly. Otherwise the account + must have exactly one market location with an active electricity agreement: two (a household + meter and a heat pump meter, say) are an error naming them, since pricing either could be wrong. + The agreements returned are the active one followed by its successors, each the single + non-revoked agreement starting at (or first after) the previous one's end, so a tariff change + inside the fetch window hands over and no two agreements ever cover the same instant. Raises + SpotPriceError. + """ + if not isinstance(data, dict): + raise SpotPriceError("Octopus Energy Germany returned an unexpected agreements response") + account = (data.get("data") or {}).get("account") + if not isinstance(account, dict): + raise SpotPriceError("Octopus Energy Germany account not found") + candidates = [] + for prop in account.get("properties") or []: + for location in (prop or {}).get("electricityMalos") or []: + agreements = [agreement for agreement in (location or {}).get("agreements") or [] if isinstance(agreement, dict)] + active = [agreement for agreement in agreements if agreement.get("isActive")] + if active: + candidates.append((str(location.get("maloNumber")), agreements, active)) + if malo not in (None, ""): + matches = [candidate for candidate in candidates if candidate[0] == str(malo).strip()] + if not matches: + raise SpotPriceError("Octopus Energy Germany market location {} has no active agreement, the account has {}".format(malo, ", ".join(c[0] for c in candidates) or "none")) + candidates = matches + if not candidates: + raise SpotPriceError("Octopus Energy Germany account has no active electricity agreement") + if len(candidates) > 1: + raise SpotPriceError("Octopus Energy Germany account has active agreements on several market locations ({}), set spotprice_octopus_de_malo".format(", ".join(c[0] for c in candidates))) + number, agreements, active = candidates[0] + if len(active) > 1: + raise SpotPriceError("Octopus Energy Germany market location {} has several active agreements ({})".format(number, ", ".join(str(a.get("id")) for a in active))) + chain = [active[0]] + # At each handover take the one agreement starting at (or first after) the current one's end, so at + # most one agreement covers any instant; two claiming the same start cannot be told apart + while chain[-1].get("validTo"): + current_end = parse_utc(chain[-1]["validTo"]) + following = [agreement for agreement in agreements if agreement not in chain and not agreement.get("isRevoked") and agreement.get("validFrom") and parse_utc(agreement["validFrom"]) >= current_end] + if not following: + break + first_start = min(parse_utc(agreement["validFrom"]) for agreement in following) + next_agreements = [agreement for agreement in following if parse_utc(agreement["validFrom"]) == first_start] + if len(next_agreements) > 1: + raise SpotPriceError("Octopus Energy Germany market location {} has several agreements starting {} ({})".format(number, first_start.isoformat(), ", ".join(str(a.get("id")) for a in next_agreements))) + chain.append(next_agreements[0]) + return number, chain + + +def _parse_time_of_day(text): + """Parse a Kraken HH:MM[:SS] time of day into minutes after midnight.""" + minutes = parse_hhmm(text) + if minutes is None: + raise SpotPriceError("Octopus Energy Germany returned an unreadable time of day {}".format(text)) + return minutes + + +def _rate_value(rate): + """The gross (VAT included) cents per kWh of one rate, or None when it has none.""" + value = rate.get("latestGrossUnitRateCentsPerKwh") + return None if value in (None, "") else float(value) + + +def _localize(naive, tz): + """Attach a timezone to a naive local datetime and convert it to UTC. + + A wall time the autumn change repeats (02:00-03:00) maps to its first occurrence, so every rule + boundary at that time lands on the same instant and a 02:00-03:00 rule takes both repeated hours + rather than leaving the first to the rule before it. A wall time spring skips maps to the moment + the clocks jump, as if it were still standard time. + """ + if not hasattr(tz, "localize"): + return naive.replace(tzinfo=tz).astimezone(timezone.utc) + try: + local = tz.localize(naive, is_dst=None) + except pytz.exceptions.AmbiguousTimeError: + local = tz.localize(naive, is_dst=True) + except pytz.exceptions.NonExistentTimeError: + local = tz.localize(naive, is_dst=False) + return local.astimezone(timezone.utc) + + +def _expand_rate_info(info, start, end, tz, what): + """Expand one unitRateInformation over [start, end) into (start, end, cents) intervals. + + A Simple rate covers the whole period. A time-of-use rate repeats each rate's timeslot activation + rules (local times in tz) every day - an end at or before the start wraps past midnight, so 00:00 + is the end of the day; a single time-of-use rate without rules covers the whole period. Several + rates where any lacks rules cannot be placed in time and are an error, as is a rate with no price. + """ + if info.get("__typename") == "TimeOfUseProductUnitRateInformation" or "rates" in info: + rates = info.get("rates") or [] + if not rates: + raise SpotPriceError("Octopus Energy Germany {} has a time-of-use rate with no timeslots".format(what)) + if len(rates) == 1 and not rates[0].get("timeslotActivationRules"): + info = rates[0] + else: + slots = [] + for rate in rates: + value = _rate_value(rate) + rules = rate.get("timeslotActivationRules") or [] + if value is None or not rules: + raise SpotPriceError("Octopus Energy Germany {} has a time-of-use rate without a price or timeslot".format(what)) + for rule in rules: + slots.append((_parse_time_of_day(rule.get("activeFromTime")), _parse_time_of_day(rule.get("activeToTime")), value)) + intervals = [] + day = start.astimezone(tz).date() - timedelta(days=1) + last_day = end.astimezone(tz).date() + while day <= last_day: + for from_minute, to_minute, value in slots: + span = (to_minute - from_minute) % (24 * 60) or 24 * 60 + local_start = datetime(day.year, day.month, day.day) + timedelta(minutes=from_minute) + slot_start = max(_localize(local_start, tz), start) + slot_end = min(_localize(local_start + timedelta(minutes=span), tz), end) + if slot_start < slot_end: + intervals.append((slot_start, slot_end, round(value, 4))) + day += timedelta(days=1) + return intervals + value = _rate_value(info) + if value is None: + raise SpotPriceError("Octopus Energy Germany {} has no unit rate".format(what)) + return [(start, end, round(value, 4))] + + +def octopus_de_is_dynamic(agreement): + """True for a dynamic (spot-linked) agreement. + + The schema has no market-linked flag, so this is the forecast itself or a product described as + dynamic in its code, names or contract type. parse_octopus_de_agreement does not rely on this + alone: without a forecast it prices only an explicit fixed or time-of-use rate. + """ + if agreement.get("unitRateForecast"): + return True + product = agreement.get("product") or {} + text = " ".join(str(product.get(field) or "") for field in ("code", "displayName", "fullName", "termsContractType")).upper() + return "DYNAMIC" in text or "DYNAMISCH" in text + + +@parse_errors_as("Octopus Energy Germany") +def parse_octopus_de_agreement(agreement, start, end, tz=OCTOPUS_DE_TIMEZONE): + """Turn an Octopus Energy Germany agreement into (start, end, cents per kWh incl. VAT) UTC intervals over [start, end). + + The window is first clipped to the agreement's own validity. A dynamic agreement is priced only + from its unitRateForecast, each entry expanded over its own period (and clipped to the window and + the agreement); with no forecast yet it gives no intervals, never a flat price - the flat + unitRateInformation of a dynamic agreement is not the price of any particular interval. A fixed + or time-of-use agreement expands its unitRateInformation over the window. + """ + valid_from = parse_utc(agreement["validFrom"]) if agreement.get("validFrom") else None + valid_to = parse_utc(agreement["validTo"]) if agreement.get("validTo") else None + if valid_from and valid_from > start: + start = valid_from + if valid_to and valid_to < end: + end = valid_to + if start >= end: + return [] + what = "agreement {}".format(agreement.get("id")) + if not octopus_de_is_dynamic(agreement): + info = agreement.get("unitRateInformation") or {} + typename = info.get("__typename") + is_time_of_use = (agreement.get("product") or {}).get("isTimeOfUse") + rates = info.get("rates") or [] + if typename == "SimpleProductUnitRateInformation" and not is_time_of_use: + explicit = True + elif typename == "TimeOfUseProductUnitRateInformation": + explicit = bool(rates) and all(rate.get("timeslotActivationRules") for rate in rates) + else: + explicit = False + if not explicit: + # Without a forecast only an explicit fixed rate or a fully timetabled time-of-use rate is a + # price for every interval; anything else may be a dynamic tariff's placeholder + raise SpotPriceError("Octopus Energy Germany {} has no price forecast and no explicit fixed or time-of-use rates".format(what)) + return sorted(_expand_rate_info(info, start, end, tz, what), key=lambda item: item[0]) + intervals = [] + for entry in agreement.get("unitRateForecast") or []: + slot_start = max(parse_utc(entry["validFrom"]), start) + slot_end = min(parse_utc(entry["validTo"]), end) + if slot_start >= slot_end: + continue + intervals.extend(_expand_rate_info(entry.get("unitRateInformation") or {}, slot_start, slot_end, tz, what + " forecast")) + return sorted(intervals, key=lambda item: item[0]) + + def parse_hhmm(text): """Parse HH:MM or HH:MM:SS into minutes after midnight (24:00 allowed), or None.""" if text is None: @@ -575,9 +1028,14 @@ def charge_zone_rate(zones, local_time): return 0.0 -def spot_import_rate(spot_mwh, markup, zone_charge, vat, exchange_rate=1.0): - """Price formula: (spot per MWh x exchange_rate / 10 + markup + zone_charge) x (1 + vat), minor units per kWh; vat is a fraction.""" - return round((spot_mwh * exchange_rate / 10.0 + markup + zone_charge) * (1.0 + vat), 4) +def spot_import_rate(spot_mwh, markup, zone_charge, vat, exchange_rate=1.0, markup_percent=0.0): + """Price formula: (spot + |spot| x markup_percent / 100 + markup + zone_charge) x (1 + vat), minor units per kWh. + + spot is the price per MWh x exchange_rate / 10; vat is a fraction. The percentage is charged on + the absolute spot price, so it is a cost in negative hours too. + """ + spot = spot_mwh * exchange_rate / 10.0 + return round((spot + abs(spot) * markup_percent / 100.0 + markup + zone_charge) * (1.0 + vat), 4) def spot_export_rate(spot_mwh, export_markup, exchange_rate=1.0): @@ -607,7 +1065,7 @@ def deserialise_intervals(data): class SpotPriceAPI(ComponentBase): - """Builds import/export rates from day-ahead spot prices (ENTSO-E, Energy-Charts) or Tibber.""" + """Builds import/export rates from day-ahead spot prices (ENTSO-E, Energy-Charts) or a supplier's own prices.""" def initialize( self, @@ -616,7 +1074,15 @@ def initialize( entsoe_token=None, tibber_token=None, tibber_home_id=None, + ostrom_client_id=None, + ostrom_client_secret=None, + ostrom_contract_id=None, + octopus_de_api_key=None, + octopus_de_account=None, + octopus_de_malo=None, + ews_api_key=None, markup=0.0, + markup_percent=None, vat=0.0, charge_zones=None, exchange_rate=1.0, @@ -627,21 +1093,37 @@ def initialize( automatic=True, ): """Store configuration and validate it. Problems are logged once here, not on every cycle.""" + self.tibber_token = tibber_token + self.ostrom_client_id = ostrom_client_id + self.ostrom_client_secret = ostrom_client_secret + self.octopus_de_api_key = octopus_de_api_key + self.ews_api_key = ews_api_key + ambiguous = None if not provider: - # Unset: any Tibber token means Tibber - even when a zone is also set, since the zone may - # only be there for spot-linked export - otherwise the keyless default - provider = "tibber" if tibber_token else "energycharts" + # Unset: the one supplier whose credentials are present - even when a zone is also set, + # since the zone may only be there for spot-linked export - otherwise the keyless default. + # Credentials for more than one supplier are a configuration error rather than a guess. + suppliers = self.suppliers_with_credentials() + if len(suppliers) > 1: + ambiguous = "credentials for several suppliers ({}) are set, set spotprice_provider to choose one".format(", ".join(suppliers)) + provider = suppliers[0] if len(suppliers) == 1 else "energycharts" self.provider = str(provider).strip().lower() if self.provider not in SPOTPRICE_PROVIDERS: self.log("Warn: SpotPrice: unknown spotprice_provider '{}', expected one of {} - using energycharts".format(provider, ", ".join(SPOTPRICE_PROVIDERS))) self.provider = "energycharts" self.zone, self.zone_eic = self.resolve_zone(zone) self.entsoe_token = entsoe_token - self.tibber_token = tibber_token self.tibber_home_id = tibber_home_id # The configured home, kept apart from one picked at run time, so the cache name is stable self.tibber_home_id_configured = tibber_home_id + self.ostrom_contract_id = ostrom_contract_id + self.octopus_de_account = str(octopus_de_account).strip() if octopus_de_account else None + # The configured account, kept apart from one looked up at run time, so the cache name is stable + self.octopus_de_account_configured = self.octopus_de_account + self.octopus_de_malo = str(octopus_de_malo).strip() if octopus_de_malo else None self.markup = self.to_float(markup, "spotprice_markup") + # Unset: the published percentage of the provider's tariff (aWATTar/tado HOURLY), else none + self.markup_percent = self.to_float(markup_percent, "spotprice_markup_percent", default=AWATTAR_MARKUP_PERCENT if self.provider == "awattar" else 0.0) self.vat = self.to_float(vat, "spotprice_vat") if self.vat >= 1: # spotprice_vat is a fraction (0.19); 1 or more can only have meant a percentage (1 = 1%, not 100%) @@ -658,13 +1140,15 @@ def initialize( self.export_zero_on_negative = bool(export_zero_on_negative) self.automatic = bool(automatic) + if self.provider == "awattar" and (self.zone or self.zone_eic) and self.zone not in AWATTAR_URLS: + self.log("Warn: SpotPrice: aWATTar publishes prices for {} only, not {} - using {}".format(" and ".join(AWATTAR_URLS), self.zone or self.zone_eic, "ENTSO-E" if self.entsoe_token else "Energy-Charts")) if self.provider == "entsoe" and not self.entsoe_token and (self.energycharts_covers_zone() or not (self.zone or self.zone_eic)): # Only where Energy-Charts can actually stand in; an ENTSO-E-only zone gets the config error below instead self.log("Warn: SpotPrice: spotprice_provider is entsoe but spotprice_entsoe_token is not set - using Energy-Charts") if self.provider == "tibber" and not self.tibber_token: self.log("Warn: SpotPrice: spotprice_provider is tibber but spotprice_tibber_token is not set") - if self.provider == "tibber" and self.needs_spot() and not (self.zone or self.zone_eic): - # Tibber only needs spot prices for the export side; without a zone they cannot be fetched, + if self.provider in ALL_IN_PROVIDERS and self.needs_spot() and not (self.zone or self.zone_eic): + # A supplier's price only needs spot prices for the export side; without a zone they cannot be fetched, # so switch that export off once here (leaving rates_export in charge) rather than failing # every refresh self.log( @@ -681,11 +1165,11 @@ def initialize( # When ENTSO-E is not usable either there is nothing to fetch, ever: say so once and stop, # rather than failing every refresh with "Energy-Charts does not publish zone ...". self.config_error = None - entsoe_usable = bool(self.entsoe_token) and self.provider in ("entsoe", "tibber") + entsoe_usable = bool(self.entsoe_token) and self.provider != "energycharts" if self.needs_spot() and (self.zone or self.zone_eic) and not self.energycharts_covers_zone() and not entsoe_usable: zone_name = self.zone or self.zone_eic - if self.provider == "tibber": - # Only the export side needs spot prices - keep Tibber's import prices and drop that export + if self.provider in ALL_IN_PROVIDERS: + # Only the export side needs spot prices - keep the supplier's import prices and drop that export self.log("Warn: SpotPrice: spotprice_entsoe_token is required for spot prices in zone {} - spot-linked export disabled".format(zone_name)) self.export_mode = "none" self.export_zero_on_negative = False @@ -695,18 +1179,36 @@ def initialize( self.config_error = "Energy-Charts does not publish zone {}, use spotprice_provider entsoe with spotprice_entsoe_token".format(zone_name) if self.config_error: self.log("Error: SpotPrice: {}".format(self.config_error)) + if not self.config_error: + self.config_error = ambiguous or self.supplier_config_error() + if self.config_error: + self.log("Error: SpotPrice: {}".format(self.config_error)) # Raw source data, cached to storage. Spot intervals are per-MWh in EUR as published; - # Tibber intervals are end-user totals in major currency units per kWh. + # Tibber intervals are end-user totals in major currency units per kWh; supplier intervals + # are end-user totals in minor units per kWh (euro cents), VAT included. self.spot_intervals = [] self.spot_source = None self.tibber_intervals = [] self.tibber_currency = None - self.fetched = {"spot": None, "tibber": None} + self.supplier_intervals = [] + self.fetched = {"spot": None, "tibber": None, "supplier": None} self.source_errors = {} - # Back-off is per source: a spot outage must never hold back Tibber's poll for tomorrow - self.source_failures = {"spot": 0, "tibber": 0} - self.source_next_attempt = {"spot": None, "tibber": None} + # Back-off is per source: a spot outage must never hold back a supplier's poll for tomorrow + self.source_failures = {"spot": 0, "tibber": 0, "supplier": 0} + self.source_next_attempt = {"spot": None, "tibber": None, "supplier": None} + # Supplier session state, kept in memory only: OAuth2 access tokens are renewed shortly before + # they expire, and the contract lookup is done once per run + self.ostrom_access_token = None + self.ostrom_token_expiry = None + self.ostrom_contract = None + self.ostrom_contract_checked = None + self.octopus_de_token = None + self.octopus_de_token_expiry = None + # Which contract / market location the supplier prices are for: detected at run time, and as cached + self.supplier_identity = None + self.cached_supplier_identity = None + self.warned = set() self.last_error = None self.import_rates = [] self.export_rates = [] @@ -714,6 +1216,46 @@ def initialize( self.import_wired = False self.export_wired = False + def warn_once(self, message): + """Log a warning the first time it is seen this run.""" + if message not in self.warned: + self.warned.add(message) + self.log("Warn: SpotPrice: {}".format(message)) + + def note_supplier_identity(self, identity): + """Record which contract or market location the supplier prices are for. + + Cached prices carry the identity they were fetched for; when the one detected now differs (a new + contract, another account or market location) the restored prices are dropped instead of being + published as this one's. + """ + if self.cached_supplier_identity is not None and self.cached_supplier_identity != identity: + self.log("Warn: SpotPrice: cached prices were for {}, not {} - discarding them".format(self.cached_supplier_identity, identity)) + self.supplier_intervals = [] + self.fetched["supplier"] = None + self.cached_supplier_identity = identity + self.supplier_identity = identity + + def suppliers_with_credentials(self): + """The suppliers that have any of their credentials set, in SPOTPRICE_PROVIDERS order.""" + present = { + "tibber": bool(self.tibber_token), + "ostrom": bool(self.ostrom_client_id or self.ostrom_client_secret), + "octopus_de": bool(self.octopus_de_api_key), + "ews": bool(self.ews_api_key), + } + return [name for name in SPOTPRICE_PROVIDERS if present.get(name)] + + def supplier_config_error(self): + """A configuration problem that stops the chosen supplier from ever fetching, or None.""" + if self.provider == "ostrom" and not (self.ostrom_client_id and self.ostrom_client_secret): + return "spotprice_ostrom_client_id and spotprice_ostrom_client_secret are both required for provider ostrom" + if self.provider == "octopus_de" and not self.octopus_de_api_key: + return "spotprice_octopus_de_api_key is required for provider octopus_de" + if self.provider == "ews" and not self.ews_api_key: + return "spotprice_ews_api_key is required for provider ews (ask EWS for one at api@ews-schoenau.de)" + return None + def to_float(self, value, name, default=0.0): """Coerce a numeric setting to float, logging and falling back to default when it is not one.""" if value is None: @@ -744,7 +1286,7 @@ def energycharts_covers_zone(self): def needs_spot(self): """True when spot prices are needed: for the import price, spot-linked export, or the negative-price export rule.""" - if self.provider in ("entsoe", "energycharts"): + if self.provider in SPOT_PROVIDERS: return True return self.export_mode == "spot" or (self.export_mode == "fixed" and self.export_zero_on_negative) @@ -809,6 +1351,17 @@ async def http_get(self, url, params, expect_json): body = await response.text() return response.status, body + async def http_request(self, method, url, params=None, headers=None, data=None): + """Send a request with optional headers and form data, returning (status, parsed JSON body or None).""" + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.request(method, url, params=params, headers=headers, data=data) as response: + try: + body = await response.json(content_type=None) + except Exception: + body = None + return response.status, body + async def http_post_json(self, url, payload, headers): """POST JSON, returning (status, parsed JSON body or None).""" timeout = aiohttp.ClientTimeout(total=30) @@ -900,16 +1453,52 @@ async def fetch_energycharts(self, start, end): record_api_call("energycharts") return intervals + async def fetch_awattar(self, start, end): + """Fetch hourly spot prices from aWATTar for [start, end). Returns intervals or raises SpotPriceError.""" + url = AWATTAR_URLS.get(self.zone) + if not url: + raise SpotPriceError("aWATTar does not publish zone {}".format(self.zone or self.zone_eic)) + params = {"start": int(start.timestamp() * 1000), "end": int(end.timestamp() * 1000)} + try: + status, body = await self.http_get(url, params, expect_json=True) + except NETWORK_ERRORS as e: + record_api_call("awattar", False, "connection_error") + raise SpotPriceError("aWATTar request failed: {}".format(e)) + except Exception as e: + record_api_call("awattar", False, "decode_error") + raise SpotPriceError("aWATTar response could not be read ({}: {})".format(type(e).__name__, e)) + if status == 429: + record_api_call("awattar", False, "rate_limit") + raise SpotPriceError("aWATTar rate limit hit (HTTP 429)") + if status != 200: + record_api_call("awattar", False, "server_error" if status >= 500 else "client_error") + raise SpotPriceError("aWATTar returned HTTP {}".format(status)) + try: + intervals = parse_awattar_json(body) + except SpotPriceError: + record_api_call("awattar", False, "decode_error") + raise + intervals = [item for item in intervals if item[0] < end and item[1] > start] + if not intervals: + record_api_call("awattar", False, "client_error") + raise SpotPriceError("aWATTar has no prices for {} in this period".format(self.zone)) + record_api_call("awattar") + return intervals + async def fetch_spot(self, start, end): - """Fetch spot prices from the configured source, falling back from ENTSO-E to Energy-Charts. + """Fetch spot prices from the configured source, falling back from aWATTar to ENTSO-E to Energy-Charts. The fallback only runs one way. Provider entsoe tries ENTSO-E (when a token is set - initialize() has already warned once if not) and then Energy-Charts. Provider energycharts uses Energy-Charts only, even if a token is present. Provider tibber, whose spot prices only feed the export side, - behaves like entsoe when a token is set and like energycharts otherwise. Returns (intervals, source). + behaves like entsoe when a token is set and like energycharts otherwise, as do the other + suppliers (ostrom, octopus_de, ews). Provider awattar tries aWATTar first, then carries on as entsoe. + Returns (intervals, source). """ sources = [] - if self.entsoe_token and self.provider in ("entsoe", "tibber"): + if self.provider == "awattar" and self.zone in AWATTAR_URLS: + sources.append(("awattar", self.fetch_awattar)) + if self.entsoe_token and self.provider != "energycharts": sources.append(("entsoe", self.fetch_entsoe)) if self.energycharts_covers_zone(): # Energy-Charts does not publish every zone (IE-SEM); there the ENTSO-E error stands alone @@ -921,12 +1510,13 @@ async def fetch_spot(self, start, end): except SpotPriceError as e: errors.append(str(e)) continue - if name == "entsoe" and self.entsoe_fallback_logged: - self.log("SpotPrice: ENTSO-E is working again") + primary = SOURCE_LABELS[sources[0][0]] + if name == sources[0][0] and self.entsoe_fallback_logged: + self.log("SpotPrice: {} is working again".format(primary)) self.entsoe_fallback_logged = False elif errors and not self.entsoe_fallback_logged: - # Logged once per outage rather than on every refresh while ENTSO-E stays down - self.log("Warn: SpotPrice: {}, using {} until ENTSO-E recovers".format("; ".join(errors), name)) + # Logged once per outage rather than on every refresh while the first source stays down + self.log("Warn: SpotPrice: {}, using {} until {} recovers".format("; ".join(errors), name, primary)) self.entsoe_fallback_logged = True return intervals, name raise SpotPriceError("; ".join(errors)) @@ -979,6 +1569,278 @@ async def fetch_tibber(self): record_api_call("tibber", False, "decode_error") raise last_error + async def supplier_request(self, service, label, method, url, params=None, headers=None, data=None): + """One supplier HTTP request with the shared failure handling. Returns (status, body). + + Network failures become connection_error and anything else raised while sending or decoding + becomes decode_error, both as SpotPriceError. Status codes are left to the caller. + """ + try: + return await self.http_request(method, url, params=params, headers=headers, data=data) + except NETWORK_ERRORS as e: + record_api_call(service, False, "connection_error") + raise SpotPriceError("{} request failed: {}".format(label, e)) + except Exception as e: + record_api_call(service, False, "decode_error") + raise SpotPriceError("{} response could not be read ({}: {})".format(label, type(e).__name__, e)) + + def supplier_status_error(self, service, label, status, body, what="request"): + """Raise SpotPriceError for a non-200 supplier reply, recording the failure category.""" + detail = "" + if isinstance(body, dict): + detail = body.get("detail") or body.get("error_description") or body.get("error") or "" + if isinstance(detail, dict): + detail = detail.get("detail") or detail.get("title") or "" + detail = ": {}".format(detail) if detail else "" + if status in (401, 403): + record_api_call(service, False, "auth_error") + raise SpotPriceError("{} rejected the {} (HTTP {}){}".format(label, what, status, detail)) + if status == 429: + record_api_call(service, False, "rate_limit") + raise SpotPriceError("{} rate limit hit (HTTP 429)".format(label)) + record_api_call(service, False, "server_error" if status >= 500 else "client_error") + raise SpotPriceError("{} returned HTTP {}{}".format(label, status, detail), status=status) + + async def ostrom_token(self): + """Return an Ostrom access token, requesting a new one (client credentials grant) when none is held or it is about to expire.""" + now = self.now() + if self.ostrom_access_token and self.ostrom_token_expiry and now < self.ostrom_token_expiry: + return self.ostrom_access_token + basic = base64.b64encode("{}:{}".format(self.ostrom_client_id, self.ostrom_client_secret).encode("utf-8")).decode("ascii") + headers = {"Authorization": "Basic {}".format(basic), "Accept": "application/json", "Content-Type": "application/x-www-form-urlencoded"} + status, body = await self.supplier_request("ostrom", "Ostrom", "POST", OSTROM_AUTH_URL, headers=headers, data={"grant_type": "client_credentials"}) + if status in (400, 401, 403): + # The token endpoint answers bad client credentials with 400 invalid_client as well as 401 + status = 401 + if status != 200: + self.supplier_status_error("ostrom", "Ostrom", status, body, what="client ID and secret") + token = body.get("access_token") if isinstance(body, dict) else None + if not token: + record_api_call("ostrom", False, "decode_error") + raise SpotPriceError("Ostrom token response has no access_token") + try: + expires_in = int(body.get("expires_in") or 3600) + except (TypeError, ValueError): + expires_in = 3600 + self.ostrom_access_token = token + self.ostrom_token_expiry = now + timedelta(seconds=max(expires_in - TOKEN_EXPIRY_MARGIN_SECONDS, TOKEN_EXPIRY_MARGIN_SECONDS)) + return token + + async def ostrom_get(self, path, params=None): + """GET an Ostrom API path with a Bearer token. A 401 drops the token and retries once with a new one.""" + for attempt in (1, 2): + token = await self.ostrom_token() + headers = {"Authorization": "Bearer {}".format(token), "Accept": "application/json"} + status, body = await self.supplier_request("ostrom", "Ostrom", "GET", OSTROM_API_URL + path, params=params, headers=headers) + if status == 401 and attempt == 1: + self.ostrom_access_token = None + self.ostrom_token_expiry = None + continue + if status != 200: + self.supplier_status_error("ostrom", "Ostrom", status, body, what="access token" if status in (401, 403) else "request") + return body + return None # pragma: no cover - the loop always returns or raises + + async def resolve_ostrom_contract(self, now): + """Look the Ostrom contract up (again) and remember when. + + A different contract or postcode than before means the prices held are for the wrong place, so + they are dropped; so are they when the account no longer has an active contract, in which case + the error is raised as usual. + """ + previous = self.ostrom_contract + contracts = await self.ostrom_get("/contracts") + try: + contract = select_ostrom_contract(contracts, self.ostrom_contract_id, warn=self.warn_once) + except SpotPriceError: + record_api_call("ostrom", False, "client_error") + if previous is not None: + self.ostrom_contract = None + self.supplier_intervals = [] + self.fetched["supplier"] = None + raise + self.ostrom_contract_checked = now + self.ostrom_contract = contract + if previous is not None and (previous.get("id"), str((previous.get("address") or {}).get("zip"))) != (contract.get("id"), str((contract.get("address") or {}).get("zip"))): + self.log("Warn: SpotPrice: Ostrom contract changed from {} to {} - discarding the prices held".format(previous.get("id"), contract.get("id"))) + self.supplier_intervals = [] + self.fetched["supplier"] = None + self.note_supplier_identity("ostrom:{}".format(contract.get("id"))) + + async def fetch_ostrom(self, start, end): + """Fetch Ostrom's end-user prices for [start, end). Returns intervals in cents per kWh, VAT included, or raises SpotPriceError.""" + if not (self.ostrom_client_id and self.ostrom_client_secret): + raise SpotPriceError("Ostrom client ID and secret are not configured (spotprice_ostrom_client_id, spotprice_ostrom_client_secret)") + now = self.now() + configured = self.ostrom_contract_id not in (None, "") + if self.ostrom_contract is None or (not configured and (self.ostrom_contract_checked is None or now - self.ostrom_contract_checked >= OSTROM_CONTRACT_RECHECK)): + await self.resolve_ostrom_contract(now) + params = { + "startDate": start.strftime("%Y-%m-%dT%H:%M:%S.000Z"), + "endDate": end.strftime("%Y-%m-%dT%H:%M:%S.000Z"), + # The API's only resolution: the documented enum for /spot-prices is ["HOUR"] + "resolution": "HOUR", + "zip": str(self.ostrom_contract["address"]["zip"]).strip(), + } + try: + body = await self.ostrom_get("/spot-prices", params) + except SpotPriceError as e: + if configured or e.status not in OSTROM_CONTRACT_CHANGED_STATUSES: + raise + # The postcode may no longer be the account's (a move): look the contract up again and retry once + previous = (self.ostrom_contract.get("id"), params["zip"]) + await self.resolve_ostrom_contract(now) + params = dict(params, zip=str(self.ostrom_contract["address"]["zip"]).strip()) + if (self.ostrom_contract.get("id"), params["zip"]) == previous: + raise + body = await self.ostrom_get("/spot-prices", params) + try: + intervals = parse_ostrom_json(body) + except SpotPriceError: + record_api_call("ostrom", False, "decode_error") + raise + intervals = [item for item in intervals if item[0] < end] + if not intervals: + record_api_call("ostrom", False, "client_error") + raise SpotPriceError("Ostrom returned no prices for this period") + record_api_call("ostrom") + return intervals + + async def octopus_de_post(self, query, variables, token=None): + """POST one GraphQL request to Octopus Energy Germany. Returns the parsed body; HTTP and rate-limit failures raise SpotPriceError.""" + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if token: + # The bare JWT, with no Bearer or "JWT " prefix, as other open-source Octopus Energy Germany + # integrations send it; the live API answers an invalid token the same way with or without one + headers["Authorization"] = token + try: + status, body = await self.http_post_json(OCTOPUS_DE_URL, {"query": query, "variables": variables}, headers) + except NETWORK_ERRORS as e: + record_api_call("octopus_de", False, "connection_error") + raise SpotPriceError("Octopus Energy Germany request failed: {}".format(e)) + except Exception as e: + record_api_call("octopus_de", False, "decode_error") + raise SpotPriceError("Octopus Energy Germany response could not be read ({}: {})".format(type(e).__name__, e)) + if KRAKEN_RATE_LIMIT_CODE in kraken_error_codes(body): + record_api_call("octopus_de", False, "rate_limit") + raise SpotPriceError("Octopus Energy Germany rate limit hit ({})".format(KRAKEN_RATE_LIMIT_CODE)) + if status != 200 and not (isinstance(body, dict) and body.get("errors")): + self.supplier_status_error("octopus_de", "Octopus Energy Germany", status, body, what="API key") + if not isinstance(body, dict): + record_api_call("octopus_de", False, "decode_error") + raise SpotPriceError("Octopus Energy Germany returned an unexpected response") + return body + + async def octopus_de_auth_token(self): + """Return a Kraken JWT for the API key, obtaining a new one when none is held or it is about to expire.""" + now = self.now() + if self.octopus_de_token and self.octopus_de_token_expiry and now < self.octopus_de_token_expiry: + return self.octopus_de_token + body = await self.octopus_de_post(OCTOPUS_DE_TOKEN_MUTATION, {"input": {"APIKey": self.octopus_de_api_key}}) + token = ((body.get("data") or {}).get("obtainKrakenToken") or {}).get("token") + if not token: + codes = kraken_error_codes(body) + if any(code in KRAKEN_CREDENTIAL_ERROR_CODES for code in codes): + record_api_call("octopus_de", False, "auth_error") + raise SpotPriceError("Octopus Energy Germany rejected the API key ({})".format(kraken_error_text(body))) + record_api_call("octopus_de", False, "client_error" if codes else "decode_error") + raise SpotPriceError("Octopus Energy Germany did not issue a token ({})".format(kraken_error_text(body))) + expiry = jwt_expiry(token) or now + timedelta(hours=1) + self.octopus_de_token = token + self.octopus_de_token_expiry = expiry - timedelta(seconds=TOKEN_EXPIRY_MARGIN_SECONDS) + return token + + async def octopus_de_query(self, query, variables, what): + """Run an authenticated query. A token Kraken no longer accepts is replaced once; any other GraphQL error raises SpotPriceError.""" + for attempt in (1, 2): + token = await self.octopus_de_auth_token() + body = await self.octopus_de_post(query, variables, token) + codes = kraken_error_codes(body) + if not codes: + return body + # A rejected key (1138/1139) will not improve with a new token: report it, do not retry + if attempt == 1 and any(code in KRAKEN_TOKEN_ERROR_CODES for code in codes) and not any(code in KRAKEN_CREDENTIAL_ERROR_CODES for code in codes): + self.octopus_de_token = None + self.octopus_de_token_expiry = None + continue + auth = any(code in KRAKEN_TOKEN_ERROR_CODES or code in KRAKEN_CREDENTIAL_ERROR_CODES for code in codes) + record_api_call("octopus_de", False, "auth_error" if auth else "client_error") + raise SpotPriceError("Octopus Energy Germany {} failed: {}".format(what, kraken_error_text(body))) + return None # pragma: no cover - the loop always returns or raises + + async def fetch_octopus_de(self, start, end): + """Fetch the unit rates of the account's active agreement for [start, end). Returns intervals in cents per kWh, VAT included.""" + if not self.octopus_de_api_key: + raise SpotPriceError("Octopus Energy Germany API key is not configured (spotprice_octopus_de_api_key)") + if not self.octopus_de_account: + body = await self.octopus_de_query(OCTOPUS_DE_ACCOUNTS_QUERY, {}, "account lookup") + accounts = [account.get("number") for account in ((body.get("data") or {}).get("viewer") or {}).get("accounts") or [] if isinstance(account, dict) and account.get("number")] + if len(accounts) != 1: + record_api_call("octopus_de", False, "client_error") + if not accounts: + raise SpotPriceError("Octopus Energy Germany API key has no accounts") + raise SpotPriceError("Octopus Energy Germany API key has several accounts ({}), set spotprice_octopus_de_account".format(", ".join(accounts))) + self.octopus_de_account = accounts[0] + body = await self.octopus_de_query(OCTOPUS_DE_AGREEMENTS_QUERY, {"accountNumber": self.octopus_de_account}, "agreements query") + try: + malo, chain = select_octopus_de_agreements(body, self.octopus_de_malo) + except SpotPriceError: + record_api_call("octopus_de", False, "client_error") + raise + self.note_supplier_identity("octopus_de:{}:{}".format(self.octopus_de_account, malo)) + intervals = [] + try: + for agreement in chain: + intervals.extend(parse_octopus_de_agreement(agreement, start, end)) + except SpotPriceError: + record_api_call("octopus_de", False, "decode_error") + raise + if octopus_de_is_dynamic(chain[0]) and not parse_octopus_de_agreement(chain[0], start, end): + # Never stand a flat rate in for a dynamic tariff: fail (keeping the last prices held) and retry + record_api_call("octopus_de", False, "client_error") + raise SpotPriceError("Octopus Energy Germany dynamic agreement {} has no forecast prices yet".format(chain[0].get("id"))) + intervals.sort(key=lambda item: item[0]) + if any(intervals[i][1] > intervals[i + 1][0] for i in range(len(intervals) - 1)): + # Two rates for one instant: publish neither rather than let one silently win + record_api_call("octopus_de", False, "decode_error") + raise SpotPriceError("Octopus Energy Germany returned overlapping rates") + if not intervals: + record_api_call("octopus_de", False, "client_error") + raise SpotPriceError("Octopus Energy Germany agreement has no rates for this period") + record_api_call("octopus_de") + return intervals + + async def fetch_ews(self, start, end): + """Fetch EWS's quarter-hourly gross prices (today and tomorrow). Returns intervals in cents per kWh, VAT included.""" + if not self.ews_api_key: + raise SpotPriceError("EWS API key is not configured (spotprice_ews_api_key)") + headers = {"X-API-Key": self.ews_api_key, "Accept": "application/json"} + status, body = await self.supplier_request("ews", "EWS", "GET", EWS_URL, headers=headers) + if status != 200: + self.supplier_status_error("ews", "EWS", status, body, what="API key") + try: + intervals = parse_ews_json(body) + except SpotPriceError: + record_api_call("ews", False, "decode_error") + raise + intervals = [item for item in intervals if item[0] < end and item[1] > start] + if not intervals: + record_api_call("ews", False, "client_error") + raise SpotPriceError("EWS returned no prices for this period") + record_api_call("ews") + return intervals + + async def fetch_supplier(self, now): + """Fetch the configured supplier's end-user prices over the fetch window.""" + start, end = self.fetch_window(now) + if self.provider == "ostrom": + return await self.fetch_ostrom(start, end) + if self.provider == "octopus_de": + return await self.fetch_octopus_de(start, end) + if self.provider == "ews": + return await self.fetch_ews(start, end) + raise SpotPriceError("provider {} has no supplier price source".format(self.provider)) + # ------------------------------------------------------------------ # Rate building # ------------------------------------------------------------------ @@ -988,10 +1850,18 @@ def build_import_rates(self): if self.provider == "tibber": # Tibber's total already includes markup, grid fees and VAT - only convert to minor units return [(start, end, round(total * 100.0, 4)) for start, end, total in self.tibber_intervals] + if self.provider in SUPPLIER_PROVIDERS: + # Already the end-user price in minor units with VAT - markup, charge zones and VAT do not apply + return list(self.supplier_intervals) + spot_intervals = self.spot_intervals + if self.provider == "awattar" and self.spot_source not in (None, "awattar"): + # The HOURLY tariff bills each hour at its hourly price, so a quarter-hourly stand-in is averaged + # for import. Export keeps the native quarter hours: a negative quarter is zeroed or priced on its own. + spot_intervals = average_to_hourly(spot_intervals) rates = [] - for start, end, spot in self.spot_intervals: + for start, end, spot in spot_intervals: zone_charge = charge_zone_rate(self.charge_zones, self.to_local(start)) if self.charge_zones else 0.0 - rates.append((start, end, spot_import_rate(spot, self.markup, zone_charge, self.vat, self.exchange_rate))) + rates.append((start, end, spot_import_rate(spot, self.markup, zone_charge, self.vat, self.exchange_rate, self.markup_percent))) return rates def build_export_rates(self): @@ -1017,20 +1887,32 @@ def build_export_rates(self): return [(start, end, round(self.export_rate, 4))] def sources_needed(self): - """The price sources this configuration fetches: "spot" and/or "tibber".""" + """The price sources this configuration fetches: "spot" and/or the import source ("tibber" or "supplier").""" needed = [] if self.config_error: # Nothing can be fetched until the configuration changes (initialize() has logged why) return needed if self.needs_spot(): needed.append("spot") - if self.provider == "tibber": - needed.append("tibber") + if self.import_source() != "spot": + needed.append(self.import_source()) return needed + def import_source(self): + """The source that carries the import price: "tibber", "supplier" or "spot".""" + if self.provider == "tibber": + return "tibber" + if self.provider in SUPPLIER_PROVIDERS: + return "supplier" + return "spot" + def intervals_of(self, source): """The held intervals of one source.""" - return self.tibber_intervals if source == "tibber" else self.spot_intervals + if source == "tibber": + return self.tibber_intervals + if source == "supplier": + return self.supplier_intervals + return self.spot_intervals def source_data_end(self, source): """End of the last held interval of one source, or None.""" @@ -1041,11 +1923,11 @@ def source_data_end(self, source): def source_intervals(self): """The intervals that define how far ahead the import price is known.""" - return self.intervals_of("tibber" if self.provider == "tibber" else "spot") + return self.intervals_of(self.import_source()) def data_end(self): """End time of the last known import interval, or None.""" - return self.source_data_end("tibber" if self.provider == "tibber" else "spot") + return self.source_data_end(self.import_source()) @property def fetched_at(self): @@ -1135,6 +2017,9 @@ async def fetch_source(self, source, now): if source == "tibber": self.tibber_intervals = await self.fetch_tibber() return + if source == "supplier": + self.supplier_intervals = await self.fetch_supplier(now) + return if not (self.zone or self.zone_eic): raise SpotPriceError("spotprice_zone is not set") start, end = self.fetch_window(now) @@ -1186,15 +2071,23 @@ async def refresh(self, now): # ------------------------------------------------------------------ def cache_filename(self): - """Storage filename for this provider/zone pair, plus a digest of the Tibber token and home for tibber. + """Storage filename for this provider/zone pair, plus a digest of the supplier's credential and selectors. - Changing the Tibber token or spotprice_tibber_home_id changes the name, so one home's cached - prices are never restored - and published as fresh - for another. Only a salted, truncated - SHA-256 of these values is used; the values themselves are never written. + Swapping the credential (Tibber token, Ostrom client ID, Octopus Energy Germany or EWS API key), + or changing spotprice_tibber_home_id, the Ostrom contract or the Octopus account or market + location, changes the name, so one account's cached prices are never restored - and published + as fresh - for another. Only a salted, truncated SHA-256 of these values is used; the values + themselves are never written. """ name = "{}_{}".format(self.provider, (self.zone or self.zone_eic or "none").replace("-", "_").lower()) - if self.provider == "tibber" and (self.tibber_token or self.tibber_home_id_configured): - material = CACHE_DIGEST_SALT + "|".join(str(item or "") for item in (self.tibber_token, self.tibber_home_id_configured)) + identity = { + "tibber": [self.tibber_token, self.tibber_home_id_configured], + "ostrom": [self.ostrom_client_id, self.ostrom_contract_id], + "octopus_de": [self.octopus_de_api_key, self.octopus_de_account_configured, self.octopus_de_malo], + "ews": [self.ews_api_key], + }.get(self.provider) + if identity and any(identity): + material = CACHE_DIGEST_SALT + "|".join(str(item or "") for item in identity) name = "{}_{}".format(name, hashlib.sha256(material.encode("utf-8")).hexdigest()[:12]) return name @@ -1209,7 +2102,9 @@ async def load_cache(self): self.spot_source = data.get("spot_source") self.tibber_intervals = deserialise_intervals(data.get("tibber_intervals")) self.tibber_currency = data.get("tibber_currency") - for source in ("spot", "tibber"): + self.supplier_intervals = deserialise_intervals(data.get("supplier_intervals")) + self.cached_supplier_identity = data.get("supplier_identity") + for source in ("spot", "tibber", "supplier"): stamp = data.get("{}_fetched_at".format(source)) or data.get("fetched_at") try: self.fetched[source] = parse_utc(stamp) if stamp else None @@ -1229,6 +2124,9 @@ async def save_cache(self): "tibber_currency": self.tibber_currency, "spot_fetched_at": self.fetched["spot"].isoformat() if self.fetched.get("spot") else None, "tibber_fetched_at": self.fetched["tibber"].isoformat() if self.fetched.get("tibber") else None, + "supplier_intervals": serialise_intervals(self.supplier_intervals), + "supplier_identity": self.supplier_identity or self.cached_supplier_identity, + "supplier_fetched_at": self.fetched["supplier"].isoformat() if self.fetched.get("supplier") else None, } await self.storage.save("spotprice", self.cache_filename(), cache, format="yaml", expiry=datetime.now(timezone.utc) + timedelta(days=3)) @@ -1287,7 +2185,7 @@ def publish(self, now): "prices_until": data_end.isoformat() if data_end else None, "last_error": self.config_error or self.last_error, "source_errors": dict(self.source_errors), - "failures": dict(self.source_failures), + "failures": {source: self.source_failures.get(source, 0) for source in self.sources_needed()}, "next_attempt": {source: when.isoformat() for source, when in self.source_next_attempt.items() if when}, "icon": "mdi:chart-bell-curve", }, @@ -1379,7 +2277,15 @@ async def test_spotprice_api(args): # pragma: no cover zone=args.zone, entsoe_token=args.entsoe_token, tibber_token=args.tibber_token, + ostrom_client_id=args.ostrom_client_id, + ostrom_client_secret=args.ostrom_client_secret, + ostrom_contract_id=args.ostrom_contract_id, + octopus_de_api_key=args.octopus_de_api_key, + octopus_de_account=args.octopus_de_account, + octopus_de_malo=args.octopus_de_malo, + ews_api_key=args.ews_api_key, markup=args.markup, + markup_percent=args.markup_percent, vat=args.vat, export_mode=args.export_mode, export_rate=args.export_rate, @@ -1408,7 +2314,15 @@ def main(): # pragma: no cover parser.add_argument("--timezone", default="Europe/Berlin") parser.add_argument("--entsoe-token", dest="entsoe_token") parser.add_argument("--tibber-token", dest="tibber_token") + parser.add_argument("--ostrom-client-id", dest="ostrom_client_id") + parser.add_argument("--ostrom-client-secret", dest="ostrom_client_secret") + parser.add_argument("--ostrom-contract-id", dest="ostrom_contract_id") + parser.add_argument("--octopus-de-api-key", dest="octopus_de_api_key") + parser.add_argument("--octopus-de-account", dest="octopus_de_account") + parser.add_argument("--octopus-de-malo", dest="octopus_de_malo") + parser.add_argument("--ews-api-key", dest="ews_api_key") parser.add_argument("--markup", type=float, default=0.0) + parser.add_argument("--markup-percent", dest="markup_percent", type=float, default=None, help="Percentage of the absolute spot price added (default 3 for awattar, else 0)") parser.add_argument("--vat", type=float, default=0.0, help="VAT as a fraction, e.g. 0.19") parser.add_argument("--export-mode", dest="export_mode", default="none", choices=SPOTPRICE_EXPORT_MODES) parser.add_argument("--export-rate", dest="export_rate", type=float, default=0.0) diff --git a/apps/predbat/tests/test_spotprice.py b/apps/predbat/tests/test_spotprice.py index 31157036c..1fa603359 100644 --- a/apps/predbat/tests/test_spotprice.py +++ b/apps/predbat/tests/test_spotprice.py @@ -727,7 +727,7 @@ def test_spotprice_zones_and_registry(my_predbat=None): assert entry["args"]["entsoe_token"]["secret"] and entry["args"]["tibber_token"]["secret"] # provider defaults to energycharts, so the component must be gated on a zone or a Tibber token assert "default" not in entry["args"]["provider"] and not entry["args"]["provider"]["required"] - assert entry["required_or"] == ["zone", "tibber_token"] + assert entry["required_or"] == ["zone", "tibber_token", "ostrom_client_id", "ostrom_client_secret", "octopus_de_api_key", "ews_api_key"] assert SpotPriceAPI(FakeBase(), zone="NL").provider == "energycharts" @@ -1351,6 +1351,1366 @@ def tibber(**kwargs): assert make_api(provider="energycharts", entsoe_token=None, tibber_token="t").cache_filename() == "energycharts_de_lu" +# --------------------------------------------------------------------------- +# Ostrom +# --------------------------------------------------------------------------- + +OSTROM_CONTRACTS = { + "data": [ + {"id": 100523456, "type": "ELECTRICITY", "productCode": "SIMPLY_DYNAMIC", "status": "ACTIVE", "address": {"zip": "10997", "city": "Berlin"}}, + {"id": 100523999, "type": "ELECTRICITY", "productCode": "SIMPLY_FAIR", "status": "TERMINATED", "address": {"zip": "10997"}}, + ] +} + + +def ostrom_prices(start="2025-05-01T22:00:00.000Z", hours=48, spot=10.0, levies=19.28): + """An Ostrom /spot-prices reply: hourly entries of grossKwhPrice + grossKwhTaxAndLevies (cents incl. VAT).""" + first = dt(start) + return { + "data": [ + {"date": (first + timedelta(hours=h)).strftime("%Y-%m-%dT%H:%M:%S.000Z"), "netMwhPrice": spot * 10 / 1.19, "netKwhPrice": spot / 1.19, "grossKwhPrice": spot + h, "netKwhTaxAndLevies": levies / 1.19, "grossKwhTaxAndLevies": levies} + for h in range(hours) + ] + } + + +class FakeOstrom: + """Answers http_request for the Ostrom token, contracts and spot-prices endpoints, recording every call.""" + + def __init__(self, contracts=None, prices=None): + """Set the replies; each may be replaced or queued (a list of (status, body)) by a test.""" + self.calls = [] + self.token_replies = [] + self.api_replies = {} + self.contracts = contracts if contracts is not None else OSTROM_CONTRACTS + self.prices = prices if prices is not None else ostrom_prices() + self.issued = 0 + + async def __call__(self, method, url, params=None, headers=None, data=None): + """Route a request to the matching fake endpoint.""" + self.calls.append((method, url, params, headers, data)) + if url.endswith("/oauth2/token"): + if self.token_replies: + return self.token_replies.pop(0) + self.issued += 1 + return 200, {"access_token": "tok-{}".format(self.issued), "token_type": "Bearer", "expires_in": 3600} + path = url.split("ostrom-api.io", 1)[1] + queued = self.api_replies.get(path) + if queued: + return queued.pop(0) + return 200, self.contracts if path == "/contracts" else self.prices + + def count(self, suffix): + """How many requests went to a URL ending in suffix.""" + return sum(1 for call in self.calls if call[1].endswith(suffix)) + + +def make_ostrom(**kwargs): + """An Ostrom-provider component with a FakeOstrom behind http_request.""" + config = {"provider": "ostrom", "entsoe_token": None, "zone": None, "ostrom_client_id": "cid", "ostrom_client_secret": "the-secret"} + config.update(kwargs) + api = make_api(**config) + fake = FakeOstrom() + api.http_request = fake + return api, fake + + +def test_spotprice_ostrom_parse(my_predbat=None): + """Ostrom's price is grossKwhPrice + grossKwhTaxAndLevies per hour; a missing field is a gap; a reply with no levies at all (no postcode) is rejected.""" + from spotprice import parse_ostrom_json + + intervals = parse_ostrom_json(ostrom_prices(hours=3, spot=10.0, levies=19.28)) + assert [(start, end) for start, end, _v in intervals][0] == (dt("2025-05-01T22:00Z"), dt("2025-05-01T23:00Z")) + assert [value for _s, _e, value in intervals] == [29.28, 30.28, 31.28], intervals + gap = ostrom_prices(hours=3) + gap["data"][1]["grossKwhTaxAndLevies"] = None + gapped = parse_ostrom_json(gap) + assert [start for start, _e, _v in gapped] == [dt("2025-05-01T22:00Z"), dt("2025-05-02T00:00Z")], gapped + assert gapped[0][1] == dt("2025-05-01T23:00Z") + # Negative spot still adds the levies + negative = ostrom_prices(hours=1, spot=-5.0, levies=19.0) + assert parse_ostrom_json(negative)[0][2] == 14.0 + for bad in (ostrom_prices(hours=2, levies=0.0), {"data": [{"date": 12345, "grossKwhPrice": 1, "grossKwhTaxAndLevies": 1}]}, {"data": [{"grossKwhPrice": 1}]}, {"data": "x"}, None): + try: + parse_ostrom_json(bad) + except SpotPriceError: + continue + raise AssertionError("expected SpotPriceError for {}".format(bad)) + assert parse_ostrom_json({"data": []}) == [] + + +def test_spotprice_ostrom_contract_selection(my_predbat=None): + """The single active dynamic contract is used; an explicit id picks one; fixed products, unknown ids, several candidates and a missing postcode are errors.""" + from spotprice import select_ostrom_contract + + assert select_ostrom_contract(OSTROM_CONTRACTS)["id"] == 100523456 + assert select_ostrom_contract(OSTROM_CONTRACTS, 100523456)["id"] == 100523456 + assert select_ostrom_contract(OSTROM_CONTRACTS, "100523456")["id"] == 100523456 + v2 = {"data": [{"id": 1, "productCode": "SimplyDynamic_V2", "status": "ACTIVE", "address": {"zip": "80331"}}]} + assert select_ostrom_contract(v2)["id"] == 1 + two_active = {"data": [dict(OSTROM_CONTRACTS["data"][0]), dict(OSTROM_CONTRACTS["data"][0], id=7)]} + fixed_active = {"data": [dict(OSTROM_CONTRACTS["data"][1], status="ACTIVE")]} + # Only active contracts are priced: a terminated or pending one is never a stand-in, even when it is the only one + only_inactive = {"data": [dict(OSTROM_CONTRACTS["data"][0], status="TERMINATED"), dict(OSTROM_CONTRACTS["data"][0], id=8, status="PENDING")]} + cases = [ + ((fixed_active, None), "not a dynamic tariff"), + ((OSTROM_CONTRACTS, 100523999), "100523999 is TERMINATED, not active"), + ((only_inactive, None), "no active Ostrom contract"), + (({"data": [dict(OSTROM_CONTRACTS["data"][0], status=None)]}, None), "no active Ostrom contract"), + ((OSTROM_CONTRACTS, 42), "not found"), + ((two_active, None), "spotprice_ostrom_contract_id"), + (({"data": [dict(OSTROM_CONTRACTS["data"][0], address={})]}, None), "no postcode"), + (({"data": []}, None), "no contracts"), + (({"nope": 1}, None), "unexpected"), + ] + for args, phrase in cases: + try: + select_ostrom_contract(*args) + except SpotPriceError as e: + assert phrase in str(e), (phrase, str(e)) + continue + raise AssertionError("expected an error containing '{}'".format(phrase)) + + +def test_spotprice_ostrom_fetch_and_token_cache(my_predbat=None): + """Client credentials are sent as Basic auth; the token is reused until shortly before expiry; contracts are read once; prices are asked for by postcode, hourly, in UTC.""" + import base64 + + api, fake = make_ostrom() + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + intervals = run(api.fetch_supplier(now)) + assert len(intervals) == 48 and intervals[0][2] == 29.28 + method, url, _params, headers, data = fake.calls[0] + assert method == "POST" and url == "https://auth.production.ostrom-api.io/oauth2/token" + assert headers["Authorization"] == "Basic " + base64.b64encode(b"cid:the-secret").decode("ascii") + assert data == {"grant_type": "client_credentials"} + assert fake.calls[1][1] == "https://production.ostrom-api.io/contracts" and fake.calls[1][3]["Authorization"] == "Bearer tok-1" + _m, url, params, headers, _d = fake.calls[2] + assert url == "https://production.ostrom-api.io/spot-prices" and headers["Authorization"] == "Bearer tok-1" + assert params == {"startDate": "2025-04-30T22:00:00.000Z", "endDate": "2025-05-03T22:00:00.000Z", "resolution": "HOUR", "zip": "10997"}, params + + # Within the token's life: no new token and no second contracts lookup + later = now + timedelta(minutes=50) + pin_now(api, later) + run(api.fetch_supplier(later)) + assert fake.count("/oauth2/token") == 1 and fake.count("/contracts") == 1 and fake.count("/spot-prices") == 2 + # Inside the renewal margin (expires_in 3600 less 60 seconds) a new token is requested + renew = now + timedelta(seconds=3540) + pin_now(api, renew) + run(api.fetch_supplier(renew)) + assert fake.count("/oauth2/token") == 2 and fake.calls[-1][3]["Authorization"] == "Bearer tok-2" + + +def test_spotprice_ostrom_auth_errors(my_predbat=None): + """A 401 from the API renews the token and retries once; a second 401, or rejected client credentials, is an auth_error SpotPriceError.""" + import spotprice as spotprice_module + + recorded = [] + original = spotprice_module.record_api_call + spotprice_module.record_api_call = lambda service, success=True, reason=None: recorded.append((service, success, reason)) + try: + now = dt("2025-05-02T08:00Z") + api, fake = make_ostrom() + pin_now(api, now) + fake.api_replies["/contracts"] = [(401, {"type": "unauthorized", "detail": "Unauthorized API access."})] + assert len(run(api.fetch_supplier(now))) == 48 + assert fake.count("/oauth2/token") == 2 and fake.count("/contracts") == 2 + + api, fake = make_ostrom() + pin_now(api, now) + fake.api_replies["/contracts"] = [(401, None), (401, {"detail": "Unauthorized API access."})] + recorded.clear() + try: + run(api.fetch_supplier(now)) + raise AssertionError("second 401 should raise") + except SpotPriceError as e: + assert "Ostrom rejected the access token (HTTP 401)" in str(e), str(e) + assert ("ostrom", False, "auth_error") in recorded + assert fake.count("/contracts") == 2, fake.calls + + for status in (400, 401): + api, fake = make_ostrom(ostrom_client_secret="wrong-secret") + pin_now(api, now) + fake.token_replies = [(status, {"error": "invalid_client"})] + recorded.clear() + try: + run(api.fetch_supplier(now)) + raise AssertionError("bad credentials should raise") + except SpotPriceError as e: + assert "client ID and secret" in str(e) and "wrong-secret" not in str(e), str(e) + assert recorded == [("ostrom", False, "auth_error")], recorded + + for status, reason in ((429, "rate_limit"), (503, "server_error"), (400, "client_error")): + api, fake = make_ostrom() + pin_now(api, now) + fake.api_replies["/spot-prices"] = [(status, {"detail": "nope"})] + recorded.clear() + try: + run(api.fetch_supplier(now)) + raise AssertionError("HTTP {} should raise".format(status)) + except SpotPriceError: + pass + assert recorded[-1] == ("ostrom", False, reason), recorded + + api, fake = make_ostrom() + pin_now(api, now) + fake.token_replies = [(200, {"token_type": "Bearer"})] + try: + run(api.fetch_supplier(now)) + raise AssertionError("a token reply without access_token should raise") + except SpotPriceError as e: + assert "access_token" in str(e) + finally: + spotprice_module.record_api_call = original + + +def test_spotprice_ostrom_all_in_and_refresh(my_predbat=None): + """Ostrom's price is used as-is (no markup, charge zones or VAT); failures back off on the supplier source only and the prices survive a restart.""" + storage = FakeStorage() + api, fake = make_ostrom(markup=15, vat=0.19, charge_zones=[{"from": "00:00", "to": "00:00", "charge": 9}], storage=storage) + assert api.sources_needed() == ["supplier"] and not api.needs_spot() + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert run(api.refresh(now)) is True + assert api.fetched["supplier"] == now and api.data_end() == dt("2025-05-03T22:00Z") + rates = api.build_import_rates() + assert rates[0] == (dt("2025-05-01T22:00Z"), dt("2025-05-01T23:00Z"), 29.28), rates[0] + api.publish(now) + assert api.base.entities["sensor.predbat_spotprice_import_rates"]["state"] == 39.28 + assert api.base.entities["sensor.predbat_spotprice_status"]["state"] == "ok" + assert ("metric_octopus_import", "sensor.predbat_spotprice_import_rates") in api.base.set_args + + restored, _fake = make_ostrom(storage=storage) + run(restored.load_cache()) + assert restored.supplier_intervals == api.supplier_intervals and restored.fetched_at == now + + fake.api_replies["/spot-prices"] = [(503, None)] + later = now + timedelta(hours=7) + pin_now(api, later) + assert run(api.refresh(later)) is False + assert api.source_failures == {"spot": 0, "tibber": 0, "supplier": 1} and api.source_next_attempt["supplier"] == later + timedelta(minutes=5) + assert "Ostrom returned HTTP 503" in api.last_error + + # With spot-linked export the spot source is fetched alongside, and its failure does not block Ostrom + both, both_fake = make_ostrom(zone="DE-LU", export_mode="spot") + assert both.sources_needed() == ["spot", "supplier"] + + async def spot_down(start, end): + """Spot source down.""" + raise SpotPriceError("Energy-Charts returned HTTP 500") + + both.fetch_energycharts = spot_down + pin_now(both, now) + assert run(both.refresh(now)) is False + assert both.fetched["supplier"] == now and both.source_failures["spot"] == 1 and both.source_failures["supplier"] == 0 + + +def test_spotprice_ostrom_provider_inference(my_predbat=None): + """With the provider unset the one supplier with credentials is used; credentials for two suppliers, or half an Ostrom pair, are one clear config error.""" + assert make_api(provider=None, entsoe_token=None, zone=None, ostrom_client_id="i", ostrom_client_secret="s").provider == "ostrom" + assert make_api(provider=None, entsoe_token=None, zone="DE-LU", ostrom_client_id="i", ostrom_client_secret="s").provider == "ostrom" + # An explicit provider stays the authority + explicit = make_api(provider="energycharts", entsoe_token=None, ostrom_client_id="i", ostrom_client_secret="s", tibber_token="t") + assert explicit.provider == "energycharts" and explicit.config_error is None + both = make_api(provider=None, entsoe_token=None, zone="DE-LU", tibber_token="t", ostrom_client_id="i", ostrom_client_secret="s") + assert both.config_error == "credentials for several suppliers (tibber, ostrom) are set, set spotprice_provider to choose one", both.config_error + assert both.sources_needed() == [] and sum(line.startswith("Error:") for line in both.base.logs) == 1 + half = make_api(provider=None, entsoe_token=None, zone=None, ostrom_client_id="i") + assert half.provider == "ostrom" and "spotprice_ostrom_client_secret" in half.config_error and half.sources_needed() == [] + + +# --------------------------------------------------------------------------- +# Octopus Energy Germany +# --------------------------------------------------------------------------- + + +def make_jwt(exp): + """An unsigned JWT whose payload carries the given exp (an aware datetime).""" + import base64 + import json + + def encode(obj): + """Base64url without padding.""" + return base64.urlsafe_b64encode(json.dumps(obj).encode("utf-8")).decode("ascii").rstrip("=") + + return "{}.{}.sig".format(encode({"alg": "HS256", "typ": "JWT"}), encode({"exp": int(exp.timestamp())})) + + +def octopus_de_agreement(forecast=None, info=None, active=True, valid_from="2025-01-01T00:00:00+01:00", valid_to=None, product=None): + """An agreements response holding one market location with one agreement (dynamic product when a forecast is given).""" + agreement = { + "id": "4711", + "isActive": active, + "isRevoked": False, + "validFrom": valid_from, + "validTo": valid_to, + "product": {"code": product or ("DYNAMIC-OCTOPUS" if forecast else "OCTOPUS-FIX-12M")}, + "unitRateInformation": info or {"__typename": "SimpleProductUnitRateInformation", "latestGrossUnitRateCentsPerKwh": "31.5"}, + "unitRateForecast": forecast or [], + } + return {"data": {"account": {"properties": [{"electricityMalos": [{"maloNumber": "50000000001", "agreements": [agreement]}]}]}}} + + +def octopus_de_forecast(start="2025-05-01T22:00:00+00:00", count=8, minutes=15, base=20.0): + """unitRateForecast entries, one per slot, with gross rates as strings (as Kraken sends them).""" + first = dt(start) + return [ + { + "validFrom": (first + timedelta(minutes=minutes * i)).isoformat(), + "validTo": (first + timedelta(minutes=minutes * (i + 1))).isoformat(), + "unitRateInformation": {"__typename": "TimeOfUseProductUnitRateInformation", "rates": [{"latestGrossUnitRateCentsPerKwh": "{:.2f}".format(base + i)}]}, + } + for i in range(count) + ] + + +class FakeKraken: + """Answers http_post_json for the token mutation, the accounts lookup and the agreements query.""" + + def __init__(self, agreements=None, accounts=("A-1234ABCD",)): + """Set the replies; queued (status, body) tuples per operation override them.""" + self.calls = [] + self.queued = {"token": [], "accounts": [], "agreements": []} + self.agreements = agreements or octopus_de_agreement(forecast=octopus_de_forecast()) + self.accounts = list(accounts) + self.issued = 0 + self.token_exp = dt("2025-05-02T09:00Z") + + async def __call__(self, url, payload, headers): + """Route by the GraphQL operation.""" + query = payload["query"] + operation = "token" if "obtainKrakenToken" in query else ("accounts" if "viewer" in query else "agreements") + self.calls.append((operation, url, payload, headers)) + if self.queued[operation]: + return self.queued[operation].pop(0) + if operation == "token": + self.issued += 1 + return 200, {"data": {"obtainKrakenToken": {"token": make_jwt(self.token_exp)}}} + if operation == "accounts": + return 200, {"data": {"viewer": {"accounts": [{"number": number} for number in self.accounts]}}} + return 200, self.agreements + + def count(self, operation): + """How many requests of one operation were made.""" + return sum(1 for call in self.calls if call[0] == operation) + + +def make_octopus_de(**kwargs): + """An octopus_de-provider component with a FakeKraken behind http_post_json.""" + config = {"provider": "octopus_de", "entsoe_token": None, "zone": None, "octopus_de_api_key": "sk_live_test"} + config.update(kwargs) + api = make_api(**config) + fake = FakeKraken() + api.http_post_json = fake + return api, fake + + +def kraken_error(code, description="error"): + """A Kraken GraphQL error reply (HTTP 200).""" + return 200, {"errors": [{"message": description, "extensions": {"errorCode": code, "errorDescription": description}}], "data": None} + + +def test_spotprice_octopus_de_dynamic(my_predbat=None): + """The API key is exchanged for a JWT sent without Bearer; the account is looked up once; dynamic forecast slots are used as-is with no markup or VAT.""" + from spotprice import OCTOPUS_DE_URL + + api, fake = make_octopus_de(markup=15, vat=0.19, charge_zones=[{"from": "00:00", "to": "00:00", "charge": 9}]) + assert api.sources_needed() == ["supplier"] + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert run(api.refresh(now)) is True + operation, url, payload, headers = fake.calls[0] + assert operation == "token" and url == OCTOPUS_DE_URL and payload["variables"] == {"input": {"APIKey": "sk_live_test"}} and "Authorization" not in headers + assert [call[0] for call in fake.calls] == ["token", "accounts", "agreements"] + token = fake.calls[1][3]["Authorization"] + assert token == make_jwt(fake.token_exp) and not token.startswith("Bearer") + assert fake.calls[2][2]["variables"] == {"accountNumber": "A-1234ABCD"} + rates = api.build_import_rates() + assert [rate for _s, _e, rate in rates] == [20.0 + i for i in range(8)], rates + assert rates[0][:2] == (dt("2025-05-01T22:00Z"), dt("2025-05-01T22:15Z")) + # Second refresh: the token is still valid and the account is known + pin_now(api, now + timedelta(minutes=30)) + run(api.refresh(now + timedelta(minutes=30))) + assert fake.count("token") == 1 and fake.count("accounts") == 1 and fake.count("agreements") == 2 + # An explicit account skips the lookup + explicit, explicit_fake = make_octopus_de(octopus_de_account=" A-9999 ") + pin_now(explicit, now) + run(explicit.fetch_supplier(now)) + assert explicit_fake.count("accounts") == 0 and explicit_fake.calls[-1][2]["variables"] == {"accountNumber": "A-9999"} + + +def test_spotprice_octopus_de_fixed_and_time_of_use(my_predbat=None): + """Without a forecast, a fixed rate spans the window (clipped to the agreement) and a time-of-use table repeats daily in local time, wrapping past midnight and across DST.""" + from spotprice import parse_octopus_de_agreement + + start, end = dt("2025-05-01T22:00Z"), dt("2025-05-03T22:00Z") + fixed = octopus_de_agreement(valid_to="2025-05-03T00:00:00+02:00")["data"]["account"]["properties"][0]["electricityMalos"][0]["agreements"][0] + assert parse_octopus_de_agreement(fixed, start, end, BERLIN) == [(start, dt("2025-05-02T22:00Z"), 31.5)] + + tou_info = { + "__typename": "TimeOfUseProductUnitRateInformation", + "rates": [ + {"latestGrossUnitRateCentsPerKwh": "22.0", "timeslotActivationRules": [{"activeFromTime": "22:00:00", "activeToTime": "06:00:00"}]}, + {"latestGrossUnitRateCentsPerKwh": "35.0", "timeslotActivationRules": [{"activeFromTime": "06:00:00", "activeToTime": "22:00:00"}]}, + ], + } + tou = octopus_de_agreement(info=tou_info)["data"]["account"]["properties"][0]["electricityMalos"][0]["agreements"][0] + intervals = parse_octopus_de_agreement(tou, start, end, BERLIN) + # 00:00-06:00 local (the tail of the previous evening's slot), then 06-22 and 22-06 alternating + assert intervals[0] == (dt("2025-05-01T22:00Z"), dt("2025-05-02T04:00Z"), 22.0), intervals[0] + assert intervals[1] == (dt("2025-05-02T04:00Z"), dt("2025-05-02T20:00Z"), 35.0) + assert intervals[2] == (dt("2025-05-02T20:00Z"), dt("2025-05-03T04:00Z"), 22.0) + assert intervals[-1][1] == end and len(intervals) == 5, intervals + # Every minute of the window is covered exactly once + assert all(intervals[i][1] == intervals[i + 1][0] for i in range(len(intervals) - 1)) + # Autumn DST: the night slot 22:00-06:00 local is 9 hours long that night + dst = parse_octopus_de_agreement(tou, dt("2025-10-25T22:00Z"), dt("2025-10-26T23:00Z"), BERLIN) + assert (dt("2025-10-25T20:00Z"), dt("2025-10-26T05:00Z")) not in [(s_, e) for s_, e, _v in dst] + assert dst[0] == (dt("2025-10-25T22:00Z"), dt("2025-10-26T05:00Z"), 22.0), dst[0] + assert dst[1] == (dt("2025-10-26T05:00Z"), dt("2025-10-26T21:00Z"), 35.0), dst[1] + + # An agreement that has ended before the window, or one with no rate at all + ended = dict(fixed, validTo="2025-04-01T00:00:00+02:00") + assert parse_octopus_de_agreement(ended, start, end, BERLIN) == [] + for bad in ( + dict(fixed, unitRateInformation={"__typename": "SimpleProductUnitRateInformation", "latestGrossUnitRateCentsPerKwh": None}), + dict(tou, unitRateInformation={"__typename": "TimeOfUseProductUnitRateInformation", "rates": []}), + dict(fixed, validFrom=12345), + ): + try: + parse_octopus_de_agreement(bad, start, end, BERLIN) + except SpotPriceError: + continue + raise AssertionError("expected SpotPriceError for {}".format(bad)) + + +def test_spotprice_octopus_de_auth_and_errors(my_predbat=None): + """The JWT is reused until shortly before its exp; a rejected token is replaced once; a bad key, rate limit, HTTP failure, several accounts or no active agreement are SpotPriceErrors with their category.""" + import spotprice as spotprice_module + + recorded = [] + original = spotprice_module.record_api_call + spotprice_module.record_api_call = lambda service, success=True, reason=None: recorded.append((service, success, reason)) + try: + now = dt("2025-05-02T08:00Z") + api, fake = make_octopus_de() + pin_now(api, now) + run(api.fetch_supplier(now)) + pin_now(api, dt("2025-05-02T08:58Z")) + run(api.fetch_supplier(dt("2025-05-02T08:58Z"))) + assert fake.count("token") == 1 + pin_now(api, dt("2025-05-02T08:59:30Z")) # inside the 60 second margin before exp + run(api.fetch_supplier(dt("2025-05-02T08:59:30Z"))) + assert fake.count("token") == 2 + + # A token the API no longer accepts is replaced and the query retried once + pin_now(api, now) + fake.queued["agreements"] = [kraken_error("KT-CT-1124", "Token has expired.")] + assert len(run(api.fetch_supplier(now))) == 8 and fake.count("token") == 3 + fake.queued["agreements"] = [kraken_error("KT-CT-1124"), kraken_error("KT-CT-1124", "Token has expired.")] + recorded.clear() + try: + run(api.fetch_supplier(now)) + raise AssertionError("repeated token error should raise") + except SpotPriceError as e: + assert "Token has expired" in str(e) + assert recorded[-1] == ("octopus_de", False, "auth_error") + + cases = [ + ("token", kraken_error("KT-CT-1139", "Authentication failed."), "rejected the API key", "auth_error"), + ("token", kraken_error("KT-CT-1199", "Too many requests."), "rate limit", "rate_limit"), + ("token", (502, None), "HTTP 502", "server_error"), + ("token", (200, {"data": {"obtainKrakenToken": None}}), "did not issue a token", "decode_error"), + ("agreements", kraken_error("KT-CT-4123", "Unauthorized."), "agreements query failed", "client_error"), + ("agreements", (200, octopus_de_agreement(active=False)), "no active electricity agreement", "client_error"), + ("agreements", (200, {"data": {"account": None}}), "account not found", "client_error"), + ] + for operation, reply, phrase, reason in cases: + api, fake = make_octopus_de(octopus_de_api_key="sk_live_secret") + pin_now(api, now) + fake.queued[operation] = [reply] + recorded.clear() + try: + run(api.fetch_supplier(now)) + raise AssertionError("expected an error for {}".format(phrase)) + except SpotPriceError as e: + assert phrase in str(e) and "sk_live_secret" not in str(e), (phrase, str(e)) + assert recorded[-1] == ("octopus_de", False, reason), (phrase, recorded) + + several, several_fake = make_octopus_de() + several_fake.accounts = ["A-1", "A-2"] + pin_now(several, now) + try: + run(several.fetch_supplier(now)) + raise AssertionError("several accounts should raise") + except SpotPriceError as e: + assert "A-1, A-2" in str(e) and "spotprice_octopus_de_account" in str(e) + + # A refresh failure backs off the supplier source like any other + failing, failing_fake = make_octopus_de() + failing_fake.queued["token"] = [kraken_error("KT-CT-1139", "Authentication failed.")] + assert run(failing.refresh(now)) is False and failing.source_failures["supplier"] == 1 and "rejected the API key" in failing.last_error + finally: + spotprice_module.record_api_call = original + + +def test_spotprice_octopus_de_inference_and_config(my_predbat=None): + """An Octopus Energy Germany API key alone selects octopus_de; with another supplier's credentials the provider must be set; the provider without a key is a config error.""" + from spotprice import jwt_expiry + + assert make_api(provider=None, entsoe_token=None, zone=None, octopus_de_api_key="k").provider == "octopus_de" + both = make_api(provider=None, entsoe_token=None, zone=None, octopus_de_api_key="k", tibber_token="t") + assert "tibber, octopus_de" in both.config_error + assert make_api(provider="octopus_de", entsoe_token=None, zone=None, octopus_de_api_key="k", tibber_token="t").config_error is None + assert make_api(provider="octopus_de", entsoe_token=None, zone=None).config_error == "spotprice_octopus_de_api_key is required for provider octopus_de" + assert jwt_expiry(make_jwt(dt("2025-05-02T09:00Z"))) == dt("2025-05-02T09:00Z") + assert jwt_expiry("not-a-jwt") is None + + +# --------------------------------------------------------------------------- +# aWATTar / tado Energy +# --------------------------------------------------------------------------- + + +def awattar_reply(start="2025-05-01T22:00Z", hours=48, price=100.0, unit="Eur/MWh"): + """An aWATTar /v1/marketdata reply: hourly entries with millisecond start/end timestamps.""" + first = int(dt(start).timestamp() * 1000) + return {"object": "list", "data": [{"start_timestamp": first + 3600000 * h, "end_timestamp": first + 3600000 * (h + 1), "marketprice": price + h, "unit": unit} for h in range(hours)], "url": "/de/v1/marketdata"} + + +def test_spotprice_awattar_parse_and_fetch(my_predbat=None): + """aWATTar's hourly EUR/MWh prices parse with their own start/end, are asked for by zone host and millisecond window, and failures map to SpotPriceError categories.""" + import spotprice as spotprice_module + from spotprice import parse_awattar_json + + intervals = parse_awattar_json(awattar_reply(hours=3)) + assert intervals == [(dt("2025-05-01T22:00Z"), dt("2025-05-01T23:00Z"), 100.0), (dt("2025-05-01T23:00Z"), dt("2025-05-02T00:00Z"), 101.0), (dt("2025-05-02T00:00Z"), dt("2025-05-02T01:00Z"), 102.0)] + with_null = awattar_reply(hours=3) + with_null["data"][1]["marketprice"] = None + assert [start for start, _e, _v in parse_awattar_json(with_null)] == [dt("2025-05-01T22:00Z"), dt("2025-05-02T00:00Z")] + for bad in (awattar_reply(hours=1, unit="Eur/kWh"), {"data": [{"start_timestamp": "x", "end_timestamp": 1, "marketprice": 1}]}, {"data": [{"start_timestamp": 2000, "end_timestamp": 1000, "marketprice": 1}]}, {"object": "list"}, None): + try: + parse_awattar_json(bad) + except SpotPriceError: + continue + raise AssertionError("expected SpotPriceError for {}".format(bad)) + + seen = [] + for zone, host in (("DE-LU", "https://api.awattar.de/v1/marketdata"), ("AT", "https://api.awattar.at/v1/marketdata")): + api = make_api(provider="awattar", entsoe_token=None, zone=zone) + + async def get(url, params, expect_json): + """Return the fixture.""" + seen.append((url, params)) + return 200, awattar_reply() + + api.http_get = get + fetched = run(api.fetch_awattar(dt("2025-05-01T22:00Z"), dt("2025-05-02T22:00Z"))) + assert len(fetched) == 24, len(fetched) # trimmed to the window + assert seen[-1] == (host, {"start": int(dt("2025-05-01T22:00Z").timestamp() * 1000), "end": int(dt("2025-05-02T22:00Z").timestamp() * 1000)}), seen[-1] + + recorded = [] + original = spotprice_module.record_api_call + spotprice_module.record_api_call = lambda service, success=True, reason=None: recorded.append((service, success, reason)) + try: + for reply, reason in (((429, None), "rate_limit"), ((503, None), "server_error"), ((404, None), "client_error"), ((200, {"data": []}), "client_error"), ((200, {"nope": 1}), "decode_error")): + api = make_api(provider="awattar", entsoe_token=None, zone="DE-LU") + + async def bad_get(url, params, expect_json, reply=reply): + """Return an error reply.""" + return reply + + api.http_get = bad_get + recorded.clear() + try: + run(api.fetch_awattar(dt("2025-05-01T22:00Z"), dt("2025-05-02T22:00Z"))) + raise AssertionError("expected an error for {}".format(reply)) + except SpotPriceError: + pass + assert recorded == [("awattar", False, reason)], (reply, recorded) + finally: + spotprice_module.record_api_call = original + + +def test_spotprice_awattar_markup_percent(my_predbat=None): + """The percentage markup is charged on the absolute spot price before VAT; awattar defaults it to 3, an explicit value (even 0) wins, other providers default to 0.""" + assert spot_import_rate(100.0, 1.5, 0.0, 0.19, markup_percent=3.0) == 14.042 + # Negative spot: 3% of |spot| is still a cost + assert spot_import_rate(-50.0, 1.5, 0.0, 0.19, markup_percent=3.0) == -3.9865 + assert spot_import_rate(100.0, 1.5, 0.0, 0.19) == 13.685 + + api = make_api(provider="awattar", entsoe_token=None, markup=1.5, vat=0.19, charge_zones=[{"from": "00:00", "to": "00:00", "charge": 10.0}]) + assert api.markup_percent == 3.0 and api.sources_needed() == ["spot"] + api.spot_intervals = [(dt("2025-05-02T10:00Z"), dt("2025-05-02T11:00Z"), 100.0), (dt("2025-05-02T11:00Z"), dt("2025-05-02T12:00Z"), -50.0)] + assert [rate for _s, _e, rate in api.build_import_rates()] == [25.942, 7.9135], api.build_import_rates() + assert make_api(provider="awattar", entsoe_token=None, markup_percent=0).markup_percent == 0.0 + assert make_api(provider="energycharts", entsoe_token=None).markup_percent == 0.0 + linked = make_api(provider="entsoe", markup=0, vat=0, markup_percent=10) + linked.spot_intervals = [(dt("2025-05-02T10:00Z"), dt("2025-05-02T11:00Z"), 100.0)] + assert linked.build_import_rates()[0][2] == 11.0 + # Export is not touched by the import markup percentage + linked.export_mode = "spot" + assert linked.build_export_rates()[0][2] == 10.0 + + +def test_spotprice_awattar_fallback(my_predbat=None): + """aWATTar falls back to ENTSO-E (with a token) and then Energy-Charts, logging the outage and the recovery once; zones aWATTar does not publish go straight to the fallbacks; awattar is never inferred.""" + calls = [] + + def source(name, fail): + """A fake fetch that records its name and fails or returns spot prices.""" + + async def fetch(start, end): + """Fake fetch.""" + calls.append(name) + if fail["on"]: + raise SpotPriceError("{} down".format(name)) + return spot_fixture() + + return fetch + + awattar_state = {"on": True} + api = make_api(provider="awattar", entsoe_token=None, zone="AT") + api.fetch_awattar = source("awattar", awattar_state) + api.fetch_energycharts = source("energycharts", {"on": False}) + assert run(api.refresh(dt("2025-05-02T08:00Z"))) is True + assert calls == ["awattar", "energycharts"] and api.spot_source == "energycharts" + assert run(api.refresh(dt("2025-05-02T14:00Z"))) is True + assert sum("until aWATTar recovers" in line for line in api.base.logs) == 1, api.base.logs + awattar_state["on"] = False + assert run(api.refresh(dt("2025-05-02T20:00Z"))) is True and api.spot_source == "awattar" + assert sum("aWATTar is working again" in line for line in api.base.logs) == 1 + + calls.clear() + with_token = make_api(provider="awattar", entsoe_token="t", zone="DE-LU") + with_token.fetch_awattar = source("awattar", {"on": True}) + with_token.fetch_entsoe = source("entsoe", {"on": True}) + with_token.fetch_energycharts = source("energycharts", {"on": False}) + assert run(with_token.refresh(dt("2025-05-02T08:00Z"))) is True + assert calls == ["awattar", "entsoe", "energycharts"], calls + + calls.clear() + elsewhere = make_api(provider="awattar", entsoe_token=None, zone="NL") + assert any("aWATTar publishes prices for DE-LU and AT only" in line for line in elsewhere.base.logs) + elsewhere.fetch_awattar = source("awattar", {"on": False}) + elsewhere.fetch_energycharts = source("energycharts", {"on": False}) + assert run(elsewhere.refresh(dt("2025-05-02T08:00Z"))) is True and calls == ["energycharts"] + + assert make_api(provider=None, entsoe_token=None, zone="DE-LU").provider == "energycharts" + + +# --------------------------------------------------------------------------- +# EWS Schönau +# --------------------------------------------------------------------------- + + +def ews_reply(start="2025-05-02T00:00:00.000+02:00", quarters=8, base=30.0, tomorrow=0): + """An EWS dynamic prices reply: today/tomorrow lists of quarter-hour {startsAt, total} with total in cents incl. VAT.""" + first = datetime.fromisoformat(start) + + def entries(offset, count): + """count quarter hours from first + offset quarters.""" + return [{"startsAt": (first + timedelta(minutes=15 * (offset + i))).isoformat(timespec="milliseconds"), "total": base + offset + i} for i in range(count)] + + return {"today": entries(0, quarters), "tomorrow": entries(96, tomorrow)} + + +def test_spotprice_ews_parse(my_predbat=None): + """EWS totals are quarter-hourly cents incl. VAT across today and tomorrow; a null total is a gap; euro-looking or malformed replies are rejected.""" + from spotprice import parse_ews_json + + intervals = parse_ews_json(ews_reply(quarters=3, tomorrow=2)) + assert [(start, end) for start, end, _v in intervals][:2] == [(dt("2025-05-01T22:00Z"), dt("2025-05-01T22:15Z")), (dt("2025-05-01T22:15Z"), dt("2025-05-01T22:30Z"))] + assert [value for _s, _e, value in intervals] == [30.0, 31.0, 32.0, 126.0, 127.0], intervals + assert intervals[3][0] == dt("2025-05-02T22:00Z") + gap = ews_reply(quarters=3) + gap["today"][1]["total"] = None + assert [start for start, _e, _v in parse_ews_json(gap)] == [dt("2025-05-01T22:00Z"), dt("2025-05-01T22:30Z")] + # Omitted quarters are not covered by their neighbour + omitted = ews_reply(quarters=4) + del omitted["today"][1] + assert parse_ews_json(omitted)[0][1] == dt("2025-05-01T22:15Z") + euros = ews_reply(quarters=4) + for entry in euros["today"]: + entry["total"] = 0.3 + for bad in (euros, {"today": [{"startsAt": 1746136800, "total": 30}]}, {"today": [{"total": 30}]}, {"today": "x"}, None): + try: + parse_ews_json(bad) + except SpotPriceError: + continue + raise AssertionError("expected SpotPriceError for {}".format(bad)) + assert parse_ews_json({"today": [], "tomorrow": None}) == [] + + +def test_spotprice_ews_fetch(my_predbat=None): + """fetch_ews sends the API key as X-API-Key, uses the totals as-is (no markup or VAT), and maps failures to SpotPriceError categories with per-source back-off.""" + import spotprice as spotprice_module + + seen = [] + api = make_api(provider="ews", entsoe_token=None, zone=None, ews_api_key="pub_dpa_test", markup=15, vat=0.19, charge_zones=[{"from": "00:00", "to": "00:00", "charge": 9}]) + + async def request(method, url, params=None, headers=None, data=None): + """Return the fixture.""" + seen.append((method, url, headers)) + return 200, ews_reply(quarters=96, tomorrow=96) + + api.http_request = request + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert api.sources_needed() == ["supplier"] + assert run(api.refresh(now)) is True + assert seen == [("GET", "https://api.ews-schoenau.de/v1/dynamicprices/EWS-OEKO-DYN", {"X-API-Key": "pub_dpa_test", "Accept": "application/json"})], seen + rates = api.build_import_rates() + assert len(rates) == 192 and rates[0] == (dt("2025-05-01T22:00Z"), dt("2025-05-01T22:15Z"), 30.0) + assert api.data_end() == dt("2025-05-03T22:00Z") + + recorded = [] + original = spotprice_module.record_api_call + spotprice_module.record_api_call = lambda service, success=True, reason=None: recorded.append((service, success, reason)) + try: + for reply, phrase, reason in ( + ((401, {"error": {"title": "Unauthorized", "status": 401, "detail": "Missing authorization"}}), "EWS rejected the API key (HTTP 401): Missing authorization", "auth_error"), + ((429, None), "rate limit", "rate_limit"), + ((500, None), "HTTP 500", "server_error"), + ((200, {"today": [], "tomorrow": []}), "no prices", "client_error"), + ((200, {"today": [{"startsAt": "2025-05-02T00:00:00.000+02:00", "total": 0.3}]}), "euros", "decode_error"), + ): + failing = make_api(provider="ews", entsoe_token=None, zone=None, ews_api_key="pub_dpa_secret") + + async def bad(method, url, params=None, headers=None, data=None, reply=reply): + """Return an error reply.""" + return reply + + failing.http_request = bad + recorded.clear() + assert run(failing.refresh(now)) is False + assert phrase in failing.last_error and "pub_dpa_secret" not in failing.last_error, (phrase, failing.last_error) + assert recorded == [("ews", False, reason)], (phrase, recorded) + assert failing.source_failures["supplier"] == 1 and failing.source_next_attempt["supplier"] == now + timedelta(minutes=5) + + async def network(method, url, params=None, headers=None, data=None): + """A connection failure.""" + raise asyncio.TimeoutError() + + offline = make_api(provider="ews", entsoe_token=None, zone=None, ews_api_key="k") + offline.http_request = network + recorded.clear() + assert run(offline.refresh(now)) is False and recorded == [("ews", False, "connection_error")] and "EWS request failed" in offline.last_error + finally: + spotprice_module.record_api_call = original + + +def test_spotprice_ews_inference_and_config(my_predbat=None): + """An EWS API key alone selects ews; with another supplier's key the provider must be set; provider ews without a key is a config error.""" + assert make_api(provider=None, entsoe_token=None, zone=None, ews_api_key="k").provider == "ews" + assert "octopus_de, ews" in make_api(provider=None, entsoe_token=None, zone=None, ews_api_key="k", octopus_de_api_key="o").config_error + missing = make_api(provider="ews", entsoe_token=None, zone=None) + assert "spotprice_ews_api_key is required" in missing.config_error and missing.sources_needed() == [] + + +def agreement_of(response, index=0): + """The index-th agreement of the first market location of an agreements response.""" + return response["data"]["account"]["properties"][0]["electricityMalos"][0]["agreements"][index] + + +TOU_DAY_NIGHT = { + "__typename": "TimeOfUseProductUnitRateInformation", + "rates": [ + {"latestGrossUnitRateCentsPerKwh": "22.0", "timeslotActivationRules": [{"activeFromTime": "22:00:00", "activeToTime": "06:00:00"}]}, + {"latestGrossUnitRateCentsPerKwh": "35.0", "timeslotActivationRules": [{"activeFromTime": "06:00:00", "activeToTime": "22:00:00"}]}, + ], +} + + +def test_spotprice_octopus_de_forecast_time_of_use(my_predbat=None): + """A forecast entry carrying several time-of-use rates is expanded by their rules over that entry's period, never collapsed to one rate; forecast slots are clipped to the agreement's validity.""" + from spotprice import parse_octopus_de_agreement + + # One forecast entry for the whole of 2 May (Berlin) holding a night and a day rate + entry = {"validFrom": "2025-05-02T00:00:00+02:00", "validTo": "2025-05-03T00:00:00+02:00", "unitRateInformation": TOU_DAY_NIGHT} + agreement = agreement_of(octopus_de_agreement(forecast=[entry])) + intervals = parse_octopus_de_agreement(agreement, dt("2025-05-01T22:00Z"), dt("2025-05-03T22:00Z")) + assert intervals == [ + (dt("2025-05-01T22:00Z"), dt("2025-05-02T04:00Z"), 22.0), + (dt("2025-05-02T04:00Z"), dt("2025-05-02T20:00Z"), 35.0), + (dt("2025-05-02T20:00Z"), dt("2025-05-02T22:00Z"), 22.0), + ], intervals + # A single rate without rules covers its whole entry + single = {"validFrom": "2025-05-02T00:00:00+02:00", "validTo": "2025-05-02T00:15:00+02:00", "unitRateInformation": {"__typename": "TimeOfUseProductUnitRateInformation", "rates": [{"latestGrossUnitRateCentsPerKwh": "27.5"}]}} + assert parse_octopus_de_agreement(agreement_of(octopus_de_agreement(forecast=[single])), dt("2025-05-01T22:00Z"), dt("2025-05-03T22:00Z")) == [(dt("2025-05-01T22:00Z"), dt("2025-05-01T22:15Z"), 27.5)] + # Several rates where one cannot be placed in time are an error, not a guess + no_timeslot = dict(entry, unitRateInformation={"__typename": "TimeOfUseProductUnitRateInformation", "rates": [{"latestGrossUnitRateCentsPerKwh": "22.0"}, TOU_DAY_NIGHT["rates"][1]]}) + try: + parse_octopus_de_agreement(agreement_of(octopus_de_agreement(forecast=[no_timeslot])), dt("2025-05-01T22:00Z"), dt("2025-05-03T22:00Z")) + raise AssertionError("rates without timeslots should raise") + except SpotPriceError as e: + assert "without a price or timeslot" in str(e), str(e) + + # 30 minute forecast slots across an agreement that starts at 22:15 and ends at 22:45 + slots = octopus_de_forecast(start="2025-05-01T22:00:00+00:00", count=2, minutes=30) + clipped = agreement_of(octopus_de_agreement(forecast=slots, valid_from="2025-05-01T22:15:00+00:00", valid_to="2025-05-01T22:45:00+00:00")) + assert parse_octopus_de_agreement(clipped, dt("2025-05-01T22:00Z"), dt("2025-05-03T22:00Z")) == [ + (dt("2025-05-01T22:15Z"), dt("2025-05-01T22:30Z"), 20.0), + (dt("2025-05-01T22:30Z"), dt("2025-05-01T22:45Z"), 21.0), + ] + + +def test_spotprice_octopus_de_dynamic_without_forecast(my_predbat=None): + """A dynamic agreement with no forecast publishes nothing - never its flat unitRateInformation - and keeps re-polling with an error.""" + from spotprice import octopus_de_is_dynamic, parse_octopus_de_agreement + + dynamic = octopus_de_agreement(product="DYNAMIC-OCTOPUS-24M") + assert octopus_de_is_dynamic(agreement_of(dynamic)) and not octopus_de_is_dynamic(agreement_of(octopus_de_agreement())) + assert octopus_de_is_dynamic(agreement_of(octopus_de_agreement(forecast=octopus_de_forecast(), product="SOMETHING-ELSE"))) + assert parse_octopus_de_agreement(agreement_of(dynamic), dt("2025-05-01T22:00Z"), dt("2025-05-03T22:00Z")) == [] + api, fake = make_octopus_de() + fake.agreements = dynamic + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert run(api.refresh(now)) is False + assert "has no forecast prices yet" in api.last_error and api.supplier_intervals == [] + api.publish(now) + assert "sensor.predbat_spotprice_import_rates" not in api.base.entities + assert api.base.entities["sensor.predbat_spotprice_status"]["state"] == "error" + assert api.source_next_attempt["supplier"] == now + timedelta(minutes=5) + # Once the forecast appears the next poll publishes it + fake.agreements = octopus_de_agreement(forecast=octopus_de_forecast(start="2025-05-02T08:00:00+00:00"), product="DYNAMIC-OCTOPUS-24M") + later = api.source_next_attempt["supplier"] + pin_now(api, later) + assert run(api.refresh(later)) is True and len(api.supplier_intervals) == 8 + + +def test_spotprice_octopus_de_market_location_selection(my_predbat=None): + """Active agreements on several market locations need spotprice_octopus_de_malo; the chosen location hands over to its next agreement inside the window; revoked agreements are skipped.""" + from spotprice import select_octopus_de_agreements + + fixed = agreement_of(octopus_de_agreement(valid_to="2025-05-03T00:00:00+02:00")) + following = dict(fixed, id="4712", isActive=False, validFrom="2025-05-03T00:00:00+02:00", validTo=None, unitRateInformation={"__typename": "SimpleProductUnitRateInformation", "latestGrossUnitRateCentsPerKwh": "28.0"}) + revoked = dict(following, id="4713", isRevoked=True, unitRateInformation={"__typename": "SimpleProductUnitRateInformation", "latestGrossUnitRateCentsPerKwh": "99.0"}) + expired = dict(fixed, id="4700", isActive=False, validFrom="2024-01-01T00:00:00+01:00", validTo="2025-01-01T00:00:00+01:00") + heat_pump = dict(fixed, id="5001", validTo=None, unitRateInformation=TOU_DAY_NIGHT) + response = { + "data": { + "account": { + "properties": [ + {"electricityMalos": [{"maloNumber": "50000000001", "agreements": [expired, revoked, following, fixed]}]}, + {"electricityMalos": [{"maloNumber": "50000000002", "agreements": [heat_pump]}]}, + ] + } + } + } + try: + select_octopus_de_agreements(response) + raise AssertionError("two market locations should raise") + except SpotPriceError as e: + assert "50000000001, 50000000002" in str(e) and "spotprice_octopus_de_malo" in str(e), str(e) + malo, chain = select_octopus_de_agreements(response, "50000000001") + assert malo == "50000000001" and [a["id"] for a in chain] == ["4711", "4712"], chain + for bad, phrase in ((("50000000009",), "50000000009 has no active agreement"),): + try: + select_octopus_de_agreements(response, *bad) + raise AssertionError(phrase) + except SpotPriceError as e: + assert phrase in str(e), str(e) + two_active = {"data": {"account": {"properties": [{"electricityMalos": [{"maloNumber": "1", "agreements": [fixed, dict(fixed, id="9")]}]}]}}} + try: + select_octopus_de_agreements(two_active) + raise AssertionError("two active agreements on one location should raise") + except SpotPriceError as e: + assert "several active agreements (4711, 9)" in str(e), str(e) + + api, fake = make_octopus_de(octopus_de_malo=50000000001) + fake.agreements = response + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert run(api.refresh(now)) is True + # The fixed agreement up to its end, then the following one; the revoked 99.0 never appears + assert api.supplier_intervals == [(dt("2025-04-30T22:00Z"), dt("2025-05-02T22:00Z"), 31.5), (dt("2025-05-02T22:00Z"), dt("2025-05-03T22:00Z"), 28.0)], api.supplier_intervals + unset, unset_fake = make_octopus_de() + unset_fake.agreements = response + pin_now(unset, now) + assert run(unset.refresh(now)) is False and "set spotprice_octopus_de_malo" in unset.last_error + + +def test_spotprice_octopus_de_berlin_time_and_dst(my_predbat=None): + """Timeslot rules are German local time whatever the Home Assistant timezone, through both DST changes.""" + from spotprice import parse_octopus_de_agreement + + tou = agreement_of(octopus_de_agreement(info=TOU_DAY_NIGHT)) + api, fake = make_octopus_de(local_tz=pytz.timezone("Europe/London")) + fake.agreements = octopus_de_agreement(info=TOU_DAY_NIGHT) + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert run(api.refresh(now)) is True + # 06:00 and 22:00 Berlin (CEST) are 04:00Z and 20:00Z - not London's 05:00Z and 21:00Z + boundaries = {start for start, _e, _v in api.supplier_intervals} + assert dt("2025-05-02T04:00Z") in boundaries and dt("2025-05-02T20:00Z") in boundaries and dt("2025-05-02T05:00Z") not in boundaries, sorted(boundaries) + # Spring forward: 22:00 CET on 28 March to 06:00 CEST on 29 March is 7 hours + spring = parse_octopus_de_agreement(tou, dt("2026-03-28T20:00Z"), dt("2026-03-29T22:00Z")) + assert (dt("2026-03-28T21:00Z"), dt("2026-03-29T04:00Z"), 22.0) in spring, spring + assert (dt("2026-03-29T04:00Z"), dt("2026-03-29T20:00Z"), 35.0) in spring, spring + # Fall back: 22:00 CEST on 25 October to 06:00 CET on 26 October is 9 hours + autumn = parse_octopus_de_agreement(tou, dt("2025-10-25T19:00Z"), dt("2025-10-26T23:00Z")) + assert (dt("2025-10-25T20:00Z"), dt("2025-10-26T05:00Z"), 22.0) in autumn, autumn + assert all(autumn[i][1] == autumn[i + 1][0] for i in range(len(autumn) - 1)) + + +def test_spotprice_supplier_cache_key(my_predbat=None): + """The cache name carries a digest of the configured Ostrom contract or Octopus Energy Germany account and market location, so changing them never restores another contract's prices.""" + names = { + "plain": make_api(provider="ostrom", entsoe_token=None, zone=None, ostrom_client_id="i", ostrom_client_secret="s").cache_filename(), + "contract_a": make_api(provider="ostrom", entsoe_token=None, zone=None, ostrom_client_id="i", ostrom_client_secret="s", ostrom_contract_id=1).cache_filename(), + "contract_b": make_api(provider="ostrom", entsoe_token=None, zone=None, ostrom_client_id="i", ostrom_client_secret="s", ostrom_contract_id=2).cache_filename(), + "octo": make_api(provider="octopus_de", entsoe_token=None, zone=None, octopus_de_api_key="k").cache_filename(), + "octo_account": make_api(provider="octopus_de", entsoe_token=None, zone=None, octopus_de_api_key="k", octopus_de_account="A-1").cache_filename(), + "octo_malo": make_api(provider="octopus_de", entsoe_token=None, zone=None, octopus_de_api_key="k", octopus_de_account="A-1", octopus_de_malo="5").cache_filename(), + } + assert names["plain"].startswith("ostrom_none_") and names["octo"].startswith("octopus_de_none_"), names + assert len(set(names.values())) == len(names), names + assert make_api(provider="ostrom", entsoe_token=None, zone=None, ostrom_client_id="i", ostrom_client_secret="s", ostrom_contract_id=1).cache_filename() == names["contract_a"] + assert make_api(provider="energycharts", entsoe_token=None, ostrom_contract_id=1).cache_filename() == "energycharts_de_lu" + + # A looked-up account does not change the name mid-run + api, _fake = make_octopus_de() + before = api.cache_filename() + pin_now(api, dt("2025-05-02T08:00Z")) + run(api.fetch_supplier(dt("2025-05-02T08:00Z"))) + assert api.octopus_de_account == "A-1234ABCD" and api.cache_filename() == before + + storage = FakeStorage() + saved = make_api(provider="ostrom", entsoe_token=None, zone=None, ostrom_client_id="i", ostrom_client_secret="s", ostrom_contract_id=1, storage=storage) + saved.supplier_intervals = [(dt("2025-05-02T08:00Z"), dt("2025-05-02T09:00Z"), 30.0)] + saved.fetched["supplier"] = dt("2025-05-02T08:00Z") + run(saved.save_cache()) + other = make_api(provider="ostrom", entsoe_token=None, zone=None, ostrom_client_id="i", ostrom_client_secret="s", ostrom_contract_id=2, storage=storage) + run(other.load_cache()) + assert other.supplier_intervals == [] + + +def test_spotprice_awattar_fallback_hourly(my_predbat=None): + """When aWATTar falls back, quarter-hourly spot prices are averaged to the hours the HOURLY tariff bills; part-covered hours are left out; aWATTar's own hours pass through.""" + from spotprice import average_to_hourly + + base = dt("2025-05-02T10:00Z") + quarters = [(base + timedelta(minutes=15 * i), base + timedelta(minutes=15 * (i + 1)), float(10 * (i + 1))) for i in range(8)] + assert average_to_hourly(quarters) == [(base, base + timedelta(hours=1), 25.0), (base + timedelta(hours=1), base + timedelta(hours=2), 65.0)] + assert average_to_hourly(quarters[:7]) == [(base, base + timedelta(hours=1), 25.0)] + # Time-weighted: a 30 minute interval counts twice a quarter hour + mixed = [(base, base + timedelta(minutes=30), 10.0), (base + timedelta(minutes=30), base + timedelta(minutes=45), 40.0), (base + timedelta(minutes=45), base + timedelta(hours=1), 40.0)] + assert average_to_hourly(mixed) == [(base, base + timedelta(hours=1), 25.0)], average_to_hourly(mixed) + hourly = [(base, base + timedelta(hours=1), 5.0)] + assert average_to_hourly(hourly) == hourly + + api = make_api(provider="awattar", entsoe_token=None, zone="DE-LU") + + async def down(start, end): + """aWATTar outage.""" + raise SpotPriceError("aWATTar returned HTTP 503") + + async def quarter_hourly(start, end): + """Energy-Charts quarter hours.""" + return quarters + + api.fetch_awattar = down + api.fetch_energycharts = quarter_hourly + assert run(api.refresh(dt("2025-05-02T08:00Z"))) is True + # The quarter hours are held as published; only the import price is averaged + assert api.spot_source == "energycharts" and api.spot_intervals == quarters + assert [(s_, e) for s_, e, _r in api.build_import_rates()] == [(s_, e) for s_, e, _v in average_to_hourly(quarters)] + # With aWATTar itself as the source nothing is averaged + api.spot_source = "awattar" + assert len(api.build_import_rates()) == 8 + # Other providers keep the quarter hours + plain = make_api(provider="energycharts", entsoe_token=None) + plain.fetch_energycharts = quarter_hourly + run(plain.refresh(dt("2025-05-02T08:00Z"))) + assert len(plain.build_import_rates()) == 8 + + +def test_spotprice_awattar_fallback_export_native(my_predbat=None): + """In a -20/-10/40/50 EUR/MWh hour from a quarter-hourly fallback, import is billed at the hourly average while export keeps each quarter: negative ones are zeroed or priced on their own.""" + base = dt("2025-05-02T10:00Z") + quarters = [(base + timedelta(minutes=15 * i), base + timedelta(minutes=15 * (i + 1)), price) for i, price in enumerate((-20.0, -10.0, 40.0, 50.0))] + for export_mode, zero, expected in (("spot", True, [0.0, 0.0, 4.0, 5.0]), ("spot", False, [-2.0, -1.0, 4.0, 5.0]), ("fixed", True, [0.0, 0.0, 8.0, 8.0])): + api = make_api(provider="awattar", entsoe_token=None, zone="DE-LU", markup=0, vat=0, markup_percent=0, export_mode=export_mode, export_rate=8.0, export_zero_on_negative=zero) + api.spot_intervals = quarters + api.spot_source = "energycharts" + # Import: one hour at the average 15 EUR/MWh = 1.5 c/kWh + assert api.build_import_rates() == [(base, base + timedelta(hours=1), 1.5)], api.build_import_rates() + export = api.build_export_rates() + assert [rate for _s, _e, rate in export] == expected, (export_mode, zero, export) + assert all(end - start == timedelta(minutes=15) for start, end, _r in export) + + +def test_spotprice_octopus_de_handover_single_successor(my_predbat=None): + """Each handover takes the one agreement starting at (or first after) the current end; two claiming the same start are an error naming both; agreements that would overlap are never chained.""" + from spotprice import select_octopus_de_agreements + + def simple(agreement_id, valid_from, valid_to, price, active=False, revoked=False): + """A fixed-rate agreement.""" + return { + "id": agreement_id, + "isActive": active, + "isRevoked": revoked, + "validFrom": valid_from, + "validTo": valid_to, + "product": {"code": "OCTOPUS-FIX"}, + "unitRateInformation": {"__typename": "SimpleProductUnitRateInformation", "latestGrossUnitRateCentsPerKwh": str(price)}, + "unitRateForecast": [], + } + + def response(*agreements): + """One market location holding the agreements.""" + return {"data": {"account": {"properties": [{"electricityMalos": [{"maloNumber": "1", "agreements": list(agreements)}]}]}}} + + current = simple("A", "2025-01-01T00:00:00+00:00", "2025-05-02T22:00:00+00:00", 30, active=True) + second = simple("B", "2025-05-02T22:00:00+00:00", "2025-05-03T10:00:00+00:00", 28) + # C starts after B began - it overlaps B and must not be chained after A; D follows B + overlapping = simple("C", "2025-05-03T00:00:00+00:00", None, 99) + third = simple("D", "2025-05-03T10:00:00+00:00", None, 26) + _malo, chain = select_octopus_de_agreements(response(third, overlapping, current, second)) + assert [a["id"] for a in chain] == ["A", "B", "D"], [a["id"] for a in chain] + # A gap is allowed: the next agreement is the first that starts after the end + late = simple("E", "2025-05-03T00:00:00+00:00", None, 25) + assert [a["id"] for a in select_octopus_de_agreements(response(current, late))[1]] == ["A", "E"] + # Two non-revoked agreements claiming the same start + twin = simple("B2", "2025-05-02T22:00:00+00:00", None, 27) + try: + select_octopus_de_agreements(response(current, second, twin)) + raise AssertionError("two successors should raise") + except SpotPriceError as e: + assert "(B, B2)" in str(e), str(e) + # ...unless one of them is revoked + assert [a["id"] for a in select_octopus_de_agreements(response(current, second, dict(twin, isRevoked=True)))[1]] == ["A", "B"] + + api, fake = make_octopus_de() + fake.agreements = response(third, overlapping, current, second) + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert run(api.refresh(now)) is True + assert api.supplier_intervals == [(dt("2025-04-30T22:00Z"), dt("2025-05-02T22:00Z"), 30.0), (dt("2025-05-02T22:00Z"), dt("2025-05-03T10:00Z"), 28.0), (dt("2025-05-03T10:00Z"), dt("2025-05-03T22:00Z"), 26.0)], api.supplier_intervals + assert all(api.supplier_intervals[i][1] <= api.supplier_intervals[i + 1][0] for i in range(len(api.supplier_intervals) - 1)) + + # Overlapping rates from any source are refused outright + import spotprice as spotprice_module + + original = spotprice_module.parse_octopus_de_agreement + spotprice_module.parse_octopus_de_agreement = lambda agreement, start, end, tz=None: [(dt("2025-05-02T00:00Z"), dt("2025-05-02T02:00Z"), 1.0), (dt("2025-05-02T01:00Z"), dt("2025-05-02T03:00Z"), 2.0)] + try: + clash, clash_fake = make_octopus_de() + pin_now(clash, now) + assert run(clash.refresh(now)) is False and "overlapping rates" in clash.last_error and clash.supplier_intervals == [] + finally: + spotprice_module.parse_octopus_de_agreement = original + + +def test_spotprice_octopus_de_flat_only_when_explicit(my_predbat=None): + """Without a forecast a flat price comes only from an explicit fixed rate; a dynamic product (by code, name or contract type), a time-of-use product with a simple rate, or rates without a type are errors.""" + from spotprice import octopus_de_is_dynamic, parse_octopus_de_agreement + + window = (dt("2025-05-01T22:00Z"), dt("2025-05-03T22:00Z")) + base = agreement_of(octopus_de_agreement()) + assert parse_octopus_de_agreement(base, *window) == [(window[0], window[1], 31.5)] + for product in ({"code": "OCT-24M", "displayName": "Octopus Dynamic"}, {"code": "X", "fullName": "Strom dynamisch"}, {"code": "X", "termsContractType": "DYNAMIC"}): + assert octopus_de_is_dynamic(dict(base, product=product)), product + assert parse_octopus_de_agreement(dict(base, product=product), *window) == [] + for bad in ( + dict(base, product={"code": "X", "isTimeOfUse": True}), + dict(base, unitRateInformation={"latestGrossUnitRateCentsPerKwh": "31.5"}), + dict(base, unitRateInformation={"__typename": "TimeOfUseProductUnitRateInformation", "rates": [{"latestGrossUnitRateCentsPerKwh": "31.5"}]}), + dict(base, unitRateInformation={"rates": TOU_DAY_NIGHT["rates"]}), + ): + try: + parse_octopus_de_agreement(bad, *window) + raise AssertionError("expected an error for {}".format(bad)) + except SpotPriceError as e: + assert "no price forecast and no explicit fixed or time-of-use rates" in str(e), str(e) + assert len(parse_octopus_de_agreement(dict(base, unitRateInformation=TOU_DAY_NIGHT, product={"code": "X", "isTimeOfUse": True}), *window)) == 5 + + +def test_spotprice_octopus_de_autumn_repeated_hour(my_predbat=None): + """On the autumn change a 02:00-03:00 rule covers both 02:00-03:00 hours (00:00Z-02:00Z), the first included, with no gap or overlap.""" + from spotprice import parse_octopus_de_agreement + + info = { + "__typename": "TimeOfUseProductUnitRateInformation", + "rates": [ + {"latestGrossUnitRateCentsPerKwh": "50.0", "timeslotActivationRules": [{"activeFromTime": "02:00:00", "activeToTime": "03:00:00"}]}, + {"latestGrossUnitRateCentsPerKwh": "20.0", "timeslotActivationRules": [{"activeFromTime": "03:00:00", "activeToTime": "02:00:00"}]}, + ], + } + agreement = agreement_of(octopus_de_agreement(info=info)) + intervals = parse_octopus_de_agreement(agreement, dt("2025-10-25T22:00Z"), dt("2025-10-26T23:00Z")) + assert intervals[:3] == [(dt("2025-10-25T22:00Z"), dt("2025-10-26T00:00Z"), 20.0), (dt("2025-10-26T00:00Z"), dt("2025-10-26T02:00Z"), 50.0), (dt("2025-10-26T02:00Z"), dt("2025-10-26T23:00Z"), 20.0)], intervals + # Spring: 02:00-03:00 does not exist, so that rule has no time and the day is contiguous + spring = parse_octopus_de_agreement(agreement, dt("2026-03-28T23:00Z"), dt("2026-03-29T22:00Z")) + assert all(spring[i][1] == spring[i + 1][0] for i in range(len(spring) - 1)), spring + assert spring[0][0] == dt("2026-03-28T23:00Z") and spring[-1][1] == dt("2026-03-29T22:00Z") + assert all(rate == 20.0 for _s, _e, rate in spring), spring + + +def test_spotprice_octopus_de_error_codes(my_predbat=None): + """KT-CT-1125 (token keys rotated) gets a new token and one retry; KT-CT-1139 on a query is an auth_error at once, with no retry.""" + import spotprice as spotprice_module + + recorded = [] + original = spotprice_module.record_api_call + spotprice_module.record_api_call = lambda service, success=True, reason=None: recorded.append((service, success, reason)) + try: + now = dt("2025-05-02T08:00Z") + api, fake = make_octopus_de() + pin_now(api, now) + fake.queued["agreements"] = [kraken_error("KT-CT-1125", "Token keys have rotated.")] + assert len(run(api.fetch_supplier(now))) == 8 and fake.count("token") == 2 and fake.count("agreements") == 2 + + api, fake = make_octopus_de() + pin_now(api, now) + fake.queued["agreements"] = [kraken_error("KT-CT-1139", "Authentication failed.")] + recorded.clear() + try: + run(api.fetch_supplier(now)) + raise AssertionError("1139 should raise") + except SpotPriceError as e: + assert "Authentication failed" in str(e) + assert fake.count("token") == 1 and fake.count("agreements") == 1, fake.calls + assert recorded[-1] == ("octopus_de", False, "auth_error"), recorded + + # Even alongside a token code, a rejected key is not retried with a new token + api, fake = make_octopus_de() + pin_now(api, now) + both = kraken_error("KT-CT-1139", "Authentication failed.") + both[1]["errors"].append({"message": "expired", "extensions": {"errorCode": "KT-CT-1124"}}) + fake.queued["agreements"] = [both] + try: + run(api.fetch_supplier(now)) + raise AssertionError("1139 with 1124 should raise") + except SpotPriceError: + pass + assert fake.count("token") == 1 and fake.count("agreements") == 1, fake.calls + finally: + spotprice_module.record_api_call = original + + +def test_spotprice_supplier_cache_identity(my_predbat=None): + """Cached supplier prices remember the contract or market location they were for, and are dropped when the one detected now differs.""" + storage = FakeStorage() + now = dt("2025-05-02T08:00Z") + first, _fake = make_ostrom(storage=storage) + pin_now(first, now) + assert run(first.refresh(now)) is True + cached = storage.data[("spotprice", first.cache_filename())] + assert cached["supplier_identity"] == "ostrom:100523456" + + # Same contract detected after a restart: the cache stands + same, same_fake = make_ostrom(storage=storage) + run(same.load_cache()) + assert same.supplier_intervals and same.cached_supplier_identity == "ostrom:100523456" + same.note_supplier_identity("ostrom:100523456") + assert same.supplier_intervals + + # The account now has a different active contract: the restored prices are discarded on detection + moved, moved_fake = make_ostrom(storage=storage) + run(moved.load_cache()) + assert moved.supplier_intervals + moved_fake.contracts = {"data": [dict(OSTROM_CONTRACTS["data"][0], id=200000001)]} + moved_fake.api_replies["/spot-prices"] = [(503, None)] + pin_now(moved, now) + assert run(moved.refresh(now)) is False + assert moved.supplier_intervals == [] and any("discarding" in line for line in moved.base.logs), moved.base.logs + + # Octopus: account and market location form the identity + octo, octo_fake = make_octopus_de() + octo.cached_supplier_identity = "octopus_de:A-1234ABCD:50000000009" + octo.supplier_intervals = [(now, now + timedelta(hours=1), 99.0)] + pin_now(octo, now) + run(octo.fetch_supplier(now)) + assert octo.supplier_identity == "octopus_de:A-1234ABCD:50000000001" and any("discarding" in line for line in octo.base.logs) + + +def test_spotprice_ostrom_status_handling(my_predbat=None): + """ACTIVE matches in any case; when no contract has a status field at all the single dynamic contract is used with one warning; otherwise a missing status is not active.""" + from spotprice import select_ostrom_contract + + lower = {"data": [dict(OSTROM_CONTRACTS["data"][0], status=" active "), OSTROM_CONTRACTS["data"][1]]} + assert select_ostrom_contract(lower)["id"] == 100523456 + assert select_ostrom_contract(lower, "100523456")["id"] == 100523456 + + def without_status(contract, **changes): + """A copy of contract with no status key.""" + copy = {key: value for key, value in contract.items() if key != "status"} + copy.update(changes) + return copy + + warnings = [] + no_status = {"data": [without_status(OSTROM_CONTRACTS["data"][0]), without_status(OSTROM_CONTRACTS["data"][1])]} + assert select_ostrom_contract(no_status, warn=warnings.append)["id"] == 100523456 + assert len(warnings) == 1 and "no status field" in warnings[0] + assert select_ostrom_contract(no_status, 100523456, warn=warnings.append)["id"] == 100523456 + for data, contract_id in (({"data": [without_status(OSTROM_CONTRACTS["data"][0]), without_status(OSTROM_CONTRACTS["data"][0], id=5)]}, None), (no_status, 100523999)): + try: + select_ostrom_contract(data, contract_id) + raise AssertionError("expected an error") + except SpotPriceError as e: + assert "carry no status" in str(e), str(e) + # One contract with a status makes the others' missing status count as not active + mixed = {"data": [without_status(OSTROM_CONTRACTS["data"][0]), OSTROM_CONTRACTS["data"][1]]} + try: + select_ostrom_contract(mixed) + raise AssertionError("expected no active contract") + except SpotPriceError as e: + assert "no active Ostrom contract" in str(e), str(e) + + # Through the component the warning is logged once however often contracts are read + api, fake = make_ostrom() + fake.contracts = no_status + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + assert run(api.refresh(now)) is True + api.ostrom_contract = None + run(api.fetch_supplier(now)) + assert sum("no status field" in line for line in api.base.logs) == 1, api.base.logs + + +def test_spotprice_cache_credential_swap(my_predbat=None): + """Swapping a supplier credential with no selector set changes the cache name, so the old account's prices are never restored; only a salted 12-character digest appears, never the credential.""" + from spotprice import CACHE_DIGEST_SALT + import hashlib + + cases = [ + ("tibber", {"tibber_token": "tibber-old"}, {"tibber_token": "tibber-new"}), + ("ostrom", {"ostrom_client_id": "client-old", "ostrom_client_secret": "s"}, {"ostrom_client_id": "client-new", "ostrom_client_secret": "s"}), + ("octopus_de", {"octopus_de_api_key": "sk_live_old"}, {"octopus_de_api_key": "sk_live_new"}), + ("ews", {"ews_api_key": "pub_dpa_old"}, {"ews_api_key": "pub_dpa_new"}), + ] + now = dt("2025-05-02T08:00Z") + for provider, old, new in cases: + storage = FakeStorage() + before = make_api(provider=provider, entsoe_token=None, zone=None, storage=storage, **old) + after = make_api(provider=provider, entsoe_token=None, zone=None, storage=storage, **new) + same = make_api(provider=provider, entsoe_token=None, zone=None, storage=storage, **old) + assert before.cache_filename() != after.cache_filename() and before.cache_filename() == same.cache_filename(), provider + credential = list(old.values())[0] + expected = hashlib.sha256((CACHE_DIGEST_SALT + credential + "|" * {"tibber": 1, "ostrom": 1, "octopus_de": 2}.get(provider, 0)).encode("utf-8")).hexdigest()[:12] + assert before.cache_filename() == "{}_none_{}".format(provider, expected), (provider, before.cache_filename()) + # Prices cached under the old credential are not restored, nor treated as fresh, under the new one + intervals = [(now, now + timedelta(hours=1), 0.3 if provider == "tibber" else 30.0)] + if provider == "tibber": + before.tibber_intervals = intervals + else: + before.supplier_intervals = intervals + before.fetched[before.import_source()] = now + run(before.save_cache()) + run(after.load_cache()) + assert after.source_intervals() == [] and after.fetched_at is None and after.refresh_due(now), provider + run(same.load_cache()) + assert same.source_intervals() == intervals, provider + # The credential itself is written nowhere: not in the name, not in the stored blob + for (_module, filename), blob in storage.data.items(): + assert credential not in filename and credential not in repr(blob), (provider, filename) + + +def test_spotprice_octopus_de_bare_jwt(my_predbat=None): + """The Kraken JWT is sent bare - no Bearer or JWT prefix - and the token request carries no Authorization at all.""" + api, fake = make_octopus_de() + now = dt("2025-05-02T08:00Z") + pin_now(api, now) + run(api.fetch_supplier(now)) + token_headers = [call[3] for call in fake.calls if call[0] == "token"] + query_headers = [call[3] for call in fake.calls if call[0] != "token"] + assert all("Authorization" not in headers for headers in token_headers) + assert query_headers and all(headers["Authorization"] == make_jwt(fake.token_exp) for headers in query_headers), query_headers + assert not any(headers["Authorization"].split(" ")[0] in ("JWT", "Bearer") for headers in query_headers) + + +def test_spotprice_ostrom_move_house(my_predbat=None): + """Without spotprice_ostrom_contract_id the contract is looked up again daily and straight after a 404/422 on the postcode; a move switches postcode and drops the old prices; no active contract is an error with nothing held.""" + moved = {"data": [dict(OSTROM_CONTRACTS["data"][0], status="TERMINATED"), dict(OSTROM_CONTRACTS["data"][0], id=200000001, address={"zip": "80331", "city": "Munich"})]} + now = dt("2025-05-02T08:00Z") + + # Daily re-check: same day no lookup, next day the new contract and postcode + api, fake = make_ostrom() + pin_now(api, now) + assert run(api.refresh(now)) is True and fake.count("/contracts") == 1 + pin_now(api, now + timedelta(hours=7)) + assert run(api.refresh(now + timedelta(hours=7))) is True and fake.count("/contracts") == 1 + fake.contracts = moved + day_later = now + timedelta(days=1) + pin_now(api, day_later) + assert run(api.refresh(day_later)) is True + assert fake.count("/contracts") == 2 and fake.calls[-1][2]["zip"] == "80331", fake.calls[-1] + assert api.ostrom_contract["id"] == 200000001 and any("contract changed" in line for line in api.base.logs) + + # A move found while the new postcode's prices cannot be fetched yet: the old postcode's prices are not kept + api, fake = make_ostrom() + pin_now(api, now) + run(api.refresh(now)) + assert api.supplier_intervals + fake.contracts = moved + fake.api_replies["/spot-prices"] = [(503, None)] + pin_now(api, day_later) + assert run(api.refresh(day_later)) is False and api.supplier_intervals == [] and api.ostrom_contract["id"] == 200000001 + # ...also when the contract keeps its number and only the address changes + api, fake = make_ostrom() + pin_now(api, now) + run(api.refresh(now)) + fake.contracts = {"data": [dict(OSTROM_CONTRACTS["data"][0], address={"zip": "80331"})]} + fake.api_replies["/spot-prices"] = [(503, None)] + pin_now(api, day_later) + assert run(api.refresh(day_later)) is False and api.supplier_intervals == [] and api.ostrom_contract["address"]["zip"] == "80331" + + # A 404 for the old postcode re-checks at once and retries with the new one in the same fetch + api, fake = make_ostrom() + pin_now(api, now) + run(api.refresh(now)) + fake.contracts = moved + fake.api_replies["/spot-prices"] = [(404, {"detail": "zip not found"})] + later = now + timedelta(hours=1) + pin_now(api, later) + api.fetched["supplier"] = None + assert run(api.refresh(later)) is True, api.last_error + assert [call[2]["zip"] for call in fake.calls if call[1].endswith("/spot-prices")] == ["10997", "10997", "80331"] + assert fake.count("/contracts") == 2 + + # A 422 with the contract unchanged is reported, not retried in a loop + api, fake = make_ostrom() + pin_now(api, now) + run(api.refresh(now)) + fake.api_replies["/spot-prices"] = [(422, {"detail": "bad zip"})] + api.fetched["supplier"] = None + assert run(api.refresh(later)) is False and "HTTP 422" in api.last_error and fake.count("/contracts") == 2 and fake.count("/spot-prices") == 2 + + # Server errors do not trigger a re-check + api, fake = make_ostrom() + pin_now(api, now) + run(api.refresh(now)) + fake.api_replies["/spot-prices"] = [(503, None)] + api.fetched["supplier"] = None + assert run(api.refresh(later)) is False and fake.count("/contracts") == 1 + + # Moved out with no active contract: an error, and the old postcode's prices are no longer held + api, fake = make_ostrom() + pin_now(api, now) + run(api.refresh(now)) + assert api.supplier_intervals + fake.contracts = {"data": [dict(OSTROM_CONTRACTS["data"][0], status="TERMINATED")]} + pin_now(api, day_later) + assert run(api.refresh(day_later)) is False and "no active Ostrom contract" in api.last_error + assert api.supplier_intervals == [] and api.ostrom_contract is None + api.publish(day_later) + assert api.base.entities["sensor.predbat_spotprice_status"]["state"] == "error" + + # A configured contract is not re-checked daily, nor after a 404 + fixed, fixed_fake = make_ostrom(ostrom_contract_id=100523456) + pin_now(fixed, now) + run(fixed.refresh(now)) + pin_now(fixed, day_later) + run(fixed.refresh(day_later)) + fixed_fake.api_replies["/spot-prices"] = [(404, None)] + fixed.fetched["supplier"] = None + assert run(fixed.refresh(day_later + timedelta(hours=1))) is False and fixed_fake.count("/contracts") == 1 + + SPOTPRICE_TESTS = [ test_spotprice_entsoe_a03_gap_fill, test_spotprice_entsoe_a01_missing_point_not_filled, @@ -1394,6 +2754,38 @@ def tibber(**kwargs): test_spotprice_refresh_all_blocked, test_spotprice_omitted_timestamps, test_spotprice_tibber_cache_key, + test_spotprice_ostrom_parse, + test_spotprice_ostrom_contract_selection, + test_spotprice_ostrom_fetch_and_token_cache, + test_spotprice_ostrom_auth_errors, + test_spotprice_ostrom_all_in_and_refresh, + test_spotprice_ostrom_provider_inference, + test_spotprice_octopus_de_dynamic, + test_spotprice_octopus_de_fixed_and_time_of_use, + test_spotprice_octopus_de_auth_and_errors, + test_spotprice_octopus_de_inference_and_config, + test_spotprice_awattar_parse_and_fetch, + test_spotprice_awattar_markup_percent, + test_spotprice_awattar_fallback, + test_spotprice_ews_parse, + test_spotprice_ews_fetch, + test_spotprice_ews_inference_and_config, + test_spotprice_octopus_de_forecast_time_of_use, + test_spotprice_octopus_de_dynamic_without_forecast, + test_spotprice_octopus_de_market_location_selection, + test_spotprice_octopus_de_berlin_time_and_dst, + test_spotprice_supplier_cache_key, + test_spotprice_awattar_fallback_hourly, + test_spotprice_awattar_fallback_export_native, + test_spotprice_octopus_de_handover_single_successor, + test_spotprice_octopus_de_flat_only_when_explicit, + test_spotprice_octopus_de_autumn_repeated_hour, + test_spotprice_octopus_de_error_codes, + test_spotprice_supplier_cache_identity, + test_spotprice_ostrom_status_handling, + test_spotprice_cache_credential_swap, + test_spotprice_octopus_de_bare_jwt, + test_spotprice_ostrom_move_house, ] diff --git a/docs/components.md b/docs/components.md index 1d7edc95b..df730d229 100644 --- a/docs/components.md +++ b/docs/components.md @@ -1786,24 +1786,35 @@ Builds import (and optionally export) rates for any European bidding zone from t Most European dynamic tariffs are priced as the spot price plus a supplier markup plus grid fees and levies, with VAT on top, and this component applies exactly that: ```text -import rate = (spot EUR/MWh x spotprice_exchange_rate / 10 + spotprice_markup + charge zone) x (1 + spotprice_vat) +spot = spot EUR/MWh x spotprice_exchange_rate / 10 +import rate = (spot + |spot| x spotprice_markup_percent / 100 + spotprice_markup + charge zone) x (1 + spotprice_vat) ``` +`spotprice_markup_percent` is 0 unless set (3 for `awattar`), so for most tariffs the formula is simply spot + markup + charge zone, plus VAT. + With the default exchange rate of 1 the result is in euro cents per kWh. The markup, charge zones and export rates are entered in your minor currency unit per kWh (as set by `currency_symbols`) and **exclude VAT**; `spotprice_vat` is a fraction (`0.19` for 19%). Prices are kept at their native resolution - 15 minutes for most of Europe since October 2025, 30 minutes for Ireland's SEM, 60 minutes where a market still publishes hourly prices. -Tomorrow's prices are published around 12:00-13:00 CET; from 12:00 CET the component checks every 15 minutes until it holds the whole of the next market day (which runs midnight to midnight CET/CEST everywhere - 23:00 to 23:00 local time in Ireland and Portugal, 01:00 to 01:00 in Finland and the Baltics), otherwise it refreshes every 6 hours. Failed fetches back off from 5 minutes up to 2 hours - each source on its own, so a spot price outage never delays Tibber prices - and the last prices fetched are cached so they survive a restart. +Tomorrow's prices are published around 12:00-13:00 CET; from 12:00 CET the component checks every 15 minutes until it holds the whole of the next market day (which runs midnight to midnight CET/CEST everywhere - 23:00 to 23:00 local time in Ireland and Portugal, 01:00 to 01:00 in Finland and the Baltics), otherwise it refreshes every 6 hours. Failed fetches back off from 5 minutes up to 2 hours - each source on its own, so a spot price outage never delays a supplier's prices - and the last prices fetched are cached so they survive a restart. -Price sources (`spotprice_provider`; when it is not set the default is `tibber` if `spotprice_tibber_token` is set, otherwise `energycharts`): +Price sources (`spotprice_provider`; when it is not set and the credentials of exactly one supplier (Tibber, Ostrom, Octopus Energy Germany, EWS) are set, that supplier is used, otherwise `energycharts`. Credentials for more than one supplier without a `spotprice_provider` are reported as a configuration error, so set the provider explicitly): - `entsoe` - the [ENTSO-E Transparency Platform](https://transparency.entsoe.eu/) day-ahead prices (document A44). Needs a free security token: register on the platform, then email with the subject "Restful API access" and generate the token in your account settings. If the token is missing or ENTSO-E fails, the component falls back to Energy-Charts automatically and logs it once (and again once ENTSO-E recovers). The fallback only runs this way round: with `energycharts` ENTSO-E is never used, even if a token is set. - `energycharts` - [Energy-Charts](https://api.energy-charts.info/) from Fraunhofer ISE, no key needed. Prices for AT, BE, CH, CZ, DE-LU, DK1, DK2, FR, HU, IT-North, NL, NO2, PL, SE4 and SI are licensed CC BY 4.0 (Bundesnetzagentur | SMARD.de); Energy-Charts states that prices for other zones are for private use only. Ireland (IE-SEM) is not available from Energy-Charts, so it needs `entsoe` with a token: without one the component logs a single error, shows it on the status sensor and fetches nothing until the configuration is fixed, and an ENTSO-E failure there is not retried against Energy-Charts. +- `awattar` - the [aWATTar](https://www.awattar.de/) market data API (`api.awattar.de` for `DE-LU`, `api.awattar.at` for `AT`), no key needed. These are the hourly prices tado Energy's dynamic HOURLY tariff (formerly aWATTar's) is billed on; the provider keeps the aWATTar name because the price API still does. The published tariff is the hourly spot price plus 3% (`spotprice_markup_percent`, which defaults to 3 for this provider) plus 1.5 ct/kWh, plus grid fees, levies and taxes passed on at cost, plus VAT. Set `spotprice_markup` to 1.5 plus your grid fees and levies per kWh (net, from your tariff sheet) and `spotprice_vat` to `0.19` (DE) or `0.20` (AT); check your own tariff sheet, as older contracts may differ. If aWATTar fails the component falls back to ENTSO-E (with a token) and then Energy-Charts, and averages their quarter-hourly prices to the hourly prices the tariff bills for import (an hour is left out unless all four quarters are known); spot-linked export and the negative-price rule keep the quarter hours, so a negative quarter is zeroed or priced on its own. Other zones are not published by aWATTar and go straight to those fallbacks. + - Unverified assumptions: the 3% is charged on the absolute spot price, so it is a cost in negative hours too. aWATTar's Austrian tariff page says so; the German support article only says "× 1.03", which would instead make negative hours slightly more negative. The difference is 3% of the negative price. Whether existing tado Energy contracts still use exactly this formula has not been checked against a real bill. - `tibber` - [Tibber](https://developer.tibber.com/)'s own end-user price, read with your personal access token. Tibber's price already includes markup, grid fees and VAT, so `spotprice_markup`, `spotprice_vat` and the charge zones are **not** applied to it. Quarter-hourly prices are requested. +- `ostrom` - [Ostrom](https://www.ostrom.de/) (Germany) dynamic tariff, read from the [Ostrom API](https://docs.ostrom-api.io/). Create a production client in the [Ostrom developer portal](https://developer.ostrom-api.io/) and set its client ID and secret. The component uses your active dynamic contract (set `spotprice_ostrom_contract_id` if the account has more than one active contract; terminated or pending contracts are never used, and with no active contract the status shows an error - only if Ostrom returns no contract status at all is the single dynamic contract used, with a warning). Unless `spotprice_ostrom_contract_id` is set, the contract is looked up again once a day and straight after Ostrom rejects the postcode, so after a move the new contract's postcode is priced and the old one's prices are dropped and asks for prices at the contract's postcode, which is what makes Ostrom include the local taxes, levies and variable grid fees. The price used is Ostrom's documented variable price per kWh - the spot price plus taxes, levies and grid fees, VAT included - so, as with Tibber, `spotprice_markup`, `spotprice_vat` and the charge zones are **not** applied. Ostrom's monthly base fee and fixed grid fees are not per kWh and are left out. Ostrom's API offers hourly prices only (its only documented resolution is `HOUR`), with tomorrow's arriving between 15:00 and 17:00 CET. A fixed-price Ostrom contract (Simply Fair) is reported as an error: set its price with `rates_import` instead. + - Unverified assumptions: not yet tried against a live Ostrom account. Whether the taxes and levies Ostrom returns follow time-variable grid fees (section 14a module 3) is not documented; if your grid fees vary by time of day, compare with your bill. +- `octopus_de` - [Octopus Energy Germany](https://octopusenergy.de/), read from its Kraken GraphQL API with an API key (`spotprice_octopus_de_api_key`). The component prices the market location (MaLo) with an active electricity agreement; if the account has several - a household meter and a heat pump meter, say - the status shows an error until you set `spotprice_octopus_de_malo` to the 11-digit MaLo number to price (and `spotprice_octopus_de_account`, e.g. `A-1234ABCD`, if the key can see more than one account). A dynamic agreement is priced only from its published price forecast, interval by interval (a forecast entry holding several timeslot rates is split by its timeslots); until the forecast is out the status shows an error and nothing is published for that period, rather than a flat price. Without a forecast, only an explicit fixed rate or a time-of-use rate whose every price has timeslots is used: a time-of-use agreement repeats its timeslots each day in German local time (Europe/Berlin; on the autumn change a 02:00-03:00 slot covers both of those hours), and a fixed agreement uses its unit rate. Anything else is reported as an error. When the agreement ends inside the fetch window, the one agreement starting next on the same MaLo takes over; two agreements claiming the same start, or rates that overlap, are reported as an error rather than published. These are Octopus's gross unit rates - VAT, grid fees and levies included - so `spotprice_markup`, `spotprice_vat` and the charge zones are **not** applied. Octopus Energy Germany has no export tariff here; use `spotprice_export_mode` for your feed-in tariff. + - Unverified assumptions: not yet tried against a live account - the queries are checked against the live API schema only. A dynamic agreement is recognised by its price forecast or by "dynamic"/"dynamisch" in its product code, names or contract type, as the API has no market-linked flag; the real product codes have not been seen. Whether Octopus Energy Germany issues API keys to everyone on its tariffs has not been confirmed. +- `ews` - [EWS Elektrizitätswerke Schönau](https://www.ews-schoenau.de/oekostrom/oekostrom-dynamisch/) "Ökostrom Dynamisch", read from EWS's dynamic price API with an API key, which EWS issues on request at . EWS publishes quarter-hourly gross prices (VAT, grid fees and levies included, the same across Germany) for the next day between 15:00 and 17:00, and they are used as-is - `spotprice_markup`, `spotprice_vat` and the charge zones are **not** applied. EWS describes the API as being in a test phase. + - Unverified assumptions: EWS publishes no API documentation. The key is sent as an `X-API-Key` header and the reply is read as `today` and `tomorrow` lists of `startsAt` and `total`, with `total` in cents per kWh including VAT - all taken from another open-source project's integration, not from EWS. The API answers an invalid key exactly as it answers a missing one, so even the header name could not be confirmed. A reply whose typical price is below 1 is rejected rather than read as cents. #### When to enable (spotprice) - You are on a dynamic tariff that follows the day-ahead spot price (e.g. most German, Austrian, Dutch, Belgian or Nordic dynamic tariffs) -- Your electricity comes from Tibber and you want Tibber's prices used directly +- Your electricity comes from Tibber, Ostrom, Octopus Energy Germany or EWS Schönau and you want the supplier's own prices used directly - You have a fixed feed-in tariff, or a spot-linked export tariff, and want export priced alongside #### Important notes (spotprice) @@ -1818,12 +1829,20 @@ Price sources (`spotprice_provider`; when it is not set the default is `tibber` | Option | Type | Required | Default | Config Key | Description | | ------ | ---- | -------- | ------- | ---------- | ----------- | -| `provider` | String | No | `energycharts` | `spotprice_provider` | `entsoe`, `energycharts` or `tibber` | -| `zone` | String | Yes, except for tibber | - | `spotprice_zone` | Bidding zone, e.g. `DE-LU`, `AT`, `NL`, `BE`, `FR`, `DK1`, `SE3`, `NO1`, `IT-North`, `IE-SEM`, or a raw ENTSO-E EIC area code | +| `provider` | String | No | `energycharts` | `spotprice_provider` | `entsoe`, `energycharts`, `awattar`, `tibber`, `ostrom`, `octopus_de` or `ews` | +| `zone` | String | Yes, except for tibber, ostrom, octopus_de and ews | - | `spotprice_zone` | Bidding zone, e.g. `DE-LU`, `AT`, `NL`, `BE`, `FR`, `DK1`, `SE3`, `NO1`, `IT-North`, `IE-SEM`, or a raw ENTSO-E EIC area code | | `entsoe_token` | String | For entsoe | - | `spotprice_entsoe_token` | ENTSO-E Transparency Platform security token | | `tibber_token` | String | For tibber | - | `spotprice_tibber_token` | Tibber personal access token | | `tibber_home_id` | String | No | First home | `spotprice_tibber_home_id` | Tibber home id when your account has more than one home | +| `ostrom_client_id` | String | For ostrom | - | `spotprice_ostrom_client_id` | Ostrom API client ID (production client from the Ostrom developer portal) | +| `ostrom_client_secret` | String | For ostrom | - | `spotprice_ostrom_client_secret` | Ostrom API client secret | +| `ostrom_contract_id` | String | No | The only active contract | `spotprice_ostrom_contract_id` | Ostrom contract number, needed only when the account has more than one active contract; it must itself be active | +| `octopus_de_api_key` | String | For octopus_de | - | `spotprice_octopus_de_api_key` | Octopus Energy Germany API key | +| `octopus_de_account` | String | No | The key's only account | `spotprice_octopus_de_account` | Octopus Energy Germany account number, needed only when the key can see more than one account | +| `octopus_de_malo` | String | No | The only MaLo with an active agreement | `spotprice_octopus_de_malo` | Octopus Energy Germany market location (MaLo) number to price, needed only when the account has active agreements on more than one | +| `ews_api_key` | String | For ews | - | `spotprice_ews_api_key` | EWS Schönau dynamic price API key | | `markup` | Float | No | 0 | `spotprice_markup` | Supplier markup plus flat grid fees and levies, minor units per kWh, excluding VAT | +| `markup_percent` | Float | No | 0 (3 for awattar) | `spotprice_markup_percent` | Percentage of the absolute spot price added before VAT (e.g. tado Energy / aWATTar HOURLY's 3%). Set `0` to turn the awattar default off | | `vat` | Float | No | 0 | `spotprice_vat` | VAT as a fraction, e.g. `0.19`. A value of 1 or more is read as a percentage with a warning, so `1` means 1% | | `charge_zones` | List | No | - | `spotprice_charge_zones` | Time-of-day charges added before VAT: entries of `from`, `to` (HH:MM local), `charge` and optional `days` - day names (e.g. `[mon, tue, wed, thu, fri]`) or numbers 1-7 with Monday = 1, as for `day_of_week` in `rates_import` (`day_of_week` is also accepted here, with a warning). The first matching entry wins; a `to` at or before `from` wraps past midnight, so `to: "00:00"` runs up to midnight. Any other key (such as `start`, `end` or `rate`) is reported in the log | | `exchange_rate` | Float | No | 1.0 | `spotprice_exchange_rate` | Major currency units per euro, for tariffs not priced in EUR | @@ -1839,6 +1858,10 @@ Keep tokens in `secrets.yaml`: # secrets.yaml entsoe_token: YOUR_ENTSOE_TOKEN tibber_token: YOUR_TIBBER_TOKEN +ostrom_client_id: YOUR_OSTROM_CLIENT_ID +ostrom_client_secret: YOUR_OSTROM_CLIENT_SECRET +octopus_de_api_key: YOUR_OCTOPUS_DE_API_KEY +ews_api_key: YOUR_EWS_API_KEY ``` #### apps.yaml configuration example (spotprice) @@ -1880,6 +1903,27 @@ Tibber, with spot-linked export: spotprice_export_markup: -1.5 ``` +tado Energy (aWATTar) HOURLY in Germany, with 1.5 ct/kWh plus an example 18.9 ct/kWh of grid fees and levies: + +```yaml + spotprice_provider: awattar + spotprice_zone: DE-LU + spotprice_markup: 20.4 + spotprice_vat: 0.19 +``` + +Ostrom, with a fixed feed-in tariff that pays nothing at negative spot prices (the zone is needed only for that rule): + +```yaml + spotprice_provider: ostrom + spotprice_ostrom_client_id: !secret ostrom_client_id + spotprice_ostrom_client_secret: !secret ostrom_client_secret + spotprice_zone: DE-LU + spotprice_export_mode: fixed + spotprice_export_rate: 7.94 + spotprice_export_zero_on_negative: true +``` + A commented template is available in [templates/spotprice.yaml](https://raw.githubusercontent.com/springfall2008/batpred/main/templates/spotprice.yaml). #### Published entities (spotprice) diff --git a/templates/spotprice.yaml b/templates/spotprice.yaml index 10b1edd38..af3ed1b8a 100644 --- a/templates/spotprice.yaml +++ b/templates/spotprice.yaml @@ -4,7 +4,8 @@ # spot-linked tariff, without needing a supplier-specific integration. Merge the keys below into your # existing pred_bat section in apps.yaml - this file is not a complete configuration on its own. # -# import rate = (spot EUR/MWh / 10 + spotprice_markup + charge zone) x (1 + spotprice_vat) +# import rate = (spot + |spot| x spotprice_markup_percent / 100 + spotprice_markup + charge zone) x (1 + spotprice_vat) +# where spot = spot EUR/MWh / 10; spotprice_markup_percent is 0 unless set (3 for awattar) # # Markup, charge zones and export rates are in your minor currency unit per kWh (e.g. euro cents) # and exclude VAT. See docs/components.md for every option. @@ -15,15 +16,28 @@ pred_bat: - 'c' # Price source (default energycharts): entsoe (needs a free ENTSO-E token, falls back to Energy-Charts), - # energycharts (no key needed) or tibber (Tibber's own end-user price, needs a Tibber token) + # energycharts (no key needed), awattar (tado Energy / aWATTar HOURLY prices for DE-LU and AT, no key needed, + # adds 3% of the spot price by default - add the 1.5 ct/kWh and your grid fees to spotprice_markup), or a supplier's own end-user price, used as-is with no markup or VAT added: + # tibber (needs a Tibber token), ostrom (needs an Ostrom API client ID and secret) or + # octopus_de (Octopus Energy Germany, needs an API key) or ews (EWS Schönau Ökostrom Dynamisch, needs an API key + # from api@ews-schoenau.de) spotprice_provider: energycharts # Bidding zone, e.g. DE-LU, AT, NL, BE, FR, DK1, SE3, NO1, IE-SEM (ENTSO-E only) spotprice_zone: DE-LU # spotprice_entsoe_token: !secret entsoe_token # spotprice_tibber_token: !secret tibber_token + # spotprice_ostrom_client_id: !secret ostrom_client_id + # spotprice_ostrom_client_secret: !secret ostrom_client_secret + # spotprice_ostrom_contract_id: 100523456 # only if the account has more than one active contract + # spotprice_octopus_de_api_key: !secret octopus_de_api_key + # spotprice_octopus_de_account: A-1234ABCD # only if the API key can see more than one account + # spotprice_octopus_de_malo: "50000000001" # only if the account has active agreements on several market locations + # spotprice_ews_api_key: !secret ews_api_key # Supplier markup plus any flat grid fees and levies, per kWh, excluding VAT spotprice_markup: 18.5 + # Optional percentage of the absolute spot price, added before VAT (defaults to 3 for awattar, else 0) + # spotprice_markup_percent: 3 # VAT as a fraction (0.19 = 19%) spotprice_vat: 0.19