Native decoder test suite with real meter payload corpus

.github/workflows/build.yml

@@ -11,7 +11,7 @@ on:
       - platformio.ini
       - .github/workflows/**
     branches:
-      - '*'
+      - main
     tags:
       - '*'
       - '!v*.*.*'
@@ -49,6 +49,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -U platformio css_html_js_minify
+    - name: Install mbedTLS (native decoder tests)
+      run: sudo apt-get update && sudo apt-get install -y libmbedtls-dev
     - name: Set up node
       uses: actions/setup-node@v4
       with:
@@ -65,5 +67,16 @@ jobs:
       run: pio pkg install
     - name: PlatformIO run
       # Device firmware envs only. The 'native' unit-test env is built via
-      # `pio test -e native`, not `pio run` (added in the native-test follow-up).
+      # `pio test -e native`, not `pio run`.
       run: pio run -e esp8266 -e esp32 -e esp32s2 -e esp32solo -e esp32c3 -e esp32s3
+    - name: Decoder unit tests (native)
+      # Encrypted-fixture decode tests read keys from these secrets; without
+      # them those tests self-skip (unencrypted + golden tests still run).
+      env:
+        AMS_TEST_KEY_GH501_EK: ${{ secrets.AMS_TEST_KEY_GH501_EK }}
+        AMS_TEST_KEY_GH501_AK: ${{ secrets.AMS_TEST_KEY_GH501_AK }}
+        AMS_TEST_KEY_GH787_EK: ${{ secrets.AMS_TEST_KEY_GH787_EK }}
+        AMS_TEST_KEY_GH905_EK: ${{ secrets.AMS_TEST_KEY_GH905_EK }}
+        AMS_TEST_KEY_EM20200710_EK: ${{ secrets.AMS_TEST_KEY_EM20200710_EK }}
+        AMS_TEST_KEY_EM20200710_AK: ${{ secrets.AMS_TEST_KEY_EM20200710_AK }}
+      run: pio test -e native

.github/workflows/pull-request.yml

@@ -47,6 +47,34 @@ jobs:
       env: esp8266
       is_esp32: false
 
+  test:
+    name: Decoder unit tests (native)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code from repo
+        uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -U platformio
+      - name: Install mbedTLS (native decrypt tests)
+        run: sudo apt-get update && sudo apt-get install -y libmbedtls-dev
+      - name: Run decoder tests
+        # Encrypted-fixture tests read keys from these secrets; absent (e.g. on
+        # fork PRs) they self-skip, while unencrypted + golden tests still run.
+        env:
+          AMS_TEST_KEY_GH501_EK: ${{ secrets.AMS_TEST_KEY_GH501_EK }}
+          AMS_TEST_KEY_GH501_AK: ${{ secrets.AMS_TEST_KEY_GH501_AK }}
+          AMS_TEST_KEY_GH787_EK: ${{ secrets.AMS_TEST_KEY_GH787_EK }}
+          AMS_TEST_KEY_GH905_EK: ${{ secrets.AMS_TEST_KEY_GH905_EK }}
+          AMS_TEST_KEY_EM20200710_EK: ${{ secrets.AMS_TEST_KEY_EM20200710_EK }}
+          AMS_TEST_KEY_EM20200710_AK: ${{ secrets.AMS_TEST_KEY_EM20200710_AK }}
+        run: pio test -e native
+
   comment:
     needs:
       - build-esp32s2

.gitignore

@@ -26,3 +26,4 @@ node_modules
 /scripts/*dev
 localazy-keys.json
 localazy/language
+/test/payloads/keys/keys.local.json

platformio.ini

@@ -163,6 +163,8 @@ build_flags =
     -std=c++17
 test_framework = unity
 test_filter = test_decoder
+test_build_src = yes
+extra_scripts = pre:scripts/native_crypto.py
 build_src_filter =
     -<*>
     +<decoder/src/HdlcParser.cpp>
@@ -174,8 +176,10 @@ build_src_filter =
     +<decoder/src/Cosem.cpp>
     +<decoder/src/crc.cpp>
     +<decoder/src/ntohll.cpp>
+    +<decoder/src/GcmParser.cpp>
     +<decoder/src/IEC6205675.cpp>
     +<decoder/src/IEC6205621.cpp>
     +<hexutils.cpp>
     +<AmsData.cpp>
-    +<AmsConfiguration.cpp>
+    +<LNG.cpp>
+    +<LNG2.cpp>

scripts/build_payload_manifest.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Reconstruct a consolidated machine-readable fixtures manifest from the
+committed per-manufacturer README tables under test/payloads/."""
+import glob, re, json, os
+
+BASE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "test", "payloads")
+EDGE = re.compile(r"corrupt|truncat|overflow|garbl|wrong parity|wrong; correct|"
+                  r"malformed|partial|misalign|mis-sampl|not (?:usable|cleanly)|"
+                  r"negative|edge-case|implausible|clipped|cut short|overwritten",
+                  re.I)
+
+rows = []
+for f in sorted(glob.glob(f"{BASE}/*/README.md")):
+    man = f.split("/")[-2]
+    for line in open(f):
+        if not line.startswith("| `"):
+            continue
+        cells = [c.strip() for c in line.strip().strip("|").split("|")]
+        if len(cells) < 7:
+            continue
+        filecell, src, model, country, proto, enc, notes = cells[:7]
+        fn = filecell.replace("`", "").replace("🔑", "").strip()
+        if not re.search(r"\.(hex|txt)$", fn):
+            continue
+        encrypted = "🔒" in enc
+        edge = bool(EDGE.search(notes))
+        rows.append({
+            "file": f"{man}/{fn}",
+            "manufacturer": man,
+            "source": re.sub(r"\]\(.*?\)", "", src).strip("[]") or src,
+            "model": model, "country": country, "protocol": proto,
+            "format": "txt" if fn.endswith(".txt") else "hex",
+            "encrypted": encrypted,
+            "expect": "edge" if edge else "ok",
+        })
+
+# attach known decryption-key secret names from keymap.json
+keymap = {m["payload"]: m for m in json.load(open(f"{BASE}/keys/keymap.json"))}
+for r in rows:
+    if r["file"] in keymap:
+        r["ek_secret"] = keymap[r["file"]]["ek_secret"]
+        r["ak_secret"] = keymap[r["file"]]["ak_secret"]
+
+json.dump(rows, open(f"{BASE}/manifest.json", "w"), indent=1)
+
+# summary
+from collections import Counter
+print("total fixtures:", len(rows))
+def c(pred): return sum(1 for r in rows if pred(r))
+print("  unencrypted ok   :", c(lambda r: not r["encrypted"] and r["expect"]=="ok"))
+print("  unencrypted edge :", c(lambda r: not r["encrypted"] and r["expect"]=="edge"))
+print("  encrypted total  :", c(lambda r: r["encrypted"]))
+print("  encrypted w/ key :", c(lambda r: r["encrypted"] and r.get("ek_secret")))
+print("  .txt (DSMR)      :", c(lambda r: r["format"]=="txt"))
+print("\nunencrypted-ok by manufacturer:")
+for k,v in Counter(r["manufacturer"] for r in rows if not r["encrypted"] and r["expect"]=="ok").most_common():
+    print(f"   {v:3} {k}")
+print("\nedge (unencrypted) files:")
+for r in rows:
+    if not r["encrypted"] and r["expect"]=="edge":
+        print("   ", r["file"])

scripts/gen_fixtures_header.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import json, os
+BASE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "test", "payloads")
+rows = json.load(open(f"{BASE}/manifest.json"))
+
+def esc(s): return (s or "").replace('"', '\\"')
+
+def emit(rows):
+    out = []
+    for r in rows:
+        out.append('  {{ "test/payloads/{f}", "{man}", "{src}" }},'.format(
+            f=r["file"], man=esc(r["manufacturer"]), src=esc(r["source"])))
+    return "\n".join(out)
+
+unenc_ok   = [r for r in rows if not r["encrypted"] and r["expect"] == "ok"]
+unenc_edge = [r for r in rows if not r["encrypted"] and r["expect"] == "edge"]
+enc_keyed  = [r for r in rows if r["encrypted"] and r.get("ek_secret")]
+enc_nokey  = [r for r in rows if r["encrypted"] and not r.get("ek_secret")]
+
+h = []
+h.append("/**")
+h.append(" * @copyright Utilitech AS 2023-2026")
+h.append(" * License: Fair Source")
+h.append(" *")
+h.append(" * AUTO-GENERATED from test/payloads/manifest.json by")
+h.append(" * scripts/gen_fixtures_header.py — do not edit by hand.")
+h.append(" */")
+h.append("#ifndef _FIXTURES_GENERATED_H")
+h.append("#define _FIXTURES_GENERATED_H")
+h.append("")
+h.append("struct Fixture { const char* path; const char* manufacturer; const char* source; };")
+h.append("struct KeyedFixture { const char* path; const char* ek_secret; const char* ak_secret; };")
+h.append("")
+h.append(f"// {len(unenc_ok)} unencrypted frames expected to decode to a valid list")
+h.append("static const Fixture UNENC_OK[] = {")
+h.append(emit(unenc_ok))
+h.append("};")
+h.append("")
+h.append(f"// {len(unenc_edge)} unencrypted but corrupt/truncated/segment — must NOT crash")
+h.append("static const Fixture UNENC_EDGE[] = {")
+h.append(emit(unenc_edge))
+h.append("};")
+h.append("")
+h.append(f"// {len(enc_keyed)} encrypted frames for which we hold a verified key")
+h.append("static const KeyedFixture ENC_KEYED[] = {")
+for r in enc_keyed:
+    h.append('  {{ "test/payloads/{f}", "{ek}", {ak} }},'.format(
+        f=r["file"], ek=r["ek_secret"],
+        ak=('"%s"' % r["ak_secret"]) if r.get("ak_secret") else "nullptr"))
+h.append("};")
+h.append("")
+h.append(f"// {len(enc_nokey)} encrypted frames with no key — framing/GCM-header coverage only")
+h.append("static const Fixture ENC_NOKEY[] = {")
+h.append(emit(enc_nokey))
+h.append("};")
+h.append("")
+h.append("#endif")
+OUT = os.path.join(os.path.dirname(BASE), "test_decoder", "fixtures_generated.h")
+open(OUT, "w").write("\n".join(h) + "\n")
+print(f"wrote fixtures_generated.h: {len(unenc_ok)} ok, {len(unenc_edge)} edge, {len(enc_keyed)} keyed, {len(enc_nokey)} nokey")

scripts/native_crypto.py

@@ -0,0 +1,24 @@
+"""PlatformIO pre-build hook for the native test env.
+
+Probes for mbedTLS development headers. If found, defines HAVE_MBEDTLS and
+links libmbedcrypto so GcmParser can decrypt on host; otherwise the build
+falls back to the no-crypto stub and encrypted tests self-ignore.
+
+Install headers (Debian/Ubuntu): sudo apt-get install -y libmbedtls-dev
+"""
+import os
+
+Import("env")  # noqa: F821  (injected by PlatformIO/SCons)
+
+SEARCH = ["/usr/include", "/usr/local/include", "/opt/homebrew/include", "/usr/local/opt/mbedtls/include"]
+hdr = next((p for p in SEARCH if os.path.exists(os.path.join(p, "mbedtls", "gcm.h"))), None)
+
+if hdr:
+    print("native_crypto: mbedTLS found at %s -> enabling HAVE_MBEDTLS" % hdr)
+    env.Append(CPPDEFINES=["HAVE_MBEDTLS"], CPPPATH=[hdr], LIBS=["mbedcrypto"])
+    libdir = os.path.join(os.path.dirname(hdr), "lib")
+    if os.path.isdir(libdir):
+        env.Append(LIBPATH=[libdir])
+else:
+    print("native_crypto: mbedTLS headers not found; encrypted-decode tests will be skipped.")
+    print("               install with: sudo apt-get install -y libmbedtls-dev")

src/decoder/src/DsmrParser.cpp

@@ -14,17 +14,17 @@ int8_t DSMRParser::parse(uint8_t *buf, DataParserContext &ctx, bool verified, Pr
     uint16_t lenBefore = ctx.length;
     uint16_t crcPos = 0;
     bool reachedEnd = verified;
-    uint8_t lastByte = 0x00;
     for(uint16_t pos = 0; pos < ctx.length; pos++) {
         uint8_t b = *(buf+pos);
         if(pos == 0 && b != '/') return DATA_PARSE_BOUNDARY_FLAG_MISSING;
         if(pos > 0 && b == '!') crcPos = pos+1;
-        if(crcPos > 0 && b == 0x0A && lastByte == 0x0D) {
+        // End of telegram is the first LF after the CRC line. Accept both
+        // CRLF (\r\n, the P1 standard) and bare LF (\n) line endings.
+        if(crcPos > 0 && b == 0x0A) {
             reachedEnd = true;
             ctx.length = pos;
             break;
         }
-        lastByte = b;
     }
     if(!reachedEnd) return DATA_PARSE_INCOMPLETE;
     buf[ctx.length+1] = '\0';

src/decoder/src/GbtParser.cpp

@@ -7,6 +7,7 @@
 #include "GbtParser.h"
 #include "byteorder.h"
 #include <string.h>
+#include <stdlib.h>   // malloc/free (was relying on a transitive include)
 
 int8_t GBTParser::parse(uint8_t *d, DataParserContext &ctx) {
     GBTHeader* h = (GBTHeader*) (d);

src/decoder/src/GcmParser.cpp

@@ -10,6 +10,13 @@
 #include "bearssl/bearssl.h"
 #elif defined(ESP32)
 #include "mbedtls/gcm.h"
+#elif defined(NATIVE_TEST) && defined(HAVE_MBEDTLS)
+#include "mbedtls/gcm.h"   // host (native tests) — system mbedTLS (2.x or 3.x)
+#if defined(__has_include) && __has_include("mbedtls/build_info.h")
+#include "mbedtls/build_info.h"   // 3.x defines MBEDTLS_VERSION_MAJOR here
+#elif defined(__has_include) && __has_include("mbedtls/version.h")
+#include "mbedtls/version.h"      // 2.x defines it here
+#endif
 #endif
 #include <string.h>
 
@@ -92,6 +99,9 @@ int8_t GCMParser::parse(uint8_t *d, DataParserContext &ctx, bool hastag) {
     uint8_t authentication_tag[12];
     uint8_t authkeylen = 0, aadlen = 0;
     if((sec & 0x10) == 0x10) {
+        // Need at least the 12-byte auth tag plus the 5 security/frame-counter
+        // bytes; otherwise the tag memcpy and ciphertext length underflow.
+        if(len < 17) return GCM_DECRYPT_FAILED;
         authkeylen = 12;
         aadlen = 17;
         footersize += authkeylen;
@@ -100,6 +110,10 @@ int8_t GCMParser::parse(uint8_t *d, DataParserContext &ctx, bool hastag) {
         for(uint8_t i = 0; i < 16; i++) authenticate |= authentication_key[i] > 0;
     }
 
+    // Guard the ciphertext length (len - authkeylen - 5) against underflow so a
+    // short/garbage length can't blow the stack via the cipher_text buffer.
+    if(len < (uint32_t)authkeylen + 5) return GCM_DECRYPT_FAILED;
+
     #if defined(ESP8266)
         br_gcm_context gcmCtx;
         br_aes_ct_ctr_keys bc;
@@ -148,6 +162,51 @@ int8_t GCMParser::parse(uint8_t *d, DataParserContext &ctx, bool hastag) {
             }
         }
         mbedtls_gcm_free(&m_ctx);
+    #elif defined(NATIVE_TEST) && defined(HAVE_MBEDTLS)
+        // Native host tests with system mbedTLS (3.x API).
+        uint8_t cipher_text[len - authkeylen - 5];
+        memcpy(cipher_text, ptr, len - authkeylen - 5);
+
+        mbedtls_gcm_context m_ctx;
+        mbedtls_gcm_init(&m_ctx);
+        if (mbedtls_gcm_setkey(&m_ctx, MBEDTLS_CIPHER_ID_AES, encryption_key, 128) != 0) {
+            mbedtls_gcm_free(&m_ctx);
+            return GCM_ENCRYPTION_KEY_FAILED;
+        }
+        if (authenticate) {
+            int rc = mbedtls_gcm_auth_decrypt(&m_ctx, sizeof(cipher_text),
+                initialization_vector, sizeof(initialization_vector),
+                additional_authenticated_data, aadlen,
+                authentication_tag, authkeylen,
+                cipher_text, (unsigned char*)(ptr));
+            if (authkeylen > 0 && rc == MBEDTLS_ERR_GCM_AUTH_FAILED) {
+                mbedtls_gcm_free(&m_ctx);
+                return GCM_AUTH_FAILED;
+            } else if (rc != 0) {
+                mbedtls_gcm_free(&m_ctx);
+                return GCM_DECRYPT_FAILED;
+            }
+        } else {
+        #if defined(MBEDTLS_VERSION_MAJOR) && MBEDTLS_VERSION_MAJOR >= 3
+            size_t olen = 0;
+            if (mbedtls_gcm_starts(&m_ctx, MBEDTLS_GCM_DECRYPT,
+                    initialization_vector, sizeof(initialization_vector)) != 0 ||
+                mbedtls_gcm_update(&m_ctx, cipher_text, sizeof(cipher_text),
+                    (unsigned char*)(ptr), sizeof(cipher_text), &olen) != 0) {
+                mbedtls_gcm_free(&m_ctx);
+                return GCM_DECRYPT_FAILED;
+            }
+        #else   // mbedTLS 2.x (e.g. Ubuntu libmbedtls-dev) — older API
+            if (mbedtls_gcm_starts(&m_ctx, MBEDTLS_GCM_DECRYPT,
+                    initialization_vector, sizeof(initialization_vector), NULL, 0) != 0 ||
+                mbedtls_gcm_update(&m_ctx, sizeof(cipher_text),
+                    cipher_text, (unsigned char*)(ptr)) != 0) {
+                mbedtls_gcm_free(&m_ctx);
+                return GCM_DECRYPT_FAILED;
+            }
+        #endif
+        }
+        mbedtls_gcm_free(&m_ctx);
     #else
         // Native / unsupported platform: decryption not available
         return GCM_DECRYPT_FAILED;

test/payloads/.gitattributes

@@ -0,0 +1,2 @@
+# Captured meter payloads are byte-exact wire data — never normalize line endings.
+* -text