perf(prediction): build the kernel's ctypes arrays via array.array (1.9x faster planning)

.cspell/custom-dictionary-workspace.txt

@@ -234,6 +234,7 @@ invname
 isinstance
 isoformat
 isort
+itemsize
 itemtype
 ivtime
 jedlix
@@ -507,6 +508,8 @@ treforsiphone
 treforsouthwell
 tunables
 twinx
+typecode
+typecodes
 tzfile
 tzpath
 unconfigured

apps/predbat/prediction_kernel.py

@@ -20,6 +20,7 @@
 rejected at load time rather than producing divergent results.
 """
 
+import array
 import ctypes
 import os
 import platform
@@ -260,14 +261,48 @@ def load_kernel(log=None):
     return KERNEL_LIB
 
 
+def select_array_typecode(candidates, ctype):
+    """Pick an array.array typecode whose itemsize matches ctype exactly, or None if none does.
+
+    array.array's integer typecodes are C types, so their widths are platform-defined - 'i' is a C
+    int, which is 32 bit everywhere predbat runs but is not guaranteed to be. Getting this wrong is
+    not a loud failure: from_buffer only checks the buffer is big enough, so a wider backing type is
+    accepted and the kernel silently reads interleaved garbage. Matching the size up front, and
+    falling back to the slower construction when nothing matches, keeps that impossible.
+    """
+    size = ctypes.sizeof(ctype)
+    for typecode in candidates:
+        if array.array(typecode).itemsize == size:
+            return typecode
+    return None
+
+
+DOUBLE_TYPECODE = select_array_typecode(("d", "f"), ctypes.c_double)
+INT32_TYPECODE = select_array_typecode(("i", "l", "h"), ctypes.c_int32)
+
+
 def double_array(values):
-    """Create a ctypes double array from a Python list"""
-    return (ctypes.c_double * len(values))(*values)
+    """Create a ctypes double array from a Python list.
+
+    Built via array.array rather than (ctypes.c_double * n)(*values): the latter unpacks the list as
+    positional arguments and is several times slower, which matters because these are rebuilt on
+    every simulation. from_buffer returns a view over the array.array, and ctypes keeps the backing
+    object alive through the view's _objects, so the buffer cannot be collected while the kernel is
+    using it. Each pool worker is a separate process (multiprocessing with fork), so no buffer is
+    ever shared between workers.
+    """
+    if DOUBLE_TYPECODE is None:
+        return (ctypes.c_double * len(values))(*values)
+    backing = array.array(DOUBLE_TYPECODE, values)
+    return (ctypes.c_double * len(backing)).from_buffer(backing)
 
 
 def int32_array(values):
-    """Create a ctypes int32 array from a Python list"""
-    return (ctypes.c_int32 * len(values))(*values)
+    """Create a ctypes int32 array from a Python list - see double_array for why array.array is used"""
+    if INT32_TYPECODE is None:
+        return (ctypes.c_int32 * len(values))(*values)
+    backing = array.array(INT32_TYPECODE, values)
+    return (ctypes.c_int32 * len(backing)).from_buffer(backing)
 
 
 def kernel_context_free(handle):

apps/predbat/tests/test_kernel_parity.py

@@ -20,7 +20,10 @@
 PREDBAT_KERNEL_REQUIRED=1 is set (CI) in which case it fails.
 """
 
+import array
 import copy
+import ctypes
+import gc
 import os
 import random
 import subprocess
@@ -380,6 +383,51 @@ def dual_run(name, my_predbat, pv_step, pv10_step, load_step, load10_step, charg
     return failed
 
 
+def run_marshalling_tests():
+    """Check the ctypes buffer helpers, returns True on failure.
+
+    double_array/int32_array build their buffers with from_buffer, which returns a view over an
+    array.array rather than a copy. If ctypes did not keep the backing object alive the kernel would
+    read freed memory - silently, and only sometimes - so that guarantee is asserted here rather than
+    assumed, along with the values surviving the round trip.
+    """
+    print("**** Running kernel marshalling tests ****")
+    failed = False
+
+    # The typecode chosen for the backing array must match the ctypes element exactly. from_buffer
+    # only checks the buffer is large enough, so a wider backing type is accepted and then read as
+    # interleaved garbage - a silent corruption rather than an exception.
+    for name, typecode, ctype in (("DOUBLE_TYPECODE", prediction_kernel.DOUBLE_TYPECODE, ctypes.c_double), ("INT32_TYPECODE", prediction_kernel.INT32_TYPECODE, ctypes.c_int32)):
+        if typecode is not None and array.array(typecode).itemsize != ctypes.sizeof(ctype):
+            print("ERROR: {} is '{}' with itemsize {} but the ctypes element is {} bytes".format(name, typecode, array.array(typecode).itemsize, ctypes.sizeof(ctype)))
+            failed = True
+
+    for name, builder, values, typecode in (("double_array", prediction_kernel.double_array, [0.0, -1.5, 3.25, 1e6], prediction_kernel.DOUBLE_TYPECODE), ("int32_array", prediction_kernel.int32_array, [0, -7, 42, 100000], prediction_kernel.INT32_TYPECODE)):
+        # Build from a temporary so the source list/array is unreferenced by the time it is read
+        buffer = builder(list(values))
+        gc.collect()
+        got = [buffer[i] for i in range(len(values))]
+        if got != values:
+            print("ERROR: {} round trip expected {} but got {}".format(name, values, got))
+            failed = True
+        # Only the from_buffer path holds a view that needs its backing kept alive; the fallback
+        # copies the values, so it has nothing to retain and is safe without _objects. Truthiness
+        # rather than "is not None": an empty _objects would mean nothing is retained, which is just
+        # as unsafe as the attribute being absent.
+        if typecode is not None and not getattr(buffer, "_objects", None):
+            print("ERROR: {} did not retain its backing buffer - the kernel could read freed memory".format(name))
+            failed = True
+
+        empty = builder([])
+        if len(empty) != 0:
+            print("ERROR: {}([]) should be empty, got length {}".format(name, len(empty)))
+            failed = True
+
+    if not failed:
+        print("PASS")
+    return failed
+
+
 def run_edge_case_tests(my_predbat):
     """Deterministic scenarios pinning each kernel branch, returns True on failure"""
     failed = False
@@ -840,7 +888,8 @@ def run_kernel_parity_tests(my_predbat):
 
     state = snapshot_scenario_state(my_predbat)
     try:
-        failed = run_edge_case_tests(my_predbat)
+        failed = run_marshalling_tests()
+        failed |= run_edge_case_tests(my_predbat)
         if not failed:
             failed |= run_random_sweep_tests(my_predbat)
         if not failed:

coverage/cases/random_results.json

@@ -2,7 +2,7 @@
   "run_info": {
     "template_yaml": "cases/predbat_debug_agile1.yaml",
     "scenarios_file": "cases/random_scenarios.yaml",
-    "timestamp": "2026-08-13T10:34:13.978454+00:00"
+    "timestamp": "2026-08-13T11:57:01.949843+00:00"
   },
   "results": [
     {
@@ -17,7 +17,7 @@
       "soc_final": 6.8616,
       "battery_cycles": 21.0198,
       "carbon_g": 12959.33,
-      "runtime_s": 3.993,
+      "runtime_s": 2.1,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -34,7 +34,7 @@
       "soc_final": 4.8,
       "battery_cycles": 10.5525,
       "carbon_g": 4613.47,
-      "runtime_s": 3.174,
+      "runtime_s": 2.772,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -51,7 +51,7 @@
       "soc_final": 2.2382,
       "battery_cycles": 5.8265,
       "carbon_g": 9017.53,
-      "runtime_s": 3.094,
+      "runtime_s": 1.561,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -68,7 +68,7 @@
       "soc_final": 2.0559,
       "battery_cycles": 9.7908,
       "carbon_g": 9673.45,
-      "runtime_s": 0.236,
+      "runtime_s": 0.211,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -85,7 +85,7 @@
       "soc_final": 4.8,
       "battery_cycles": 7.3257,
       "carbon_g": 15944.91,
-      "runtime_s": 0.423,
+      "runtime_s": 0.364,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -102,7 +102,7 @@
       "soc_final": 2.3066,
       "battery_cycles": 34.749,
       "carbon_g": 884.71,
-      "runtime_s": 0.433,
+      "runtime_s": 0.39,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -119,7 +119,7 @@
       "soc_final": 0.38,
       "battery_cycles": 38.4449,
       "carbon_g": 8065.01,
-      "runtime_s": 18.238,
+      "runtime_s": 8.31,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -136,7 +136,7 @@
       "soc_final": 0.38,
       "battery_cycles": 14.2039,
       "carbon_g": 18350.09,
-      "runtime_s": 4.406,
+      "runtime_s": 3.376,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -153,7 +153,7 @@
       "soc_final": 0.38,
       "battery_cycles": 10.6638,
       "carbon_g": 12867.91,
-      "runtime_s": 0.417,
+      "runtime_s": 0.361,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -170,7 +170,7 @@
       "soc_final": 9.152,
       "battery_cycles": 25.215,
       "carbon_g": 4663.11,
-      "runtime_s": 0.566,
+      "runtime_s": 0.512,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -187,7 +187,7 @@
       "soc_final": 1.0713,
       "battery_cycles": 25.7368,
       "carbon_g": 7751.35,
-      "runtime_s": 2.156,
+      "runtime_s": 1.404,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -204,7 +204,7 @@
       "soc_final": 0.38,
       "battery_cycles": 6.174,
       "carbon_g": 13156.97,
-      "runtime_s": 1.181,
+      "runtime_s": 0.768,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -221,7 +221,7 @@
       "soc_final": 0.38,
       "battery_cycles": 28.7593,
       "carbon_g": 15144.61,
-      "runtime_s": 3.545,
+      "runtime_s": 2.101,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -238,7 +238,7 @@
       "soc_final": 6.2665,
       "battery_cycles": 9.68,
       "carbon_g": 12838.7,
-      "runtime_s": 0.499,
+      "runtime_s": 0.433,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -255,7 +255,7 @@
       "soc_final": 1.5389,
       "battery_cycles": 5.6825,
       "carbon_g": 16611.75,
-      "runtime_s": 0.477,
+      "runtime_s": 0.415,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -272,7 +272,7 @@
       "soc_final": 4.3726,
       "battery_cycles": 19.543,
       "carbon_g": 16613.71,
-      "runtime_s": 4.61,
+      "runtime_s": 2.725,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -289,7 +289,7 @@
       "soc_final": 0.38,
       "battery_cycles": 4.9586,
       "carbon_g": 15848.43,
-      "runtime_s": 0.297,
+      "runtime_s": 0.255,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -306,7 +306,7 @@
       "soc_final": 0.38,
       "battery_cycles": 25.8549,
       "carbon_g": 153.34,
-      "runtime_s": 5.097,
+      "runtime_s": 1.887,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -323,7 +323,7 @@
       "soc_final": 0.5251,
       "battery_cycles": 9.8445,
       "carbon_g": 11504.26,
-      "runtime_s": 0.212,
+      "runtime_s": 0.19,
       "failed": false,
       "error": null,
       "end_record": 1440
@@ -340,7 +340,7 @@
       "soc_final": 19.6907,
       "battery_cycles": 60.7987,
       "carbon_g": 18744.27,
-      "runtime_s": 21.279,
+      "runtime_s": 9.089,
       "failed": false,
       "error": null,
       "end_record": 1440