From 047b53e232ad892825f233f3b01449a9c34ebbb3 Mon Sep 17 00:00:00 2001 From: Jeffrey Sperling Date: Sat, 4 Jul 2026 23:39:19 -0700 Subject: [PATCH] perf: raise ONNX default batch size to 64 Measured on the default int8 bge-m3 model (CPU, Apple M-series, 128 texts): 13.2s at batch 32, 11.7s at 64, 10.6s at 128. 64 gives ~11% indexing throughput over the old default; 128 is left to explicit configuration because worst-case padded batches of 8192-token inputs materialize multi-GB activation tensors. --- src/memsearch/embeddings/onnx.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/memsearch/embeddings/onnx.py b/src/memsearch/embeddings/onnx.py index de99e31b..98aecea3 100644 --- a/src/memsearch/embeddings/onnx.py +++ b/src/memsearch/embeddings/onnx.py @@ -19,7 +19,11 @@ class OnnxEmbedding: - Models with ``last_hidden_state`` output — CLS pooling + L2 normalize applied """ - _DEFAULT_BATCH_SIZE = 32 + # 64 measured ~11% faster than 32 on the default int8 bge-m3 model + # (CPU, Apple M-series; 128 texts: 13.2s @ 32 -> 11.7s @ 64 -> 10.6s @ 128). + # 128 is not the default because worst-case padded batches (8192-token + # inputs) materialize multi-GB activation tensors; 64 keeps that bounded. + _DEFAULT_BATCH_SIZE = 64 def __init__( self,