From 7e8c74365241c156f052ee20040d64fb65390da9 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:54:12 +0000 Subject: [PATCH] Fix remaining_length accounting after Huffman decode in regression/composed predictors RegressionPredictor::load decremented remaining_length by the uncompressed index count (coeff_size * sizeof(int)) after Huffman decode(), but decode() advances the read pointer only by the compressed stream size. The uncompressed count overshoots the compressed stream, understating remaining_length for the subsequent quantizer encoder.load(), whose bound check then spuriously rejects valid data with "SZ3 Huffman: tree exceeds compressed buffer". RegressionPredictor is only used by ALGO_LORENZO_REG, so a column compressed with CODEC(SZ3('ALGO_LORENZO_REG', ...)) could be written on insert but fail every subsequent read with CORRUPTED_DATA (effective data loss). Small element counts are affected; large counts happen to leave enough slack to survive the understated bound. Account for exactly the bytes decode() consumed via pointer difference. ComposedPredictor::load had the mirror defect (it never decremented remaining_length for its selection stream); tightened the same way. Co-Authored-By: Claude Opus 4.8 --- include/SZ3/predictor/ComposedPredictor.hpp | 4 ++++ include/SZ3/predictor/RegressionPredictor.hpp | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/SZ3/predictor/ComposedPredictor.hpp b/include/SZ3/predictor/ComposedPredictor.hpp index 4e4b2b16..46281ac9 100644 --- a/include/SZ3/predictor/ComposedPredictor.hpp +++ b/include/SZ3/predictor/ComposedPredictor.hpp @@ -72,8 +72,12 @@ class ComposedPredictor : public concepts::PredictorInterface { if (selection_size > 0) { HuffmanEncoder selection_encoder; selection_encoder.load(c, remaining_length); + /// decode() advances `c` past the encoded stream but does not update `remaining_length`; + /// account for exactly the bytes it consumed so the bound stays tight for later reads. + const uchar *decode_start = c; this->selection = selection_encoder.decode(c, selection_size); selection_encoder.postprocess_decode(); + remaining_length -= static_cast(c - decode_start); } } diff --git a/include/SZ3/predictor/RegressionPredictor.hpp b/include/SZ3/predictor/RegressionPredictor.hpp index 51b3a089..f23b9aa4 100644 --- a/include/SZ3/predictor/RegressionPredictor.hpp +++ b/include/SZ3/predictor/RegressionPredictor.hpp @@ -114,9 +114,14 @@ class RegressionPredictor : public concepts::PredictorInterface { quantizer_liner.load(c, remaining_length); HuffmanEncoder encoder = HuffmanEncoder(); encoder.load(c, remaining_length); + /// decode() advances `c` past the encoded stream but does not update `remaining_length`; + /// account for exactly the bytes it consumed. The previous `coeff_size * sizeof(int)` used the + /// uncompressed index count, which overshoots the (Huffman-compressed) stream and understates + /// remaining_length, making a later encoder.load() bound check spuriously reject valid data. + const uchar *decode_start = c; regression_coeff_quant_inds = encoder.decode(c, coeff_size); encoder.postprocess_decode(); - remaining_length -= coeff_size * sizeof(int); + remaining_length -= static_cast(c - decode_start); std::fill(current_coeffs.begin(), current_coeffs.end(), 0); regression_coeff_index = 0; }