Fix 14 pre-existing defects found while working on fz

.gitignore

@@ -11,4 +11,4 @@ install
 .vscode
 sz3_install
 sz3_build
-test
+/test

include/SZ3/api/impl/SZAlgoBioMD.hpp

@@ -1,14 +1,15 @@
 #ifndef SZ3_SZ_BIOMD_HPP
 #define SZ3_SZ_BIOMD_HPP
 
+#include "SZ3/compressor/SZGenericCompressor.hpp"
 #include "SZ3/decomposition/SZBioMDDecomposition.hpp"
 #include "SZ3/decomposition/SZBioMDXtcDecomposition.hpp"
 #include "SZ3/def.hpp"
+#include "SZ3/encoder/HuffmanEncoder.hpp"
+#include "SZ3/encoder/HuffmanEncoderV2.hpp"
 #include "SZ3/encoder/XtcBasedEncoder.hpp"
 #include "SZ3/lossless/Lossless_bypass.hpp"
 #include "SZ3/lossless/Lossless_zstd.hpp"
-#include "SZ3/encoder/HuffmanEncoderV2.hpp"
-#include "SZ3/encoder/HuffmanEncoder.hpp"
 #include "SZ3/quantizer/LinearQuantizer.hpp"
 #include "SZ3/utils/Config.hpp"
 #include "SZ3/utils/Statistic.hpp"

include/SZ3/api/impl/SZAlgoInterp.hpp

@@ -1,6 +1,8 @@
 #ifndef SZ3_SZALGO_INTERP_HPP
 #define SZ3_SZALGO_INTERP_HPP
 
+#include <memory>
+
 #include "SZ3/api/impl/SZAlgoLorenzoReg.hpp"
 #include "SZ3/decomposition/BlockwiseDecomposition.hpp"
 #include "SZ3/decomposition/InterpolationDecomposition.hpp"

include/SZ3/compressor/SZGenericCompressor.hpp

@@ -2,6 +2,7 @@
 #define SZ3_COMPRESSOR_TYPE_ONE_HPP
 
 #include <cstring>
+#include <memory>
 
 #include "SZ3/compressor/Compressor.hpp"
 #include "SZ3/decomposition/Decomposition.hpp"

include/SZ3/compressor/specialized/SZExaaltCompressor.hpp

@@ -1,6 +1,11 @@
 #ifndef SZ3_EXAALT_COMPRESSSOR_HPP
 #define SZ3_EXAALT_COMPRESSSOR_HPP
 
+#include <iostream>
+#include <limits>
+#include <memory>
+
+#include "SZ3/compressor/Compressor.hpp"
 #include "SZ3/def.hpp"
 #include "SZ3/encoder/Encoder.hpp"
 #include "SZ3/lossless/Lossless.hpp"

include/SZ3/decomposition/BlockwiseDecomposition.hpp

@@ -2,15 +2,17 @@
 #define SZ3_BLOCKWISE_DECOMPOSITION_HPP
 
 #include <cstring>
+#include <limits>
+#include <memory>
 
 #include "Decomposition.hpp"
 #include "SZ3/def.hpp"
 #include "SZ3/predictor/LorenzoPredictor.hpp"
 #include "SZ3/predictor/Predictor.hpp"
 #include "SZ3/quantizer/LinearQuantizer.hpp"
+#include "SZ3/utils/BlockwiseIterator.hpp"
 #include "SZ3/utils/Config.hpp"
 #include "SZ3/utils/FileUtil.hpp"
-#include "SZ3/utils/BlockwiseIterator.hpp"
 #include "SZ3/utils/Timer.hpp"
 
 namespace SZ3 {

include/SZ3/decomposition/Decomposition.hpp

@@ -1,9 +1,11 @@
 #ifndef SZ3_DECOMPOSITION_INTERFACE
 #define SZ3_DECOMPOSITION_INTERFACE
 
+#include <utility>
 #include <vector>
 
 #include "SZ3/def.hpp"
+#include "SZ3/utils/Config.hpp"
 
 namespace SZ3::concepts {
 

include/SZ3/decomposition/InterpolationDecomposition.hpp

@@ -3,10 +3,12 @@
 
 #include <cmath>
 #include <cstring>
+#include <memory>
 
 #include "Decomposition.hpp"
 #include "SZ3/def.hpp"
 #include "SZ3/quantizer/Quantizer.hpp"
+#include "SZ3/utils/BlockwiseIterator.hpp"
 #include "SZ3/utils/Config.hpp"
 #include "SZ3/utils/FileUtil.hpp"
 #include "SZ3/utils/Interpolators.hpp"
@@ -146,6 +148,11 @@ class InterpolationDecomposition : public concepts::DecompositionInterface<T, in
         return quant_inds_vec;
     }
 
+    size_t size_est() override {
+        return sizeof(original_dimensions) + sizeof(blocksize) + sizeof(interp_id) + sizeof(direction_sequence_id) +
+               sizeof(anchor_stride) + sizeof(eb_alpha) + sizeof(eb_beta) + quantizer_size_est(quantizer) + 128;
+    }
+
     void save(uchar *&c) override {
         write(original_dimensions.data(), N, c);
         write(blocksize, c);

include/SZ3/decomposition/NoPredictionDecomposition.hpp

@@ -32,6 +32,8 @@ class NoPredictionDecomposition : public concepts::DecompositionInterface<T, int
         return quant_inds;
     }
 
+    size_t size_est() override { return quantizer_size_est(quantizer) + 64; }
+
     void save(uchar *&c) override { quantizer.save(c); }
 
     void load(const uchar *&c, size_t &remaining_length) override { quantizer.load(c, remaining_length); }

include/SZ3/decomposition/SZBioMDXtcDecomposition.hpp

@@ -6,6 +6,7 @@
 #ifndef SZ3_SZBIOMDXTCBASED_FRONTEND
 #define SZ3_SZBIOMDXTCBASED_FRONTEND
 
+#include <limits>
 #include <list>
 
 #include "Decomposition.hpp"

include/SZ3/decomposition/TimeSeriesDecomposition.hpp

@@ -1,6 +1,9 @@
 #ifndef SZ3_TIME_SERIES_DECOMPOSITION_HPP
 #define SZ3_TIME_SERIES_DECOMPOSITION_HPP
 
+#include <cassert>
+#include <limits>
+
 #include "Decomposition.hpp"
 #include "SZ3/def.hpp"
 #include "SZ3/predictor/LorenzoPredictor.hpp"
@@ -34,6 +37,9 @@ class TimeSeriesDecomposition : public concepts::DecompositionInterface<T, int,
     std::vector<int> compress(const Config& conf, T* data) override {
         std::vector<int> quant_inds(num_elements);
         size_t quant_count = 0;
+        // The timestep loop below predicts from the reconstruction of timestep 0.
+        const T* ts0_recon = data;
+        std::shared_ptr<block_data<T, N - 1>> data_with_padding;
         if (data_ts0 != nullptr) {
             for (size_t j = 0; j < conf.dims[1]; j++) {
                 quant_inds[quant_count++] = quantizer.quantize_and_overwrite(data[j], data_ts0[j]);
@@ -44,7 +50,7 @@ class TimeSeriesDecomposition : public concepts::DecompositionInterface<T, int,
                 spatial_dims[i] = conf.dims[i + 1];
             };
 
-            auto data_with_padding =
+            data_with_padding =
                 std::make_shared<block_data<T, N - 1>>(data, spatial_dims, predictor.get_padding(), true);
             auto block = data_with_padding->block_iter(conf.blockSize);
             do {
@@ -58,13 +64,16 @@ class TimeSeriesDecomposition : public concepts::DecompositionInterface<T, int,
                     quant_inds[quant_count++] = quantizer.quantize_and_overwrite(*c, pred);
                 });
             } while (block.next());
+
+            ts0_recon = data_with_padding->values();
         }
 
         for (size_t j = 0; j < conf.dims[1]; j++) {
+            T prev = ts0_recon[j];
             for (size_t i = 1; i < conf.dims[0]; i++) {
                 size_t idx = i * conf.dims[1] + j;
-                size_t idx_prev = (i - 1) * conf.dims[1] + j;
-                quant_inds[quant_count++] = quantizer.quantize_and_overwrite(data[idx], data[idx_prev]);
+                quant_inds[quant_count++] = quantizer.quantize_and_overwrite(data[idx], prev);
+                prev = data[idx];  // quantize_and_overwrite left the reconstruction here
             }
         }
         assert(quant_count == num_elements);

include/SZ3/encoder/ArithmeticEncoder.hpp

@@ -2,6 +2,7 @@
 #define SZ3_ArithmeticEncoder_HPP
 
 #include <cassert>
+#include <cmath>
 #include <iostream>
 
 #include "SZ3/encoder/Encoder.hpp"
@@ -527,7 +528,7 @@ class ArithmeticEncoder : public concepts::EncoderInterface<T> {
         size_t total_frequency = ariCoder.total_frequency;
         const uchar *sp = bytes + 5;
         unsigned int offset = 4;
-        size_t value = (bytesToInt64_bigEndian(bytes) >> 20);  // alignment with the MAX_CODE
+        size_t value = (static_cast<uint64_t>(bytesToInt64_bigEndian(bytes)) >> 20);  // alignment with the MAX_CODE
         size_t s_counter = sizeof(int);
 
         for (i = 0; i < targetLength; i++) {

include/SZ3/encoder/HuffmanEncoder.hpp

@@ -533,6 +533,10 @@ class HuffmanEncoder : public concepts::EncoderInterface<T> {
             }
         }
 
+        // The state table is sized by the bin range, not the distinct count.
+        if (static_cast<double>(max) - static_cast<double>(offset) > 2e9) {
+            throw std::invalid_argument("HuffmanEncoder: bin range too wide; use HuffmanEncoderV2");
+        }
         int stateNum = max - offset + 2;
         huffmanTree = createHuffmanTree(stateNum);
 

include/SZ3/encoder/RunlengthEncoder.hpp

@@ -12,7 +12,9 @@ namespace SZ3 {
 template <class T>
 class RunlengthEncoder : public concepts::EncoderInterface<T> {
    public:
-    void preprocess_encode(const std::vector<T> &bins, int stateNum) override {}
+    void preprocess_encode(const std::vector<T> &bins, int stateNum) override { num_bins = bins.size(); }
+
+    size_t size_est() override { return num_bins * (sizeof(T) + sizeof(int)); }
 
     size_t encode(const std::vector<T> &bins, uchar *&bytes) override {
         auto bytespos = bytes;
@@ -60,6 +62,8 @@ class RunlengthEncoder : public concepts::EncoderInterface<T> {
     void save(uchar *&c) override {}
 
     void load(const uchar *&c, size_t &remaining_length) override {}
+
+    size_t num_bins = 0;  ///< Set by preprocess_encode(), consumed by size_est()
 };
 }  // namespace SZ3
 #endif

include/SZ3/encoder/XtcBasedEncoder.hpp

@@ -9,10 +9,12 @@
 #define _SZ_XTC3_ENCODER_HPP
 
 #include <climits>
+#include <cmath>
 #include <vector>
 
 #include "SZ3/def.hpp"
 #include "SZ3/encoder/Encoder.hpp"
+#include "SZ3/utils/Config.hpp"
 
 // #define DEBUG_OUTPUT
 

include/SZ3/lossless/Lossless.hpp

@@ -5,6 +5,10 @@
 #ifndef SZ3_LOSSLESS_HPP
 #define SZ3_LOSSLESS_HPP
 
+#include <cstddef>
+
+#include "SZ3/def.hpp"
+
 namespace SZ3::concepts {
 
 /**

include/SZ3/lossless/Lossless_bypass.hpp

@@ -5,16 +5,21 @@
 #ifndef SZ3_LOSSLESS_BYPASS_HPP
 #define SZ3_LOSSLESS_BYPASS_HPP
 
+#include <cstdlib>
 #include <cstring>
+#include <stdexcept>
+
 #include "SZ3/def.hpp"
 #include "SZ3/lossless/Lossless.hpp"
 
 namespace SZ3 {
 class Lossless_bypass : public concepts::LosslessInterface {
-public:
+   public:
     size_t compress(const uchar *src, size_t srcLen, uchar *dst, size_t dstCap) override {
+        if (dstCap < srcLen) {
+            throw std::length_error(SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH);
+        }
         std::memcpy(dst, src, srcLen);
-        // dst = src;
         return srcLen;
     }
 

include/SZ3/predictor/LorenzoPredictor.hpp

@@ -1,6 +1,9 @@
 #ifndef SZ3_LORENZO_PREDICTOR_HPP
 #define SZ3_LORENZO_PREDICTOR_HPP
 
+#include <iostream>
+#include <limits>
+
 #include "SZ3/predictor/Predictor.hpp"
 
 namespace SZ3 {

include/SZ3/predictor/RegressionPredictor.hpp

@@ -114,9 +114,11 @@ class RegressionPredictor : public concepts::PredictorInterface<T, N> {
             quantizer_liner.load(c, remaining_length);
             HuffmanEncoder<int> encoder = HuffmanEncoder<int>();
             encoder.load(c, remaining_length);
+            const uchar *coeff_start = c;
             regression_coeff_quant_inds = encoder.decode(c, coeff_size);
             encoder.postprocess_decode();
-            remaining_length -= coeff_size * sizeof(int);
+            // decode() advances `c` by the encoded byte count, not by the decoded bin count.
+            remaining_length -= static_cast<size_t>(c - coeff_start);
             std::fill(current_coeffs.begin(), current_coeffs.end(), 0);
             regression_coeff_index = 0;
         }

include/SZ3/preprocessor/PreFilter.hpp

@@ -5,6 +5,10 @@
 #ifndef SZ3_PREFILTER_HPP
 #define SZ3_PREFILTER_HPP
 
+#include <array>
+#include <cstddef>
+#include <vector>
+
 #include "SZ3/preprocessor/PreProcessor.hpp"
 
 namespace SZ3 {

include/SZ3/preprocessor/Transpose.hpp

@@ -5,6 +5,10 @@
 #ifndef SZ3_TRANSPOSE_H
 #define SZ3_TRANSPOSE_H
 
+#include <array>
+#include <cstddef>
+#include <vector>
+
 #include "SZ3/preprocessor/PreProcessor.hpp"
 
 namespace SZ3 {

include/SZ3/quantizer/LinearQuantizer.hpp

@@ -83,7 +83,12 @@ class LinearQuantizer : public concepts::QuantizerInterface<T, int> {
         return pred + 2 * (quant_index - this->radius) * this->error_bound;
     }
 
-    ALWAYS_INLINE T recover_unpred() { return unpred[index++]; }
+    ALWAYS_INLINE T recover_unpred() {
+        if (index >= unpred.size()) {
+            throw std::runtime_error("LinearQuantizer: more unpredictable bins than stored values");
+        }
+        return unpred[index++];
+    }
 
     ALWAYS_INLINE int force_save_unpred(T ori) override {
         unpred.push_back(ori);

include/SZ3/quantizer/Quantizer.hpp

@@ -2,6 +2,9 @@
 #define SZ3_QUANTIZER_HPP
 
 #include <SZ3/def.hpp>
+#include <cstddef>
+#include <type_traits>
+#include <utility>
 
 namespace SZ3::concepts {
 
@@ -70,4 +73,22 @@ class   QuantizerInterface {
 };
 }  // namespace SZ3::concepts
 
+namespace SZ3 {
+/// Detects the optional (non-virtual) `size_est()` some quantizers expose.
+template <class Q, class = void>
+struct quantizer_has_size_est : std::false_type {};
+template <class Q>
+struct quantizer_has_size_est<Q, std::void_t<decltype(std::declval<Q &>().size_est())>> : std::true_type {};
+
+/// The quantizer's serialized-size estimate, or 0 when it does not expose one.
+template <class Q>
+size_t quantizer_size_est(Q &q) {
+    if constexpr (quantizer_has_size_est<Q>::value) {
+        return q.size_est();
+    } else {
+        return 0;
+    }
+}
+}  // namespace SZ3
+
 #endif