[DO NOT MERGE] Pending bug fixes, for review

.gitignore

@@ -1,14 +1,23 @@
 test.dat
 compressed.out
 perf.data*
-build
+# Anchored to the repo root, and glob so build-release and friends are covered too
+/build*
 debug
 cmake-build-debug
 cmake-build-release
 .idea
 install
 .DS_Store
 .vscode
+.cache
+.claude
+tmp
+docs/site/html
 sz3_install
 sz3_build
-test
+/test
+
+# Python bytecode
+__pycache__/
+*.pyc

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.18)
-project(SZ3 VERSION 3.3.2)
+project(SZ3 VERSION 3.3.3)
 
 #data version defines the version of the compressed data format
 #it is not always equal to the program version (e.g., SZ3 v3.1.0 and SZ3 v.3.1.1 may use the same data version of v.3.1.0)

README.md

@@ -79,6 +79,7 @@ Version New features
 * SZ 3.3.0 Add key QoZ v1 and v2 features to improve compression speed and data quality. The full QoZ is available from **a separate branch** (https://github.com/szcompressor/SZ3/tree/QoZ). 
 * SZ 3.3.1: SZ3 Windows support for both Visual Studio and MinGW toolchains. pySZ v1 released and available via `pip install pysz`. Bio algorithms added.
 * SZ 3.3.2: bugfix for compressed format.
+* SZ 3.3.3: decompression is bounds-checked against corrupted input. The compressed format is unchanged.
 
 ## 3rd party libraries/tools
 * [Zstandard](https://facebook.github.io/zstd/) v1.4.5 will be fetched if libzstd can not be found by pkg-config.

include/SZ3/api/impl/SZAlgoBioMD.hpp

@@ -1,14 +1,15 @@
 #ifndef SZ3_SZ_BIOMD_HPP
 #define SZ3_SZ_BIOMD_HPP
 
+#include "SZ3/compressor/SZGenericCompressor.hpp"
 #include "SZ3/decomposition/SZBioMDDecomposition.hpp"
 #include "SZ3/decomposition/SZBioMDXtcDecomposition.hpp"
 #include "SZ3/def.hpp"
+#include "SZ3/encoder/HuffmanEncoder.hpp"
+#include "SZ3/encoder/HuffmanEncoderV2.hpp"
 #include "SZ3/encoder/XtcBasedEncoder.hpp"
 #include "SZ3/lossless/Lossless_bypass.hpp"
 #include "SZ3/lossless/Lossless_zstd.hpp"
-#include "SZ3/encoder/HuffmanEncoderV2.hpp"
-#include "SZ3/encoder/HuffmanEncoder.hpp"
 #include "SZ3/quantizer/LinearQuantizer.hpp"
 #include "SZ3/utils/Config.hpp"
 #include "SZ3/utils/Statistic.hpp"

include/SZ3/api/impl/SZAlgoInterp.hpp

@@ -1,6 +1,8 @@
 #ifndef SZ3_SZALGO_INTERP_HPP
 #define SZ3_SZALGO_INTERP_HPP
 
+#include <memory>
+
 #include "SZ3/api/impl/SZAlgoLorenzoReg.hpp"
 #include "SZ3/decomposition/BlockwiseDecomposition.hpp"
 #include "SZ3/decomposition/InterpolationDecomposition.hpp"

include/SZ3/api/impl/SZDispatcher.hpp

@@ -1,10 +1,13 @@
 #ifndef SZ3_IMPL_SZDISPATCHER_HPP
 #define SZ3_IMPL_SZDISPATCHER_HPP
 
+#include <memory>
+#include <stdexcept>
+
+#include "SZ3/api/impl/SZAlgoBioMD.hpp"
 #include "SZ3/api/impl/SZAlgoInterp.hpp"
 #include "SZ3/api/impl/SZAlgoLorenzoReg.hpp"
 #include "SZ3/api/impl/SZAlgoNopred.hpp"
-#include "SZ3/api/impl/SZAlgoBioMD.hpp"
 #include "SZ3/utils/Config.hpp"
 #include "SZ3/utils/Statistic.hpp"
 
@@ -62,15 +65,14 @@ size_t SZ_compress_dispatcher(Config &conf, const T *data, uchar *cmpData, size_
     if (conf.num * sizeof(T) / 1.0 / cmpSize < 3) {
         auto zstd = Lossless_zstd();
         auto zstdCmpCap = ZSTD_compressBound(conf.num * sizeof(T)) + sizeof(size_t);
-        auto zstdCmpData = static_cast<uchar *>(malloc(zstdCmpCap));
-        size_t zstdCmpSize =
-            zstd.compress(reinterpret_cast<const uchar *>(data), conf.num * sizeof(T), zstdCmpData, zstdCmpCap);
+        std::unique_ptr<uchar[]> zstdCmpData(new uchar[zstdCmpCap]);
+        size_t zstdCmpSize = zstd.compress(reinterpret_cast<const uchar *>(data), conf.num * sizeof(T),
+                                           zstdCmpData.get(), zstdCmpCap);
         if (zstdCmpSize < cmpSize && zstdCmpSize <= cmpCap) {
             conf.cmprAlgo = ALGO_LOSSLESS;
-            memcpy(cmpData, zstdCmpData, zstdCmpSize);
+            memcpy(cmpData, zstdCmpData.get(), zstdCmpSize);
             cmpSize = zstdCmpSize;
         }
-        free(zstdCmpData);
     }
     return cmpSize;
 }
@@ -79,7 +81,7 @@ template <class T, uint N>
 void SZ_decompress_dispatcher(Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) {
     if (conf.cmprAlgo == ALGO_LOSSLESS) {
         auto zstd = Lossless_zstd();
-        size_t decDataSize = 0;
+        size_t decDataSize = conf.num * sizeof(T);
         auto decDataPos = reinterpret_cast<uchar *>(decData);
         zstd.decompress(cmpData, cmpSize, decDataPos, decDataSize);
         if (decDataSize != conf.num * sizeof(T)) {

include/SZ3/api/impl/SZImplOMP.hpp

@@ -2,13 +2,16 @@
 #define SZ3_IMPL_SZDISPATCHER_OMP_HPP
 
 #include <cmath>
+#include <cstdlib>
 #include <memory>
+#include <new>
 
 #include "SZ3/api/impl/SZDispatcher.hpp"
 
 #ifdef _OPENMP
 
 #include <omp.h>
+#include <stdexcept>
 
 #endif
 namespace SZ3 {
@@ -70,8 +73,11 @@ size_t SZ_compress_OMP(Config& conf, const T* data, uchar* cmpData, size_t cmpCa
 
         conf_t[tid] = conf;
         conf_t[tid].setDims(dims_t.begin(), dims_t.end());
-        size_t cmp_size_cap = ZSTD_compressBound(conf_t[tid].num * sizeof(T));
-        compressed_t[tid] = static_cast<uchar*>(malloc(cmp_size_cap));
+        // Room for the size header Lossless_zstd::compress writes ahead of the zstd stream; without it the
+        // direct lossless path in SZ_compress_dispatcher throws on poorly compressible chunks.
+        size_t cmp_size_cap = sizeof(size_t) + ZSTD_compressBound(conf_t[tid].num * sizeof(T));
+        std::unique_ptr<uchar[]> compressed_owner(new uchar[cmp_size_cap]);
+        compressed_t[tid] = compressed_owner.get();
         // we have to use conf_t[tid].N instead of N since each chunk may be a slice of the original data
         if (conf_t[tid].N == 1) {
             cmp_size_t[tid] = SZ_compress_dispatcher<T, 1>(conf_t[tid], data_t, compressed_t[tid], cmp_size_cap);
@@ -105,7 +111,6 @@ size_t SZ_compress_OMP(Config& conf, const T* data, uchar* cmpData, size_t cmpCa
         }
 
         memcpy(buffer_pos + cmp_start_t[tid], compressed_t[tid], cmp_size_t[tid]);
-        free(compressed_t[tid]);
     }
 
     return buffer_pos - cmpData + cmp_start_t[nThreads];
@@ -121,14 +126,21 @@ void SZ_decompress_OMP(Config& conf, const uchar* cmpData, size_t cmpSize, T* de
 #ifdef _OPENMP
 
     auto cmpr_data_pos = cmpData;
+    const uchar* const cmp_end = cmpData + cmpSize;
     int nThreads = 1;
+    if (static_cast<size_t>(cmp_end - cmpr_data_pos) < sizeof(nThreads))
+        throw std::out_of_range("SZ3 OMP: truncated thread count");
     read(nThreads, cmpr_data_pos);
+    // Each thread contributes at least a config and a size, so the count cannot exceed the buffer size.
+    if (nThreads <= 0 || static_cast<size_t>(nThreads) > cmpSize)
+        throw std::out_of_range("SZ3 OMP: invalid thread count");
     omp_set_num_threads(nThreads);
     printf("OpenMP enabled for decompression, threads = %d\n", nThreads);
 
     std::vector<Config> conf_t(nThreads);
     for (int i = 0; i < nThreads; i++) {
-        conf_t[i].load(cmpr_data_pos);
+        size_t confRemaining = static_cast<size_t>(cmp_end - cmpr_data_pos);
+        conf_t[i].load(cmpr_data_pos, confRemaining);
     }
 
     if (conf_t[0].sz3MagicNumber != SZ3_MAGIC_NUMBER) {
@@ -145,12 +157,18 @@ void SZ_decompress_OMP(Config& conf, const uchar* cmpData, size_t cmpSize, T* de
 
     std::vector<size_t> cmp_start_t, cmp_size_t;
     cmp_size_t.resize(nThreads);
+    if (static_cast<size_t>(cmp_end - cmpr_data_pos) < static_cast<size_t>(nThreads) * sizeof(size_t))
+        throw std::out_of_range("SZ3 OMP: truncated per-thread sizes");
     read(cmp_size_t.data(), nThreads, cmpr_data_pos);
     auto cmpr_data_p = cmpr_data_pos;
 
     cmp_start_t.resize(nThreads + 1);
     cmp_start_t[0] = 0;
+    // The payloads follow back-to-back; build the offsets without overflowing so no slice points out.
+    const size_t payload_avail = static_cast<size_t>(cmp_end - cmpr_data_p);
     for (int i = 1; i <= nThreads; i++) {
+        if (cmp_size_t[i - 1] > payload_avail - cmp_start_t[i - 1])
+            throw std::out_of_range("SZ3 OMP: per-thread compressed sizes exceed the buffer");
         cmp_start_t[i] = cmp_start_t[i - 1] + cmp_size_t[i - 1];
     }
 
@@ -199,8 +217,9 @@ size_t SZ_compress_size_bound_omp(const Config& conf) {
     }
     size_t chunk_size = conf.dims[0] / static_cast<size_t>(nThreads) * (conf.num / conf.dims[0]);
     size_t last_chunk_size = (conf.dims[0] - conf.dims[0] / nThreads * (nThreads - 1)) * (conf.num / conf.dims[0]);
-    //for each thread, we save conf, compressed size, and compressed data
-    return sizeof(int) + nThreads * conf.size_est() + nThreads * sizeof(size_t) +
+    // for each thread, we save conf, compressed size, and compressed data
+    // the per-chunk compressed data may carry the size header written by Lossless_zstd::compress
+    return sizeof(int) + nThreads * conf.size_est() + 2 * nThreads * sizeof(size_t) +
            (nThreads - 1) * ZSTD_compressBound(chunk_size * sizeof(T)) +
            ZSTD_compressBound(last_chunk_size * sizeof(T));
 #else

include/SZ3/api/sz.hpp

@@ -21,10 +21,12 @@
 #ifndef SZ3_SZ_HPP
 #define SZ3_SZ_HPP
 
+#include <memory>
+#include <stdexcept>
+
 #include "SZ3/api/impl/SZImpl.hpp"
 #include "SZ3/version.hpp"
 
-
 /**
  * Compresses the input data using the provided configuration and stores the result in a pre-allocated buffer.
  * @tparam T The data type of the source data.
@@ -95,10 +97,10 @@ char* SZ_compress(const SZ3::Config& config, const T* data, size_t& cmpSize) {
     using namespace SZ3;
 
     size_t bufferLen = SZ_compress_size_bound<T>(config);
-    auto buffer = new char[bufferLen];
-    cmpSize = SZ_compress(config, data, buffer, bufferLen);
+    std::unique_ptr<char[]> buffer(new char[bufferLen]);
+    cmpSize = SZ_compress(config, data, buffer.get(), bufferLen);
 
-    return buffer;
+    return buffer.release();
 }
 
 /**
@@ -119,6 +121,11 @@ void SZ_decompress(SZ3::Config& config, const char* cmpData, size_t cmpSize, T*&
 
     auto cmpDataPos = reinterpret_cast<const uchar*>(cmpData);
 
+    // Header layout: magic number (4) + data version (4) + compressed payload size (8) = 16 bytes.
+    if (cmpSize < 16) {
+        throw std::out_of_range("SZ3: compressed data is smaller than the header");
+    }
+
     read(config.sz3MagicNumber, cmpDataPos);
     if (config.sz3MagicNumber != SZ3_MAGIC_NUMBER) {
         throw std::invalid_argument("magic number mismatch, the input data is not compressed by SZ3");
@@ -137,8 +144,13 @@ void SZ_decompress(SZ3::Config& config, const char* cmpData, size_t cmpSize, T*&
     uint64_t cmpDataSize = 0;
     read(cmpDataSize, cmpDataPos);
 
+    // The compressed payload is followed by the serialized config; both must fit in the remaining bytes.
+    if (cmpDataSize > cmpSize - 16) {
+        throw std::out_of_range("SZ3: compressed payload size exceeds the buffer");
+    }
     auto cmpConfPos = cmpDataPos + cmpDataSize;
-    config.load(cmpConfPos);
+    size_t confRemaining = cmpSize - 16 - cmpDataSize;
+    config.load(cmpConfPos, confRemaining);
 
     if (decData == nullptr) {
         decData = new T[config.num];

include/SZ3/compressor/SZGenericCompressor.hpp

@@ -1,7 +1,11 @@
 #ifndef SZ3_COMPRESSOR_TYPE_ONE_HPP
 #define SZ3_COMPRESSOR_TYPE_ONE_HPP
 
+#include <cstdlib>
 #include <cstring>
+#include <memory>
+#include <stdexcept>
+#include <type_traits>
 
 #include "SZ3/compressor/Compressor.hpp"
 #include "SZ3/decomposition/Decomposition.hpp"
@@ -13,6 +17,7 @@
 #include "SZ3/utils/Timer.hpp"
 
 namespace SZ3 {
+
 /**
  * SZGenericCompressor glues together decomposition, encoder, and lossless modules to form the compressor.
  * It only takes Decomposition, not Predictor.
@@ -45,39 +50,52 @@ class SZGenericCompressor : public concepts::CompressorInterface<T> {
         size_t bufferSize = std::max<size_t>(
             1000, 2 * (decomposition.size_est() + encoder.size_est() + sizeof(T) * quant_inds.size()));
 
-        auto buffer = static_cast<uchar *>(malloc(bufferSize));
+        // Owned: the encoder and the lossless layer can throw and the caller continues, so a bare delete
+        // at the end leaks on every failed compression.
+        std::unique_ptr<uchar[]> buffer_owner(new uchar[bufferSize]);
+        uchar *const buffer = buffer_owner.get();
         uchar *buffer_pos = buffer;
 
         decomposition.save(buffer_pos);
         encoder.save(buffer_pos);
 
-        //store the size of quant_inds is necessary as it is not always equal to conf.num
+        // store the size of quant_inds is necessary as it is not always equal to conf.num
         write<size_t>(quant_inds.size(), buffer_pos);
         encoder.encode(quant_inds, buffer_pos);
         encoder.postprocess_encode();
 
         auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap);
-        free(buffer);
 
         return cmpSize;
     }
 
     T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) override {
         uchar *buffer = nullptr;
+        // No cap: this buffer holds the uncompressed stream, whose size conf does not bound.
         size_t bufferSize = 0;
         lossless.decompress(cmpData, cmpSize, buffer, bufferSize);
 
+        // malloc'd by the lossless layer, hence the free() deleter. Owned because the parsing below is on
+        // untrusted data and can throw.
+        std::unique_ptr<uchar, void (*)(void *)> buffer_owner(buffer, &free);
+
         uchar const *bufferPos = buffer;
 
         decomposition.load(bufferPos, bufferSize);
         encoder.load(bufferPos, bufferSize);
 
         size_t quant_inds_size = 0;
-        read(quant_inds_size, bufferPos);
-        auto quant_inds = encoder.decode(bufferPos, quant_inds_size);
+        read(quant_inds_size, bufferPos, bufferSize);
+        // A decomposition takes at most one bin per element, and BIOMDXTC's multi-frame path takes fewer,
+        // so conf.num is a ceiling rather than the count. Each decomposition checks its own floor.
+        if (quant_inds_size > conf.num) {
+            throw std::out_of_range("SZ3: declared bin count exceeds the configured element count");
+        }
+        auto quant_inds = encoder.decode(bufferPos, quant_inds_size, bufferSize);
         encoder.postprocess_decode();
 
-        free(buffer);
+        // The remaining work uses `quant_inds` and `decData` only, so release the internal buffer now.
+        buffer_owner.reset();
 
         decomposition.decompress(conf, quant_inds, decData);
         return decData;

include/SZ3/compressor/specialized/SZExaaltCompressor.hpp

@@ -1,6 +1,12 @@
 #ifndef SZ3_EXAALT_COMPRESSSOR_HPP
 #define SZ3_EXAALT_COMPRESSSOR_HPP
 
+#include <iostream>
+#include <limits>
+#include <memory>
+#include <stdexcept>
+
+#include "SZ3/compressor/Compressor.hpp"
 #include "SZ3/def.hpp"
 #include "SZ3/encoder/Encoder.hpp"
 #include "SZ3/lossless/Lossless.hpp"
@@ -121,17 +127,18 @@ class SZExaaltCompressor : public SZ3::concepts::CompressorInterface<T> {
         uchar *buffer = nullptr;
         size_t bufferSize = 0;
         lossless.decompress(cmpData, cmpSize, buffer, bufferSize);
-        size_t remaining_length = cmpSize;
+        // The parsing below walks the decompressed buffer, so bufferSize is its bound, not cmpSize.
+        size_t remaining_length = bufferSize;
         uchar const *buffer_pos = buffer;
 
         quantizer.load(buffer_pos, remaining_length);
         encoder.load(buffer_pos, remaining_length);
-        auto quant_inds = encoder.decode(buffer_pos, conf.num);
+        auto quant_inds = encoder.decode(buffer_pos, conf.num, remaining_length);
         encoder.postprocess_decode();
 
         encoder.load(buffer_pos, remaining_length);
         auto pred_inds_num = (timestep_op == 1) ? conf.dims[1] : conf.num;
-        auto pred_inds = encoder.decode(buffer_pos, pred_inds_num);
+        auto pred_inds = encoder.decode(buffer_pos, pred_inds_num, remaining_length);
         encoder.postprocess_decode();
 
         free(buffer);