From 42b25f42506ca7d138c017b13d56fec6d8b3cd61 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 27 Jul 2026 06:43:53 -0700 Subject: [PATCH 1/8] Initial commit for GetSubrange --- sei-db/db_engine/litt/dbcache/cached_table.go | 33 ++++++ sei-db/db_engine/litt/disktable/disk_table.go | 44 +++++++ .../disktable/disk_table_subrange_test.go | 94 +++++++++++++++ .../litt/disktable/segment/segment.go | 50 ++++++++ sei-db/db_engine/litt/table.go | 23 ++++ .../block/littblock/litt_block_db.go | 41 +++++++ .../block/littblock/litt_block_db_test.go | 109 ++++++++++++++++++ 7 files changed, 394 insertions(+) create mode 100644 sei-db/db_engine/litt/disktable/disk_table_subrange_test.go create mode 100644 sei-db/ledger_db/block/littblock/litt_block_db_test.go diff --git a/sei-db/db_engine/litt/dbcache/cached_table.go b/sei-db/db_engine/litt/dbcache/cached_table.go index 82e85ddf9b..0f269b7d89 100644 --- a/sei-db/db_engine/litt/dbcache/cached_table.go +++ b/sei-db/db_engine/litt/dbcache/cached_table.go @@ -113,6 +113,39 @@ func (c *cachedTable) Get(key []byte) (value []byte, exists bool, err error) { return value, exists, nil } +// GetSubrange reads only the [offset, offset+length) byte range of the value stored under key. +// +// Cache policy — subrange reads intentionally bypass BOTH caches: +// +// - The read and write caches are keyed by the full key and store the full value. A subrange read cannot +// safely populate them (it would store a partial value under a key that Get is entitled to treat as the +// whole value), and it does not consult them either: even on a cache hit the whole point of a subrange +// read is to avoid materializing the full value, and slicing a cached full value would defeat the I/O +// savings the base implementation provides. +// - A more elaborate scheme could cache sub-ranges under specially structured keys, but that adds +// complexity we do not yet know we need. We start simple by skipping the cache and documenting it here, +// with the intent to revisit if profiling shows subrange reads are hot enough that the missing cache +// hurts. Until then, every GetSubrange goes straight to the base table. +// +// This still reports the read to metrics (as a cold read), for parity with Get. +func (c *cachedTable) GetSubrange(key []byte, offset uint32, length uint32) (value []byte, exists bool, err error) { + if c.metrics != nil { + start := time.Now() + defer func() { + if exists && value != nil { + // hot is always false: subrange reads never come from a cache (see the cache policy above). + c.metrics.ReportReadOperation(c.Name(), time.Since(start), uint64(len(value)), false) + } + }() + } + + value, exists, err = c.base.GetSubrange(key, offset, length) + if err != nil { + return value, exists, fmt.Errorf("failed to get subrange from base table: %w", err) + } + return value, exists, nil +} + func (c *cachedTable) Exists(key []byte) (exists bool, err error) { _, exists = c.writeCache.Get(util.UnsafeBytesToString(key)) if exists { diff --git a/sei-db/db_engine/litt/disktable/disk_table.go b/sei-db/db_engine/litt/disktable/disk_table.go index 119de9fb53..c2697b944d 100644 --- a/sei-db/db_engine/litt/disktable/disk_table.go +++ b/sei-db/db_engine/litt/disktable/disk_table.go @@ -981,6 +981,50 @@ func (d *DiskTable) Get(key []byte) (value []byte, exists bool, err error) { return data, true, nil } +// GetSubrange reads only the [offset, offset+length) byte range of the value stored under key. See the +// litt.Table.GetSubrange contract; this is the base implementation that reaches keymap + disk. +func (d *DiskTable) GetSubrange(key []byte, offset uint32, length uint32) (value []byte, exists bool, err error) { + if ok, err := d.errorMonitor.IsOk(); !ok { + return nil, false, fmt.Errorf( + "cannot process GetSubrange() request, DB is in panicked state due to error: %w", err) + } + + // Data not yet flushed lives in memory as the full value; slice the requested range out of it. + if v, ok := d.unflushedDataCache.Load(util.UnsafeBytesToString(key)); ok { + full := v.([]byte) + end := uint64(offset) + uint64(length) + if end > uint64(len(full)) { + return nil, false, fmt.Errorf( + "subrange [%d, %d) is out of bounds for value of length %d", offset, end, len(full)) + } + return full[offset:end], true, nil + } + + // Look up the address of the data. + address, ok, err := d.keymap.Get(key) + if err != nil { + return nil, false, fmt.Errorf("failed to get address: %w", err) + } + if !ok { + return nil, false, nil + } + + // Reserve the segment that contains the data. + seg, ok := d.controlLoop.getReservedSegment(address.Index()) + if !ok { + return nil, false, nil + } + defer seg.Release() + + // Read only the requested byte range from disk. + data, err := seg.ReadSubrange(key, address, offset, length) + if err != nil { + return nil, false, fmt.Errorf("failed to read data subrange: %w", err) + } + + return data, true, nil +} + func (d *DiskTable) Put(key []byte, value []byte, secondaryKeys ...*types.SecondaryKey) error { return d.PutBatch([]*types.PutRequest{{Key: key, Value: value, SecondaryKeys: secondaryKeys}}) } diff --git a/sei-db/db_engine/litt/disktable/disk_table_subrange_test.go b/sei-db/db_engine/litt/disktable/disk_table_subrange_test.go new file mode 100644 index 0000000000..999d207d94 --- /dev/null +++ b/sei-db/db_engine/litt/disktable/disk_table_subrange_test.go @@ -0,0 +1,94 @@ +package disktable + +import ( + "testing" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/types" + "github.com/sei-protocol/sei-chain/sei-db/db_engine/litt/util" + "github.com/stretchr/testify/require" +) + +// TestGetSubrange exercises the GetSubrange read path across every disk-table implementation, both before +// a flush (served from the in-memory unflushed data cache) and after (served from the keymap + a bounded +// segment read). It checks that a sub-range read returns exactly value[offset:offset+length], that it +// composes with secondary keys (which alias a sub-range of the value), that a zero-length read is valid, +// that an out-of-bounds range errors, and that a missing key reports not-found without an error. +func TestGetSubrange(t *testing.T) { + t.Parallel() + for _, tb := range tableBuilders { + t.Run(tb.name, func(t *testing.T) { + t.Parallel() + rand := util.NewTestRandom() + directory := t.TempDir() + tableName := rand.String(8) + table, err := tb.builder(time.Now, tableName, []string{directory}) + require.NoError(t, err) + + // 0 1 2 3 4 + // 0123456789012345678901234567890123456789012 + value := []byte("the quick brown fox jumps over the lazy dog") + primary := []byte("primary") + // A secondary aliasing the strict sub-range "brown fox", to prove GetSubrange composes with a + // secondary key (its address already points at a sub-range of the value's bytes). + sk := &types.SecondaryKey{Key: []byte("brown-fox"), Offset: 10, Length: 9} + require.NoError(t, table.Put(primary, value, sk)) + + valueLen := uint32(len(value)) + + verify := func(stage string) { + t.Helper() + + // The full range equals a plain Get. + got, ok, err := table.GetSubrange(primary, 0, valueLen) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, value, got, stage) + + // Assorted sub-ranges, including zero-length reads in the middle and at the very end. + ranges := []struct{ off, length uint32 }{ + {0, 3}, // "the" + {4, 5}, // "quick" + {valueLen - 3, 3}, // "dog" + {10, 0}, // zero-length in the middle + {valueLen, 0}, // zero-length at the very end + } + for _, r := range ranges { + got, ok, err := table.GetSubrange(primary, r.off, r.length) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.NotNil(t, got, stage) + require.Equal(t, value[r.off:r.off+r.length], got, stage) + } + + // A sub-range read of a secondary key stays within the secondary's aliased region. + got, ok, err = table.GetSubrange(sk.Key, 0, sk.Length) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, value[sk.Offset:sk.Offset+sk.Length], got, stage) + + got, ok, err = table.GetSubrange(sk.Key, 6, 3) // "fox" within "brown fox" + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, value[sk.Offset+6:sk.Offset+6+3], got, stage) + + // A range that runs past the end of the value is an error. + _, _, err = table.GetSubrange(primary, valueLen-1, 5) + require.Error(t, err, stage) + _, _, err = table.GetSubrange(primary, valueLen+1, 0) + require.Error(t, err, stage) + + // A missing key reports not-found with no error. + _, ok, err = table.GetSubrange([]byte("does-not-exist"), 0, 1) + require.NoError(t, err, stage) + require.False(t, ok, stage) + } + + verify("before flush") + require.NoError(t, table.Flush()) + verify("after flush") + + require.NoError(t, table.Drop()) + }) + } +} diff --git a/sei-db/db_engine/litt/disktable/segment/segment.go b/sei-db/db_engine/litt/disktable/segment/segment.go index ac55f5353d..6b5713b798 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment.go +++ b/sei-db/db_engine/litt/disktable/segment/segment.go @@ -714,6 +714,56 @@ func (s *Segment) Read(key []byte, dataAddress types.Address) ([]byte, error) { return s.maybeDecompress(value) } +// ReadSubrange fetches only the [offset, offset+length) byte range of the value identified by dataAddress, +// avoiding a read of the full value. +// +// For an uncompressed segment this issues a bounded read of exactly length bytes, seeking to the value's +// on-disk offset plus the requested sub-offset — so the I/O cost scales with length, not the full value. +// For a compressed segment the on-disk blob is a single compressed unit that cannot be sliced, so this +// falls back to reading and decompressing the whole value (via Read) and then slicing out the requested +// range. Either way the returned bytes are the plaintext value[offset:offset+length]. +// +// The requested range must lie within the value; otherwise an error is returned. +// +// It is only thread safe to read from a segment if the key being read has previously been flushed to disk. +func (s *Segment) ReadSubrange(key []byte, dataAddress types.Address, offset uint32, length uint32) ([]byte, error) { + // A compressed value cannot be partially read from disk: decompress the whole thing, then slice. The + // bounds check is against the decompressed length, since dataAddress.ValueSize() is the compressed size. + if s.IsCompressed() { + value, err := s.Read(key, dataAddress) + if err != nil { + return nil, err + } + end := uint64(offset) + uint64(length) + if end > uint64(len(value)) { + return nil, fmt.Errorf("subrange [%d, %d) is out of bounds for value of length %d", + offset, end, len(value)) + } + return value[offset:end], nil + } + + // For an uncompressed value, dataAddress.ValueSize() is the exact value length, so we can bound the + // range and read only the requested bytes directly from disk. + end := uint64(offset) + uint64(length) + if end > uint64(dataAddress.ValueSize()) { + return nil, fmt.Errorf("subrange [%d, %d) is out of bounds for value of length %d", + offset, end, dataAddress.ValueSize()) + } + + values, err := s.shardForAddress(dataAddress) + if err != nil { + return nil, fmt.Errorf("failed to resolve shard for read: %w", err) + } + + // The value starts at dataAddress.Offset(); the sub-range starts offset bytes further in. Both operands + // and their sum are bounded by the value file size (< 2^32), so the uint32 addition cannot overflow. + value, err := values.read(dataAddress.Offset()+offset, length) + if err != nil { + return nil, fmt.Errorf("failed to read value subrange: %w", err) + } + return value, nil +} + // maybeDecompress decodes an on-disk value from a compressed segment (stripping the per-value algorithm // tag and decompressing the body; see types.EncodeValue), or returns it unchanged if the segment is not // compressed. All value reads (Segment.Read and SegmentReader.Read) pass through here so the on-disk diff --git a/sei-db/db_engine/litt/table.go b/sei-db/db_engine/litt/table.go index be219dda6d..a0c895ca1b 100644 --- a/sei-db/db_engine/litt/table.go +++ b/sei-db/db_engine/litt/table.go @@ -78,6 +78,29 @@ type Table interface { // method. Get(key []byte) (value []byte, exists bool, err error) + // GetSubrange retrieves only the [offset, offset+length) byte range of the value stored under key, + // without materializing the whole value. The returned boolean indicates whether the key exists (false + // if it does not, in which case value is nil). + // + // This is intended for large values where a caller knows the exact byte range it needs (for example, a + // single serialized transaction within a serialized block): rather than read the entire value and slice + // it, an uncompressed on-disk value is read with a bounded, seek-then-read of exactly length bytes, so + // the I/O cost scales with length rather than the full value size. (A value stored in a compressed + // segment is a single compressed unit that cannot be sliced on disk, so for those the full value is + // read and decompressed before the sub-range is sliced out — still correct, but without the I/O + // savings. The default table configuration is uncompressed.) + // + // The requested range must be within the value: if offset+length exceeds the value's length, an error + // is returned. A zero-length range is valid and returns an empty (non-nil) slice when the key exists. + // + // Caching note: unlike Get, GetSubrange deliberately does not consult or populate the read/write value + // caches (see the cachedTable implementation for the rationale). Every GetSubrange therefore reaches the + // base table. + // + // As with Get, the returned data is NOT safe to mutate, and the key byte slice must not be modified + // after it is passed to this method. + GetSubrange(key []byte, offset uint32, length uint32) (value []byte, exists bool, err error) + // Exists returns true if the key exists in the database, and false otherwise. This is faster than calling Get. // // It is not safe to modify the key byte slice after it is passed to this method. diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index a514b63768..a8a54132db 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -428,6 +428,47 @@ func (s *blockDB) Close() error { return nil } +// GetTxByOffset reads a single transaction's raw bytes out of the block stored at GlobalBlockNumber n, +// without decoding the block. offset and length identify a byte range within the block's stored value — +// the exact bytes encodeBlock produced: [version:1][GlobalBlockNumber:8][proto(Block)] — typically the +// (offset, length) of one transaction recorded at write time (see the tx-location-index design under +// sei-db/ledger_db/receipt/docs). offset is measured from the start of that stored value (i.e. it includes +// the fixed prefix), matching the coordinates a writer computes against the bytes it persists. +// +// It reads only the requested bytes via LittDB's GetSubrange, skipping both the full-block disk read and +// the protobuf unmarshal that ReadBlockByNumber performs. The I/O savings apply while the ledger table is +// uncompressed (the default); on a compressed table GetSubrange still returns the correct bytes but must +// read and decompress the whole block value first (see the compression note in the design doc). +// +// This lives outside the types.BlockDB interface because the offset space is defined by this +// implementation's serialization. The result is one of: +// +// - Some(txBytes) with a nil error: the byte range was read. +// - types.ErrPruned: n is strictly below the retention watermark (matches ReadBlockByNumber). A block +// below the watermark may be stranded from its covering QC and is never served. +// - None with a nil error: no block is present at n (never written, or not yet written). +// - a non-nil error: the range is out of bounds for the block value, or the read failed. +func (s *blockDB) GetTxByOffset( + n types.GlobalBlockNumber, + offset uint32, + length uint32, +) (utils.Option[[]byte], error) { + // Refuse below-watermark blocks: they may be stranded (covering QC reclaimed). Mirrors ReadBlockByNumber. + if uint64(n) < s.watermark.Load() { + return utils.None[[]byte](), types.ErrPruned + } + + value, exists, err := s.table.GetSubrange(blockKey(n), offset, length) + if err != nil { + return utils.None[[]byte](), fmt.Errorf( + "failed to read tx range [%d, %d) in block %d: %w", offset, uint64(offset)+uint64(length), n, err) + } + if !exists { + return utils.None[[]byte](), nil + } + return utils.Some(value), nil +} + // ForceGC runs a synchronous garbage-collection pass over the table backing db, // so any pending prune takes effect immediately rather than on the periodic GC // schedule. db must be a *blockDB returned by NewBlockDB. Intended for tests and diff --git a/sei-db/ledger_db/block/littblock/litt_block_db_test.go b/sei-db/ledger_db/block/littblock/litt_block_db_test.go new file mode 100644 index 0000000000..8b0e787471 --- /dev/null +++ b/sei-db/ledger_db/block/littblock/litt_block_db_test.go @@ -0,0 +1,109 @@ +package littblock + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-tendermint/autobahn/types" + "github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils" +) + +// genBlockWithTxs returns a random block guaranteed to carry at least one transaction, so the +// tx-extraction assertions below are meaningful. +func genBlockWithTxs(rng utils.Rng) *types.Block { + for { + blk := types.GenBlock(rng) + if len(blk.Payload().Txs()) > 0 { + return blk + } + } +} + +// TestGetTxByOffset covers the block-store sub-range read primitive: it must return exactly the requested +// byte range of a block's stored value (which is what encodeBlock produced), both before and after a flush, +// and it must extract a real transaction's bytes when given that transaction's byte range within the value. +func TestGetTxByOffset(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(1) + + cfg, err := DefaultConfig(dir) + require.NoError(t, err) + db, err := NewBlockDB(cfg) + require.NoError(t, err) + defer func() { _ = db.Close() }() + impl := db.(*blockDB) + + // Write a QC covering block 0, then the block itself. + blk := genBlockWithTxs(rng) + require.NoError(t, db.WriteQC(0, 1, types.GenFullCommitQCRange(rng, 0, 1))) + require.NoError(t, db.WriteBlock(0, blk)) + + // The stored value for the primary block key is exactly encodeBlock's output; GetTxByOffset returns a + // byte range of that value. + stored := encodeBlock(0, blk) + storedLen := uint32(len(stored)) + + verify := func(stage string) { + t.Helper() + + // The full range round-trips the whole stored value. + res, err := impl.GetTxByOffset(0, 0, storedLen) + require.NoError(t, err, stage) + got, ok := res.Get() + require.True(t, ok, stage) + require.Equal(t, stored, got, stage) + + // A real transaction's raw bytes appear verbatim as a contiguous run in the value (each tx is a + // length-delimited `repeated bytes` element), so locating one gives a valid (offset, length) — + // exactly what a writer would record. Extracting that range must return the transaction. + for _, tx := range blk.Payload().Txs() { + idx := bytes.Index(stored, tx) + require.GreaterOrEqual(t, idx, 0, stage) + //nolint:gosec // small test offsets/lengths fit u32 + res, err := impl.GetTxByOffset(0, uint32(idx), uint32(len(tx))) + require.NoError(t, err, stage) + got, ok := res.Get() + require.True(t, ok, stage) + require.Equal(t, tx, got, stage) + } + + // A range past the end of the value is an error. + _, err = impl.GetTxByOffset(0, storedLen-1, 5) + require.Error(t, err, stage) + + // A block that was never written is simply absent (not an error). + res, err = impl.GetTxByOffset(1, 0, 1) + require.NoError(t, err, stage) + require.False(t, res.IsPresent(), stage) + } + + verify("before flush") + require.NoError(t, db.Flush()) + verify("after flush") +} + +// TestGetTxByOffsetPruned verifies that a block below the retention watermark is reported ErrPruned, +// matching ReadBlockByNumber, rather than served (it may be stranded from its covering QC). +func TestGetTxByOffsetPruned(t *testing.T) { + dir := t.TempDir() + rng := utils.TestRngFromSeed(2) + + db, err := NewBlockDB(strandingConfig(t, dir, 8)) + require.NoError(t, err) + defer func() { _ = db.Close() }() + impl := db.(*blockDB) + + writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QCs [0,5),[5,10),[10,15),[15,20) + require.NoError(t, db.PruneBefore(5)) // watermark to 5: blocks 0..4 are below it + + res, err := impl.GetTxByOffset(2, 0, 1) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, res.IsPresent()) + + // A block at/above the watermark is still served. + res, err = impl.GetTxByOffset(5, 0, 1) + require.NoError(t, err) + require.True(t, res.IsPresent()) +} From 1e4d70455888c052e206ef72eaa0fcbbfbe0609d Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 27 Jul 2026 07:08:42 -0700 Subject: [PATCH 2/8] Add unit test --- .../litt/disktable/compression_test.go | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/sei-db/db_engine/litt/disktable/compression_test.go b/sei-db/db_engine/litt/disktable/compression_test.go index c4b63dac19..109492052f 100644 --- a/sei-db/db_engine/litt/disktable/compression_test.go +++ b/sei-db/db_engine/litt/disktable/compression_test.go @@ -162,6 +162,60 @@ func TestCompressionFullValueAliasSecondary(t *testing.T) { verify("after flush") } +// TestGetSubrangeCompressed proves GetSubrange stays correct on a compressed table: since a compressed +// on-disk blob cannot be sliced, Segment.ReadSubrange falls back to reading and decompressing the whole +// value and then slicing the requested range out of the plaintext (see Segment.ReadSubrange). This checks +// that fallback for both a compressible value (stored S2-tagged) and an incompressible one (stored +// CompressionNone-tagged on the same compressed segment), before and after flush. +func TestGetSubrangeCompressed(t *testing.T) { + t.Parallel() + dir := t.TempDir() + table := buildCompressedMemKeyDiskTable(t, time.Now, "subrange-compressed", []string{dir}, types.CompressionS2) + defer func() { require.NoError(t, table.Close()) }() + + compressible := compressiblePayload() + incompressible := incompressiblePayload() + require.NoError(t, table.Put([]byte("compressible"), compressible)) + require.NoError(t, table.Put([]byte("incompressible"), incompressible)) + + verifyKey := func(stage string, key string, value []byte) { + t.Helper() + n := uint32(len(value)) + + // Full range. + got, ok, err := table.GetSubrange([]byte(key), 0, n) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, value, got, stage) + + // Prefix, middle, suffix, and a zero-length read. + for _, r := range []struct{ off, length uint32 }{ + {0, 10}, + {n / 2, 10}, + {n - 10, 10}, + {n / 2, 0}, + } { + got, ok, err := table.GetSubrange([]byte(key), r.off, r.length) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, value[r.off:r.off+r.length], got, stage) + } + + // Out of bounds is still an error against the decompressed (logical) length. + _, _, err = table.GetSubrange([]byte(key), n-1, 10) + require.Error(t, err, stage) + } + + verify := func(stage string) { + verifyKey(stage, "compressible", compressible) + verifyKey(stage, "incompressible", incompressible) + } + + verify("before flush") + require.NoError(t, table.Flush()) + verify("after flush") +} + func TestCompressionRejectsSubRangeSecondary(t *testing.T) { t.Parallel() dir := t.TempDir() From cdb21b2da0ab0113deda7fe8500d41b03c0aeb27 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 27 Jul 2026 12:20:45 -0700 Subject: [PATCH 3/8] Fix cache behavior --- sei-db/db_engine/litt/dbcache/cached_table.go | 58 ++++++-- sei-db/db_engine/litt/table.go | 8 +- sei-db/db_engine/litt/test/table_test.go | 131 ++++++++++++++++++ 3 files changed, 181 insertions(+), 16 deletions(-) diff --git a/sei-db/db_engine/litt/dbcache/cached_table.go b/sei-db/db_engine/litt/dbcache/cached_table.go index 0f269b7d89..100cf77a49 100644 --- a/sei-db/db_engine/litt/dbcache/cached_table.go +++ b/sei-db/db_engine/litt/dbcache/cached_table.go @@ -115,30 +115,51 @@ func (c *cachedTable) Get(key []byte) (value []byte, exists bool, err error) { // GetSubrange reads only the [offset, offset+length) byte range of the value stored under key. // -// Cache policy — subrange reads intentionally bypass BOTH caches: +// Cache policy — subrange reads consult both caches, but never populate them: // -// - The read and write caches are keyed by the full key and store the full value. A subrange read cannot -// safely populate them (it would store a partial value under a key that Get is entitled to treat as the -// whole value), and it does not consult them either: even on a cache hit the whole point of a subrange -// read is to avoid materializing the full value, and slicing a cached full value would defeat the I/O -// savings the base implementation provides. -// - A more elaborate scheme could cache sub-ranges under specially structured keys, but that adds -// complexity we do not yet know we need. We start simple by skipping the cache and documenting it here, -// with the intent to revisit if profiling shows subrange reads are hot enough that the missing cache -// hurts. Until then, every GetSubrange goes straight to the base table. +// - Populating would be unsafe. Both caches are keyed by the full key and hold the full value, so +// storing a sub-range under that key would hand a later Get a partial value it is entitled to treat +// as the whole thing. +// - Consulting is safe, because every path agrees that a key's offsets are relative to that key's own +// logical value. Put stores value[sk.Offset:sk.Offset+sk.Length] under a secondary key, the base table +// addresses that secondary at the same aliased region on disk, and readCache holds whatever Get +// returned. A cache hit therefore slices exactly the bytes the base table would have read. +// - Consulting is also strictly cheaper. A hit means the value is already materialized in memory, so +// slicing it costs no I/O at all, where the base table would pay a keymap lookup plus a disk read. +// Bypassing the caches would make GetSubrange slower than plain Get for precisely the hot, +// recently-written values a subrange read is meant to serve. // -// This still reports the read to metrics (as a cold read), for parity with Get. +// Because hits come from the same caches Get uses, the two agree on whether a key exists — including for +// a key already reclaimed from the base table by GC or TTL expiry but still resident in a cache. +// +// Caching sub-ranges themselves, under specially structured keys, would speed up misses as well, but that +// adds complexity we do not yet know we need. Revisit if profiling shows subrange misses are hot enough to +// justify it. func (c *cachedTable) GetSubrange(key []byte, offset uint32, length uint32) (value []byte, exists bool, err error) { + // hot tracks whether the value was served from one of this table's caches (a "hot" read) for metrics. + var hot bool if c.metrics != nil { start := time.Now() defer func() { if exists && value != nil { - // hot is always false: subrange reads never come from a cache (see the cache policy above). - c.metrics.ReportReadOperation(c.Name(), time.Since(start), uint64(len(value)), false) + c.metrics.ReportReadOperation(c.Name(), time.Since(start), uint64(len(value)), hot) } }() } + stringKey := util.UnsafeBytesToString(key) + + if cached, ok := c.writeCache.Get(stringKey); ok { + // The value was recently written. + hot = true + return subrangeOf(cached, offset, length) + } + if cached, ok := c.readCache.Get(stringKey); ok { + // The value was recently read in full. + hot = true + return subrangeOf(cached, offset, length) + } + value, exists, err = c.base.GetSubrange(key, offset, length) if err != nil { return value, exists, fmt.Errorf("failed to get subrange from base table: %w", err) @@ -146,6 +167,17 @@ func (c *cachedTable) GetSubrange(key []byte, offset uint32, length uint32) (val return value, exists, nil } +// subrangeOf slices a cached full value, applying the same bounds check (and reporting it the same way) +// as the base table, so a cache hit and a cache miss are indistinguishable to the caller. +func subrangeOf(value []byte, offset uint32, length uint32) ([]byte, bool, error) { + end := uint64(offset) + uint64(length) + if end > uint64(len(value)) { + return nil, false, fmt.Errorf( + "subrange [%d, %d) is out of bounds for value of length %d", offset, end, len(value)) + } + return value[offset:end], true, nil +} + func (c *cachedTable) Exists(key []byte) (exists bool, err error) { _, exists = c.writeCache.Get(util.UnsafeBytesToString(key)) if exists { diff --git a/sei-db/db_engine/litt/table.go b/sei-db/db_engine/litt/table.go index a0c895ca1b..d28312d41d 100644 --- a/sei-db/db_engine/litt/table.go +++ b/sei-db/db_engine/litt/table.go @@ -93,9 +93,11 @@ type Table interface { // The requested range must be within the value: if offset+length exceeds the value's length, an error // is returned. A zero-length range is valid and returns an empty (non-nil) slice when the key exists. // - // Caching note: unlike Get, GetSubrange deliberately does not consult or populate the read/write value - // caches (see the cachedTable implementation for the rationale). Every GetSubrange therefore reaches the - // base table. + // Caching note: for a table with caching enabled, GetSubrange consults the read/write value caches + // like Get does — a cached value is already in memory, so slicing it is cheaper than any disk read — + // but unlike Get it never populates them, since a sub-range stored under the full key would corrupt a + // later Get. A subrange read of an uncached key therefore always reaches the base table (see the + // cachedTable implementation for the full rationale). // // As with Get, the returned data is NOT safe to mutate, and the key byte slice must not be modified // after it is passed to this method. diff --git a/sei-db/db_engine/litt/test/table_test.go b/sei-db/db_engine/litt/test/table_test.go index 3ce6a906c4..20c8f05fb6 100644 --- a/sei-db/db_engine/litt/test/table_test.go +++ b/sei-db/db_engine/litt/test/table_test.go @@ -565,3 +565,134 @@ func TestSecondaryKeyBasics(t *testing.T) { }) } } + +// getSubrangeParityTest runs against every implementation, cached and uncached, and pins the property +// that makes it safe for a cached table to serve a subrange read by slicing a cached value: the read is +// indistinguishable from the same read against the base table. It covers a primary key, a secondary key +// (whose offsets are relative to its own aliased region, not to the parent value), out-of-bounds ranges, +// and the requirement that a subrange read never poisons a later full Get. +func getSubrangeParityTest(t *testing.T, tb *tableBuilder) { + rand := util.NewTestRandom() + directory := t.TempDir() + tableName := rand.String(8) + table, err := tb.builder(time.Now, tableName, directory) + require.NoError(t, err) + + // 0 1 + // 0123456789012345678 + value := []byte("the quick brown fox") + primary := []byte("primary") + // A secondary aliasing the strict sub-range "brown fox". Its cache entry holds only those bytes, and + // its on-disk address points at the same region, so both paths measure offsets from "brown". + sk := &types.SecondaryKey{Key: []byte("brown-fox"), Offset: 10, Length: 9} + require.NoError(t, table.Put(primary, value, sk)) + + verify := func(stage string) { + t.Helper() + + got, ok, err := table.GetSubrange(primary, 4, 5) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, []byte("quick"), got, stage) + + got, ok, err = table.GetSubrange(sk.Key, 6, 3) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, []byte("fox"), got, stage) + + // A zero-length range is valid and yields an empty, non-nil slice. + got, ok, err = table.GetSubrange(primary, uint32(len(value)), 0) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.NotNil(t, got, stage) + require.Empty(t, got, stage) + + // A range past the end of the value errors, for the primary against the whole value and for the + // secondary against its (shorter) aliased region. + _, _, err = table.GetSubrange(primary, uint32(len(value)), 1) + require.Error(t, err, stage) + _, _, err = table.GetSubrange(sk.Key, 0, sk.Length+1) + require.Error(t, err, stage) + + // A subrange read must never be cached under the key it was read from: a later Get still sees + // the whole value. + got, ok, err = table.Get(primary) + require.NoError(t, err, stage) + require.True(t, ok, stage) + require.Equal(t, value, got, stage) + } + + verify("before flush") + require.NoError(t, table.Flush()) + verify("after flush") + + require.NoError(t, table.Drop()) +} + +func TestGetSubrangeParity(t *testing.T) { + t.Parallel() + for _, tb := range tableBuilders { + tb := tb + t.Run(tb.name, func(t *testing.T) { + t.Parallel() + getSubrangeParityTest(t, tb) + }) + } +} + +// TestGetSubrangeServedFromCache proves that a cached table actually consults its caches rather than +// always reaching the base table, by seeding a cache entry for a key the base table has never heard of: +// only a cache hit can answer it. This is also the shape of the case that matters in production — a key +// still resident in a cache after the base table reclaimed it via GC or TTL expiry — and it shows that +// GetSubrange and Get agree about such a key existing. +func TestGetSubrangeServedFromCache(t *testing.T) { + t.Parallel() + + cacheWeight := func(k string, v []byte) uint64 { return uint64(len(k) + len(v)) } + + for _, cacheName := range []string{"write cache", "read cache"} { + cacheName := cacheName + t.Run(cacheName, func(t *testing.T) { + t.Parallel() + rand := util.NewTestRandom() + directory := t.TempDir() + tableName := rand.String(8) + + base, err := buildMemKeyDiskTable(time.Now, tableName, directory) + require.NoError(t, err) + + writeCache := util.NewFIFOCache[string, []byte](500, cacheWeight, nil) + readCache := util.NewFIFOCache[string, []byte](500, cacheWeight, nil) + table := dbcache.NewCachedTable(base, writeCache, readCache, nil) + + // Seed only the cache under test. The base table is left empty, so any answer for this key + // must have come from that cache. + value := []byte("the quick brown fox") + key := "cache-only" + if cacheName == "write cache" { + writeCache.Put(key, value) + } else { + readCache.Put(key, value) + } + + got, ok, err := table.GetSubrange([]byte(key), 4, 5) + require.NoError(t, err) + require.True(t, ok, "a cache-resident key must be served from the cache") + require.Equal(t, []byte("quick"), got) + + // Bounds are checked against the cached value's length, matching the base table's behavior. + _, _, err = table.GetSubrange([]byte(key), uint32(len(value)), 1) + require.Error(t, err) + + // GetSubrange and Get agree that the key exists, and the base table still does not have it. + _, ok, err = table.Get([]byte(key)) + require.NoError(t, err) + require.True(t, ok) + _, ok, err = base.GetSubrange([]byte(key), 4, 5) + require.NoError(t, err) + require.False(t, ok, "the base table was never written to") + + require.NoError(t, table.Drop()) + }) + } +} From 5f6715fc186c456bbfeaf2fc723b32d5ca71959c Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 27 Jul 2026 13:09:41 -0700 Subject: [PATCH 4/8] Fix some edge case --- .../block/littblock/litt_block_db.go | 29 ++++++++++++++++--- .../block/littblock/litt_block_db_test.go | 26 ++++++++++------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index a8a54132db..11c951346b 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -431,14 +431,24 @@ func (s *blockDB) Close() error { // GetTxByOffset reads a single transaction's raw bytes out of the block stored at GlobalBlockNumber n, // without decoding the block. offset and length identify a byte range within the block's stored value — // the exact bytes encodeBlock produced: [version:1][GlobalBlockNumber:8][proto(Block)] — typically the -// (offset, length) of one transaction recorded at write time (see the tx-location-index design under -// sei-db/ledger_db/receipt/docs). offset is measured from the start of that stored value (i.e. it includes +// (offset, length) of one transaction recorded at write time. Offset is measured from the start of that stored value (i.e. it includes // the fixed prefix), matching the coordinates a writer computes against the bytes it persists. // // It reads only the requested bytes via LittDB's GetSubrange, skipping both the full-block disk read and // the protobuf unmarshal that ReadBlockByNumber performs. The I/O savings apply while the ledger table is // uncompressed (the default); on a compressed table GetSubrange still returns the correct bytes but must -// read and decompress the whole block value first (see the compression note in the design doc). +// read and decompress the whole block value first. +// +// The returned bytes are NOT safe to mutate. Unlike ReadBlockByNumber, which hands back a freshly decoded +// block, this returns a slice aliasing LittDB's internal memory: before a flush it is a sub-slice of the +// live unflushed entry holding the whole block value, and on a cache hit a sub-slice of the cached copy. +// Writing through it would corrupt the block for every later reader. Copy the bytes first if they must be +// modified or must outlive the read. +// +// A recorded (offset, length) is only meaningful against the block value it was computed from. The pair +// and the value are written together and neither moves afterward, so a later bump to +// blockSerializationVersion cannot retroactively shift an already-recorded pair; what it would break is +// anything that rewrites stored block values in place, which must re-record the offsets pointing into them. // // This lives outside the types.BlockDB interface because the offset space is defined by this // implementation's serialization. The result is one of: @@ -447,12 +457,23 @@ func (s *blockDB) Close() error { // - types.ErrPruned: n is strictly below the retention watermark (matches ReadBlockByNumber). A block // below the watermark may be stranded from its covering QC and is never served. // - None with a nil error: no block is present at n (never written, or not yet written). -// - a non-nil error: the range is out of bounds for the block value, or the read failed. +// - a non-nil error: offset lands inside the fixed prefix, the range is out of bounds for the block +// value, or the read failed. func (s *blockDB) GetTxByOffset( n types.GlobalBlockNumber, offset uint32, length uint32, ) (utils.Option[[]byte], error) { + // A transaction lives in the proto body, never in the fixed prefix, so an offset inside the prefix + // means the caller measured against the wrong frame — most likely the body instead of the whole stored + // value. Reject it rather than return plausible-looking but shifted bytes. A change to the value + // layout has to revisit this bound along with encodeBlock/decodeBlock. + if offset < blockValuePrefixLen { + return utils.None[[]byte](), fmt.Errorf( + "tx offset %d is inside the %d-byte block value prefix: offsets are measured from the start of "+ + "the stored value, not of the proto body", offset, blockValuePrefixLen) + } + // Refuse below-watermark blocks: they may be stranded (covering QC reclaimed). Mirrors ReadBlockByNumber. if uint64(n) < s.watermark.Load() { return utils.None[[]byte](), types.ErrPruned diff --git a/sei-db/ledger_db/block/littblock/litt_block_db_test.go b/sei-db/ledger_db/block/littblock/litt_block_db_test.go index 8b0e787471..51f80721cd 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db_test.go @@ -48,33 +48,39 @@ func TestGetTxByOffset(t *testing.T) { verify := func(stage string) { t.Helper() - // The full range round-trips the whole stored value. - res, err := impl.GetTxByOffset(0, 0, storedLen) + // The whole payload following the fixed prefix round-trips. + res, err := impl.GetTxByOffset(0, blockValuePrefixLen, storedLen-blockValuePrefixLen) require.NoError(t, err, stage) got, ok := res.Get() require.True(t, ok, stage) - require.Equal(t, stored, got, stage) + require.Equal(t, stored[blockValuePrefixLen:], got, stage) - // A real transaction's raw bytes appear verbatim as a contiguous run in the value (each tx is a + // A real transaction's raw bytes appear verbatim as a contiguous run in the payload (each tx is a // length-delimited `repeated bytes` element), so locating one gives a valid (offset, length) — - // exactly what a writer would record. Extracting that range must return the transaction. + // exactly what a writer would record. Extracting that range must return the transaction. The + // search is scoped to the payload because that is the only place a tx can legitimately live. for _, tx := range blk.Payload().Txs() { - idx := bytes.Index(stored, tx) + idx := bytes.Index(stored[blockValuePrefixLen:], tx) require.GreaterOrEqual(t, idx, 0, stage) //nolint:gosec // small test offsets/lengths fit u32 - res, err := impl.GetTxByOffset(0, uint32(idx), uint32(len(tx))) + res, err := impl.GetTxByOffset(0, uint32(blockValuePrefixLen+idx), uint32(len(tx))) require.NoError(t, err, stage) got, ok := res.Get() require.True(t, ok, stage) require.Equal(t, tx, got, stage) } + // An offset inside the fixed prefix is rejected: no transaction can start there, so the caller + // must have measured against the proto body rather than the whole stored value. + _, err = impl.GetTxByOffset(0, blockValuePrefixLen-1, 1) + require.Error(t, err, stage) + // A range past the end of the value is an error. _, err = impl.GetTxByOffset(0, storedLen-1, 5) require.Error(t, err, stage) // A block that was never written is simply absent (not an error). - res, err = impl.GetTxByOffset(1, 0, 1) + res, err = impl.GetTxByOffset(1, blockValuePrefixLen, 1) require.NoError(t, err, stage) require.False(t, res.IsPresent(), stage) } @@ -98,12 +104,12 @@ func TestGetTxByOffsetPruned(t *testing.T) { writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QCs [0,5),[5,10),[10,15),[15,20) require.NoError(t, db.PruneBefore(5)) // watermark to 5: blocks 0..4 are below it - res, err := impl.GetTxByOffset(2, 0, 1) + res, err := impl.GetTxByOffset(2, blockValuePrefixLen, 1) require.ErrorIs(t, err, types.ErrPruned) require.False(t, res.IsPresent()) // A block at/above the watermark is still served. - res, err = impl.GetTxByOffset(5, 0, 1) + res, err = impl.GetTxByOffset(5, blockValuePrefixLen, 1) require.NoError(t, err) require.True(t, res.IsPresent()) } From 6eae178711f24b9b85f8c975b38341bb70a9d195 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Mon, 27 Jul 2026 13:58:12 -0700 Subject: [PATCH 5/8] Address comments --- sei-db/db_engine/litt/dbcache/cached_table.go | 5 ++++- sei-db/db_engine/litt/disktable/disk_table.go | 5 ++++- .../litt/disktable/disk_table_subrange_test.go | 3 +++ sei-db/db_engine/litt/disktable/segment/segment.go | 6 +++++- sei-db/db_engine/litt/table.go | 5 ++++- sei-db/db_engine/litt/test/table_test.go | 10 ++++++++++ 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/sei-db/db_engine/litt/dbcache/cached_table.go b/sei-db/db_engine/litt/dbcache/cached_table.go index 100cf77a49..cccfc02f93 100644 --- a/sei-db/db_engine/litt/dbcache/cached_table.go +++ b/sei-db/db_engine/litt/dbcache/cached_table.go @@ -175,7 +175,10 @@ func subrangeOf(value []byte, offset uint32, length uint32) ([]byte, bool, error return nil, false, fmt.Errorf( "subrange [%d, %d) is out of bounds for value of length %d", offset, end, len(value)) } - return value[offset:end], true, nil + // Capped (three-index) slice: without it the sub-range would carry spare capacity reaching into the + // rest of the cached value, and an append by the caller would silently overwrite the bytes that + // follow — corrupting the entry for every later reader. Capping forces such an append to allocate. + return value[offset:end:end], true, nil } func (c *cachedTable) Exists(key []byte) (exists bool, err error) { diff --git a/sei-db/db_engine/litt/disktable/disk_table.go b/sei-db/db_engine/litt/disktable/disk_table.go index c2697b944d..5b5d8ead12 100644 --- a/sei-db/db_engine/litt/disktable/disk_table.go +++ b/sei-db/db_engine/litt/disktable/disk_table.go @@ -997,7 +997,10 @@ func (d *DiskTable) GetSubrange(key []byte, offset uint32, length uint32) (value return nil, false, fmt.Errorf( "subrange [%d, %d) is out of bounds for value of length %d", offset, end, len(full)) } - return full[offset:end], true, nil + // Capped (three-index) slice: without it the sub-range would carry spare capacity reaching into + // the rest of the live unflushed value, and an append by the caller would silently overwrite the + // bytes that follow. Capping forces such an append to allocate instead. + return full[offset:end:end], true, nil } // Look up the address of the data. diff --git a/sei-db/db_engine/litt/disktable/disk_table_subrange_test.go b/sei-db/db_engine/litt/disktable/disk_table_subrange_test.go index 999d207d94..44aa5e8422 100644 --- a/sei-db/db_engine/litt/disktable/disk_table_subrange_test.go +++ b/sei-db/db_engine/litt/disktable/disk_table_subrange_test.go @@ -59,6 +59,9 @@ func TestGetSubrange(t *testing.T) { require.True(t, ok, stage) require.NotNil(t, got, stage) require.Equal(t, value[r.off:r.off+r.length], got, stage) + // The sub-range may alias the whole value in memory, so its capacity must stop at its + // length: otherwise an append by the caller would overwrite the bytes that follow. + require.Equal(t, len(got), cap(got), stage) } // A sub-range read of a secondary key stays within the secondary's aliased region. diff --git a/sei-db/db_engine/litt/disktable/segment/segment.go b/sei-db/db_engine/litt/disktable/segment/segment.go index 6b5713b798..c3b68c5740 100644 --- a/sei-db/db_engine/litt/disktable/segment/segment.go +++ b/sei-db/db_engine/litt/disktable/segment/segment.go @@ -739,7 +739,11 @@ func (s *Segment) ReadSubrange(key []byte, dataAddress types.Address, offset uin return nil, fmt.Errorf("subrange [%d, %d) is out of bounds for value of length %d", offset, end, len(value)) } - return value[offset:end], nil + // Capped (three-index) slice, so that every GetSubrange path returns a sub-range whose capacity + // stops at its length. The decompression buffer sliced here is not shared with another reader + // today, but keeping the guarantee uniform means callers never have to know which path served + // them before deciding whether an append is safe. + return value[offset:end:end], nil } // For an uncompressed value, dataAddress.ValueSize() is the exact value length, so we can bound the diff --git a/sei-db/db_engine/litt/table.go b/sei-db/db_engine/litt/table.go index d28312d41d..c802bc21ca 100644 --- a/sei-db/db_engine/litt/table.go +++ b/sei-db/db_engine/litt/table.go @@ -100,7 +100,10 @@ type Table interface { // cachedTable implementation for the full rationale). // // As with Get, the returned data is NOT safe to mutate, and the key byte slice must not be modified - // after it is passed to this method. + // after it is passed to this method. The returned slice may alias internal memory holding the whole + // value (an unflushed write or a cache entry), but its capacity is always capped to its length, so an + // append allocates a new array rather than overwriting whatever follows the range in that shared + // buffer. GetSubrange(key []byte, offset uint32, length uint32) (value []byte, exists bool, err error) // Exists returns true if the key exists in the database, and false otherwise. This is faster than calling Get. diff --git a/sei-db/db_engine/litt/test/table_test.go b/sei-db/db_engine/litt/test/table_test.go index 20c8f05fb6..71637eead1 100644 --- a/sei-db/db_engine/litt/test/table_test.go +++ b/sei-db/db_engine/litt/test/table_test.go @@ -594,11 +594,15 @@ func getSubrangeParityTest(t *testing.T, tb *tableBuilder) { require.NoError(t, err, stage) require.True(t, ok, stage) require.Equal(t, []byte("quick"), got, stage) + // A hit slices a buffer holding the whole value (an unflushed write, or a cache entry), so the + // capacity must stop at the length or an append by the caller would corrupt the rest of it. + require.Equal(t, len(got), cap(got), stage) got, ok, err = table.GetSubrange(sk.Key, 6, 3) require.NoError(t, err, stage) require.True(t, ok, stage) require.Equal(t, []byte("fox"), got, stage) + require.Equal(t, len(got), cap(got), stage) // A zero-length range is valid and yields an empty, non-nil slice. got, ok, err = table.GetSubrange(primary, uint32(len(value)), 0) @@ -680,6 +684,12 @@ func TestGetSubrangeServedFromCache(t *testing.T) { require.True(t, ok, "a cache-resident key must be served from the cache") require.Equal(t, []byte("quick"), got) + // The slice aliases the cache entry, so its capacity must stop at its length: an append by + // the caller must not be able to write into the cached value every later reader shares. + require.Equal(t, len(got), cap(got)) + got = append(got, "-appended"...) //nolint:gocritic // the point is that this must not alias + require.Equal(t, []byte("the quick brown fox"), value, "append must not touch the cached value") + // Bounds are checked against the cached value's length, matching the base table's behavior. _, _, err = table.GetSubrange([]byte(key), uint32(len(value)), 1) require.Error(t, err) From 8d8f4b35b73844fe26ad67374934d18051962530 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Tue, 28 Jul 2026 10:26:17 -0700 Subject: [PATCH 6/8] Address comments --- sei-db/db_engine/litt/README.md | 1 + sei-db/db_engine/litt/test/table_test.go | 45 ++++++++++++++++++- .../block/littblock/litt_block_db.go | 12 ++--- .../block/littblock/litt_block_db_test.go | 6 +++ 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/sei-db/db_engine/litt/README.md b/sei-db/db_engine/litt/README.md index ecb3109b71..7587702e4c 100644 --- a/sei-db/db_engine/litt/README.md +++ b/sei-db/db_engine/litt/README.md @@ -121,6 +121,7 @@ type Table interface { Put(key []byte, value []byte, secondaryKeys ...*types.SecondaryKey) error PutBatch(batch []*types.PutRequest) error Get(key []byte) ([]byte, bool, error) + GetSubrange(key []byte, offset uint32, length uint32) ([]byte, bool, error) Exists(key []byte) (bool, error) Flush() error Size() uint64 diff --git a/sei-db/db_engine/litt/test/table_test.go b/sei-db/db_engine/litt/test/table_test.go index 71637eead1..2d271002e1 100644 --- a/sei-db/db_engine/litt/test/table_test.go +++ b/sei-db/db_engine/litt/test/table_test.go @@ -201,6 +201,31 @@ func buildCachedMemKeyDiskTable( return dbcache.NewCachedTable(baseTable, writeCache, readCache, nil), nil } +// buildCacheDisabledMemKeyDiskTable wraps a base table in cachedTable with caches that can never hold an +// entry (max weight 0, which is what DefaultTableConfig configures and therefore what every table in this +// repo runs with today). Every read must fall through to the base table, so this covers the wrapper's +// delegation path — which the builders above cannot, since their caches are large enough that a written +// value is always still resident. +func buildCacheDisabledMemKeyDiskTable( + clock func() time.Time, + name string, + path string) (litt.ManagedTable, error) { + + baseTable, err := buildMemKeyDiskTable(clock, name, path) + if err != nil { + return nil, err + } + + writeCache := util.NewFIFOCache[string, []byte](0, func(k string, v []byte) uint64 { + return uint64(len(k) + len(v)) + }, nil) + readCache := util.NewFIFOCache[string, []byte](0, func(k string, v []byte) uint64 { + return uint64(len(k) + len(v)) + }, nil) + + return dbcache.NewCachedTable(baseTable, writeCache, readCache, nil), nil +} + func buildCachedPebbleDBKeyDiskTable( clock func() time.Time, name string, @@ -618,6 +643,13 @@ func getSubrangeParityTest(t *testing.T, tb *tableBuilder) { _, _, err = table.GetSubrange(sk.Key, 0, sk.Length+1) require.Error(t, err, stage) + // A key that was never written misses both caches and must report not-found from the base table, + // without an error. + got, ok, err = table.GetSubrange([]byte("does-not-exist"), 0, 1) + require.NoError(t, err, stage) + require.False(t, ok, stage) + require.Nil(t, got, stage) + // A subrange read must never be cached under the key it was read from: a later Get still sees // the whole value. got, ok, err = table.Get(primary) @@ -635,7 +667,18 @@ func getSubrangeParityTest(t *testing.T, tb *tableBuilder) { func TestGetSubrangeParity(t *testing.T) { t.Parallel() - for _, tb := range tableBuilders { + + // tableBuilders' cached entries hold every written value for the life of the test, so on their own + // they only ever exercise cache hits. Add a cache-disabled wrapper so the same assertions also run + // against the wrapper's fall-through to the base table. + builders := make([]*tableBuilder, 0, len(tableBuilders)+1) + builders = append(builders, tableBuilders...) + builders = append(builders, &tableBuilder{ + "cache-disabled mem keymap disk table", + buildCacheDisabledMemKeyDiskTable, + }) + + for _, tb := range builders { tb := tb t.Run(tb.name, func(t *testing.T) { t.Parallel() diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 11c951346b..3d0469d1bc 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -464,6 +464,13 @@ func (s *blockDB) GetTxByOffset( offset uint32, length uint32, ) (utils.Option[[]byte], error) { + // Refuse below-watermark blocks first: they may be stranded (covering QC reclaimed). Mirrors + // ReadBlockByNumber, and keeps ErrPruned the outcome a caller observes for a pruned n regardless of + // whether the offset is also malformed. + if uint64(n) < s.watermark.Load() { + return utils.None[[]byte](), types.ErrPruned + } + // A transaction lives in the proto body, never in the fixed prefix, so an offset inside the prefix // means the caller measured against the wrong frame — most likely the body instead of the whole stored // value. Reject it rather than return plausible-looking but shifted bytes. A change to the value @@ -474,11 +481,6 @@ func (s *blockDB) GetTxByOffset( "the stored value, not of the proto body", offset, blockValuePrefixLen) } - // Refuse below-watermark blocks: they may be stranded (covering QC reclaimed). Mirrors ReadBlockByNumber. - if uint64(n) < s.watermark.Load() { - return utils.None[[]byte](), types.ErrPruned - } - value, exists, err := s.table.GetSubrange(blockKey(n), offset, length) if err != nil { return utils.None[[]byte](), fmt.Errorf( diff --git a/sei-db/ledger_db/block/littblock/litt_block_db_test.go b/sei-db/ledger_db/block/littblock/litt_block_db_test.go index 51f80721cd..1b1e9daf72 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db_test.go @@ -108,6 +108,12 @@ func TestGetTxByOffsetPruned(t *testing.T) { require.ErrorIs(t, err, types.ErrPruned) require.False(t, res.IsPresent()) + // Retention wins over argument shape: a below-watermark block asked for with an offset inside the + // prefix still returns ErrPruned, matching ReadBlockByNumber and the documented contract. + res, err = impl.GetTxByOffset(2, 0, 1) + require.ErrorIs(t, err, types.ErrPruned) + require.False(t, res.IsPresent()) + // A block at/above the watermark is still served. res, err = impl.GetTxByOffset(5, blockValuePrefixLen, 1) require.NoError(t, err) From 78ad13da4ad9a3035891cf3c38a9f15faddd3b91 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Tue, 28 Jul 2026 10:56:54 -0700 Subject: [PATCH 7/8] Address comments --- .../block/littblock/litt_block_db.go | 43 +++++++------- .../block/littblock/litt_block_db_test.go | 57 +++++++++++-------- 2 files changed, 58 insertions(+), 42 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index 3d0469d1bc..f3a22848ff 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -2,6 +2,7 @@ package littblock import ( "fmt" + "math" "sync" "sync/atomic" @@ -429,10 +430,14 @@ func (s *blockDB) Close() error { } // GetTxByOffset reads a single transaction's raw bytes out of the block stored at GlobalBlockNumber n, -// without decoding the block. offset and length identify a byte range within the block's stored value — -// the exact bytes encodeBlock produced: [version:1][GlobalBlockNumber:8][proto(Block)] — typically the -// (offset, length) of one transaction recorded at write time. Offset is measured from the start of that stored value (i.e. it includes -// the fixed prefix), matching the coordinates a writer computes against the bytes it persists. +// without decoding the block. offset and length identify a byte range within the block's marshalled body — +// the proto(Block) bytes alone — typically the (offset, length) of one transaction recorded at write time. +// +// Offsets are body-relative, not value-relative: encodeBlock frames the stored value as +// [version:1][GlobalBlockNumber:8][proto(Block)], and this method adds that fixed prefix itself. A writer +// locating each transaction inside the marshalled body therefore records exactly what it computed, with no +// conversion step to forget, and the framing stays a private detail of this package rather than something +// baked into every offset persisted elsewhere (e.g. a receipt's stored tx location). // // It reads only the requested bytes via LittDB's GetSubrange, skipping both the full-block disk read and // the protobuf unmarshal that ReadBlockByNumber performs. The I/O savings apply while the ledger table is @@ -445,10 +450,11 @@ func (s *blockDB) Close() error { // Writing through it would corrupt the block for every later reader. Copy the bytes first if they must be // modified or must outlive the read. // -// A recorded (offset, length) is only meaningful against the block value it was computed from. The pair -// and the value are written together and neither moves afterward, so a later bump to -// blockSerializationVersion cannot retroactively shift an already-recorded pair; what it would break is -// anything that rewrites stored block values in place, which must re-record the offsets pointing into them. +// A recorded (offset, length) is meaningful only against the marshalled body it was computed from. Being +// body-relative, it survives a change to the value framing — the prefix is applied here, at read time, +// from the same constant encodeBlock uses — but not a change to how the body itself is marshalled, which +// moves the transactions within it. Should the prefix width ever vary by block version, this method is +// where the block's version would have to be resolved to pick the right one. // // This lives outside the types.BlockDB interface because the offset space is defined by this // implementation's serialization. The result is one of: @@ -457,8 +463,8 @@ func (s *blockDB) Close() error { // - types.ErrPruned: n is strictly below the retention watermark (matches ReadBlockByNumber). A block // below the watermark may be stranded from its covering QC and is never served. // - None with a nil error: no block is present at n (never written, or not yet written). -// - a non-nil error: offset lands inside the fixed prefix, the range is out of bounds for the block -// value, or the read failed. +// - a non-nil error: offset is too large to be prefixed without overflowing, the range is out of bounds +// for the block value, or the read failed. func (s *blockDB) GetTxByOffset( n types.GlobalBlockNumber, offset uint32, @@ -471,20 +477,19 @@ func (s *blockDB) GetTxByOffset( return utils.None[[]byte](), types.ErrPruned } - // A transaction lives in the proto body, never in the fixed prefix, so an offset inside the prefix - // means the caller measured against the wrong frame — most likely the body instead of the whole stored - // value. Reject it rather than return plausible-looking but shifted bytes. A change to the value - // layout has to revisit this bound along with encodeBlock/decodeBlock. - if offset < blockValuePrefixLen { + // Translate the body-relative offset into the stored value's frame. The addition is checked: an offset + // near the top of the uint32 range would otherwise wrap to a small value and quietly read bytes from + // the start of the block instead of failing. + if offset > math.MaxUint32-blockValuePrefixLen { return utils.None[[]byte](), fmt.Errorf( - "tx offset %d is inside the %d-byte block value prefix: offsets are measured from the start of "+ - "the stored value, not of the proto body", offset, blockValuePrefixLen) + "tx offset %d is too large to address within a block value", offset) } + valueOffset := offset + blockValuePrefixLen - value, exists, err := s.table.GetSubrange(blockKey(n), offset, length) + value, exists, err := s.table.GetSubrange(blockKey(n), valueOffset, length) if err != nil { return utils.None[[]byte](), fmt.Errorf( - "failed to read tx range [%d, %d) in block %d: %w", offset, uint64(offset)+uint64(length), n, err) + "failed to read tx range [%d, %d) of block %d's body: %w", offset, uint64(offset)+uint64(length), n, err) } if !exists { return utils.None[[]byte](), nil diff --git a/sei-db/ledger_db/block/littblock/litt_block_db_test.go b/sei-db/ledger_db/block/littblock/litt_block_db_test.go index 1b1e9daf72..bf2daba1bd 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db_test.go @@ -2,6 +2,7 @@ package littblock import ( "bytes" + "math" "testing" "github.com/stretchr/testify/require" @@ -40,47 +41,57 @@ func TestGetTxByOffset(t *testing.T) { require.NoError(t, db.WriteQC(0, 1, types.GenFullCommitQCRange(rng, 0, 1))) require.NoError(t, db.WriteBlock(0, blk)) - // The stored value for the primary block key is exactly encodeBlock's output; GetTxByOffset returns a - // byte range of that value. + // The stored value for the primary block key is exactly encodeBlock's output. Offsets passed to + // GetTxByOffset are relative to the marshalled body, i.e. the stored value minus its fixed prefix. stored := encodeBlock(0, blk) - storedLen := uint32(len(stored)) + body := stored[blockValuePrefixLen:] + bodyLen := uint32(len(body)) verify := func(stage string) { t.Helper() - // The whole payload following the fixed prefix round-trips. - res, err := impl.GetTxByOffset(0, blockValuePrefixLen, storedLen-blockValuePrefixLen) + // The whole body round-trips, proving the method applies the prefix itself. + res, err := impl.GetTxByOffset(0, 0, bodyLen) require.NoError(t, err, stage) got, ok := res.Get() require.True(t, ok, stage) - require.Equal(t, stored[blockValuePrefixLen:], got, stage) + require.Equal(t, body, got, stage) - // A real transaction's raw bytes appear verbatim as a contiguous run in the payload (each tx is a - // length-delimited `repeated bytes` element), so locating one gives a valid (offset, length) — - // exactly what a writer would record. Extracting that range must return the transaction. The - // search is scoped to the payload because that is the only place a tx can legitimately live. + // A real transaction's raw bytes appear verbatim as a contiguous run in the body (each tx is a + // length-delimited `repeated bytes` element), so locating one within the body gives a valid + // (offset, length) — exactly what a writer computes and records, with no prefix arithmetic. + // Extracting that range must return the transaction. for _, tx := range blk.Payload().Txs() { - idx := bytes.Index(stored[blockValuePrefixLen:], tx) + idx := bytes.Index(body, tx) require.GreaterOrEqual(t, idx, 0, stage) //nolint:gosec // small test offsets/lengths fit u32 - res, err := impl.GetTxByOffset(0, uint32(blockValuePrefixLen+idx), uint32(len(tx))) + res, err := impl.GetTxByOffset(0, uint32(idx), uint32(len(tx))) require.NoError(t, err, stage) got, ok := res.Get() require.True(t, ok, stage) require.Equal(t, tx, got, stage) } - // An offset inside the fixed prefix is rejected: no transaction can start there, so the caller - // must have measured against the proto body rather than the whole stored value. - _, err = impl.GetTxByOffset(0, blockValuePrefixLen-1, 1) + // Offset 0 addresses the first body byte, never the version byte: a body-relative read can never + // reach into the prefix. + res, err = impl.GetTxByOffset(0, 0, 1) + require.NoError(t, err, stage) + got, ok = res.Get() + require.True(t, ok, stage) + require.Equal(t, body[:1], got, stage) + + // A range past the end of the body is an error, even though those bytes exist in the stored value + // ahead of the body. + _, err = impl.GetTxByOffset(0, bodyLen-1, 5) require.Error(t, err, stage) - // A range past the end of the value is an error. - _, err = impl.GetTxByOffset(0, storedLen-1, 5) + // An offset too large to be prefixed without wrapping is rejected rather than read from the + // beginning of the block. + _, err = impl.GetTxByOffset(0, math.MaxUint32, 1) require.Error(t, err, stage) // A block that was never written is simply absent (not an error). - res, err = impl.GetTxByOffset(1, blockValuePrefixLen, 1) + res, err = impl.GetTxByOffset(1, 0, 1) require.NoError(t, err, stage) require.False(t, res.IsPresent(), stage) } @@ -104,18 +115,18 @@ func TestGetTxByOffsetPruned(t *testing.T) { writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QCs [0,5),[5,10),[10,15),[15,20) require.NoError(t, db.PruneBefore(5)) // watermark to 5: blocks 0..4 are below it - res, err := impl.GetTxByOffset(2, blockValuePrefixLen, 1) + res, err := impl.GetTxByOffset(2, 0, 1) require.ErrorIs(t, err, types.ErrPruned) require.False(t, res.IsPresent()) - // Retention wins over argument shape: a below-watermark block asked for with an offset inside the - // prefix still returns ErrPruned, matching ReadBlockByNumber and the documented contract. - res, err = impl.GetTxByOffset(2, 0, 1) + // Retention wins over argument shape: a below-watermark block asked for with an unusable offset still + // returns ErrPruned, matching ReadBlockByNumber and the documented contract. + res, err = impl.GetTxByOffset(2, math.MaxUint32, 1) require.ErrorIs(t, err, types.ErrPruned) require.False(t, res.IsPresent()) // A block at/above the watermark is still served. - res, err = impl.GetTxByOffset(5, blockValuePrefixLen, 1) + res, err = impl.GetTxByOffset(5, 0, 1) require.NoError(t, err) require.True(t, res.IsPresent()) } From 219d2592ece46ec5bb0c53e8efca08a4787e14e6 Mon Sep 17 00:00:00 2001 From: YimingZang Date: Tue, 28 Jul 2026 11:07:01 -0700 Subject: [PATCH 8/8] Rename function as suggested --- .../ledger_db/block/littblock/codec_test.go | 22 +++++++++++++ .../block/littblock/litt_block_db.go | 14 +++++--- .../block/littblock/litt_block_db_test.go | 32 +++++++++---------- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/sei-db/ledger_db/block/littblock/codec_test.go b/sei-db/ledger_db/block/littblock/codec_test.go index 3c5aff9eca..26e1735b66 100644 --- a/sei-db/ledger_db/block/littblock/codec_test.go +++ b/sei-db/ledger_db/block/littblock/codec_test.go @@ -82,6 +82,28 @@ func TestBlockRoundTrip(t *testing.T) { } } +// TestBlockValueFraming pins the exact framing of a stored block value: the version byte, the embedded +// block number, and — most importantly — that the marshalled body begins precisely at +// blockValuePrefixLen. ReadBlockSubrange translates body-relative offsets by adding that constant without +// re-reading the value's prefix, so it is correct only while the constant matches the real layout. Any +// change to the framing that leaves blockValuePrefixLen stale fails here, loudly, rather than silently +// shifting every sub-range read. +func TestBlockValueFraming(t *testing.T) { + rng := utils.TestRngFromSeed(3) + for i := range 8 { + n := types.GlobalBlockNumber(i * 1000) + blk := types.GenBlock(rng) + body := types.BlockConv.Marshal(blk) + value := encodeBlock(n, blk) + + require.Len(t, value, blockValuePrefixLen+len(body)) + require.Equal(t, blockSerializationVersion, value[0]) + require.Equal(t, n, decodeKey(value[1:blockValuePrefixLen])) + require.Equal(t, body, value[blockValuePrefixLen:], + "marshalled body must begin exactly at blockValuePrefixLen") + } +} + func TestQCRoundTrip(t *testing.T) { rng := utils.TestRngFromSeed(2) for range 16 { diff --git a/sei-db/ledger_db/block/littblock/litt_block_db.go b/sei-db/ledger_db/block/littblock/litt_block_db.go index f3a22848ff..d694898da0 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db.go @@ -429,9 +429,13 @@ func (s *blockDB) Close() error { return nil } -// GetTxByOffset reads a single transaction's raw bytes out of the block stored at GlobalBlockNumber n, -// without decoding the block. offset and length identify a byte range within the block's marshalled body — -// the proto(Block) bytes alone — typically the (offset, length) of one transaction recorded at write time. +// ReadBlockSubrange reads a byte range out of the block stored at GlobalBlockNumber n, without decoding +// the block. offset and length identify a range within the block's marshalled body — the proto(Block) +// bytes alone — typically the (offset, length) of one transaction recorded at write time. +// +// The range is returned verbatim, with no interpretation: this method does not know or check whether it +// delimits a transaction, a field, or the middle of either. Naming a range is the caller's job, so a +// caller that holds recorded transaction locations is the right place for a tx-shaped wrapper over this. // // Offsets are body-relative, not value-relative: encodeBlock frames the stored value as // [version:1][GlobalBlockNumber:8][proto(Block)], and this method adds that fixed prefix itself. A writer @@ -459,13 +463,13 @@ func (s *blockDB) Close() error { // This lives outside the types.BlockDB interface because the offset space is defined by this // implementation's serialization. The result is one of: // -// - Some(txBytes) with a nil error: the byte range was read. +// - Some(bytes) with a nil error: the byte range was read. // - types.ErrPruned: n is strictly below the retention watermark (matches ReadBlockByNumber). A block // below the watermark may be stranded from its covering QC and is never served. // - None with a nil error: no block is present at n (never written, or not yet written). // - a non-nil error: offset is too large to be prefixed without overflowing, the range is out of bounds // for the block value, or the read failed. -func (s *blockDB) GetTxByOffset( +func (s *blockDB) ReadBlockSubrange( n types.GlobalBlockNumber, offset uint32, length uint32, diff --git a/sei-db/ledger_db/block/littblock/litt_block_db_test.go b/sei-db/ledger_db/block/littblock/litt_block_db_test.go index bf2daba1bd..42e3404ed0 100644 --- a/sei-db/ledger_db/block/littblock/litt_block_db_test.go +++ b/sei-db/ledger_db/block/littblock/litt_block_db_test.go @@ -22,10 +22,10 @@ func genBlockWithTxs(rng utils.Rng) *types.Block { } } -// TestGetTxByOffset covers the block-store sub-range read primitive: it must return exactly the requested -// byte range of a block's stored value (which is what encodeBlock produced), both before and after a flush, -// and it must extract a real transaction's bytes when given that transaction's byte range within the value. -func TestGetTxByOffset(t *testing.T) { +// TestReadBlockSubrange covers the block-store sub-range read primitive: it must return exactly the +// requested byte range of a block's marshalled body, both before and after a flush, and it must yield a +// real transaction's bytes when given that transaction's range within the body. +func TestReadBlockSubrange(t *testing.T) { dir := t.TempDir() rng := utils.TestRngFromSeed(1) @@ -42,7 +42,7 @@ func TestGetTxByOffset(t *testing.T) { require.NoError(t, db.WriteBlock(0, blk)) // The stored value for the primary block key is exactly encodeBlock's output. Offsets passed to - // GetTxByOffset are relative to the marshalled body, i.e. the stored value minus its fixed prefix. + // ReadBlockSubrange are relative to the marshalled body, i.e. the stored value minus its fixed prefix. stored := encodeBlock(0, blk) body := stored[blockValuePrefixLen:] bodyLen := uint32(len(body)) @@ -51,7 +51,7 @@ func TestGetTxByOffset(t *testing.T) { t.Helper() // The whole body round-trips, proving the method applies the prefix itself. - res, err := impl.GetTxByOffset(0, 0, bodyLen) + res, err := impl.ReadBlockSubrange(0, 0, bodyLen) require.NoError(t, err, stage) got, ok := res.Get() require.True(t, ok, stage) @@ -65,7 +65,7 @@ func TestGetTxByOffset(t *testing.T) { idx := bytes.Index(body, tx) require.GreaterOrEqual(t, idx, 0, stage) //nolint:gosec // small test offsets/lengths fit u32 - res, err := impl.GetTxByOffset(0, uint32(idx), uint32(len(tx))) + res, err := impl.ReadBlockSubrange(0, uint32(idx), uint32(len(tx))) require.NoError(t, err, stage) got, ok := res.Get() require.True(t, ok, stage) @@ -74,7 +74,7 @@ func TestGetTxByOffset(t *testing.T) { // Offset 0 addresses the first body byte, never the version byte: a body-relative read can never // reach into the prefix. - res, err = impl.GetTxByOffset(0, 0, 1) + res, err = impl.ReadBlockSubrange(0, 0, 1) require.NoError(t, err, stage) got, ok = res.Get() require.True(t, ok, stage) @@ -82,16 +82,16 @@ func TestGetTxByOffset(t *testing.T) { // A range past the end of the body is an error, even though those bytes exist in the stored value // ahead of the body. - _, err = impl.GetTxByOffset(0, bodyLen-1, 5) + _, err = impl.ReadBlockSubrange(0, bodyLen-1, 5) require.Error(t, err, stage) // An offset too large to be prefixed without wrapping is rejected rather than read from the // beginning of the block. - _, err = impl.GetTxByOffset(0, math.MaxUint32, 1) + _, err = impl.ReadBlockSubrange(0, math.MaxUint32, 1) require.Error(t, err, stage) // A block that was never written is simply absent (not an error). - res, err = impl.GetTxByOffset(1, 0, 1) + res, err = impl.ReadBlockSubrange(1, 0, 1) require.NoError(t, err, stage) require.False(t, res.IsPresent(), stage) } @@ -101,9 +101,9 @@ func TestGetTxByOffset(t *testing.T) { verify("after flush") } -// TestGetTxByOffsetPruned verifies that a block below the retention watermark is reported ErrPruned, +// TestReadBlockSubrangePruned verifies that a block below the retention watermark is reported ErrPruned, // matching ReadBlockByNumber, rather than served (it may be stranded from its covering QC). -func TestGetTxByOffsetPruned(t *testing.T) { +func TestReadBlockSubrangePruned(t *testing.T) { dir := t.TempDir() rng := utils.TestRngFromSeed(2) @@ -115,18 +115,18 @@ func TestGetTxByOffsetPruned(t *testing.T) { writeSyntheticBatches(t, db, rng, 4, 5) // blocks 0..19; QCs [0,5),[5,10),[10,15),[15,20) require.NoError(t, db.PruneBefore(5)) // watermark to 5: blocks 0..4 are below it - res, err := impl.GetTxByOffset(2, 0, 1) + res, err := impl.ReadBlockSubrange(2, 0, 1) require.ErrorIs(t, err, types.ErrPruned) require.False(t, res.IsPresent()) // Retention wins over argument shape: a below-watermark block asked for with an unusable offset still // returns ErrPruned, matching ReadBlockByNumber and the documented contract. - res, err = impl.GetTxByOffset(2, math.MaxUint32, 1) + res, err = impl.ReadBlockSubrange(2, math.MaxUint32, 1) require.ErrorIs(t, err, types.ErrPruned) require.False(t, res.IsPresent()) // A block at/above the watermark is still served. - res, err = impl.GetTxByOffset(5, 0, 1) + res, err = impl.ReadBlockSubrange(5, 0, 1) require.NoError(t, err) require.True(t, res.IsPresent()) }