From 4198af5f23c1b71d793259ebb844087fd09ca4b8 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Mon, 10 Aug 2026 17:21:38 +0200 Subject: [PATCH 1/7] fix(evmrpc): return pruned errors from debug_trace* at unavailable heights (PLT-975) Guard all trace endpoints against block, receipt, and state retention before acquiring the trace semaphore so pruned heights fail fast with explicit errors instead of silent empty results or internal panics. Co-authored-by: Cursor --- evmrpc/historical_debug_trace_test.go | 82 ++++++++++++++++++- evmrpc/trace_db_reader_test.go | 6 +- evmrpc/trace_profile.go | 2 +- evmrpc/tracers.go | 74 ++++++++++------- evmrpc/tracers_semaphore_test.go | 53 +++++++----- evmrpc/utils.go | 3 + evmrpc/watermark_manager.go | 22 +++++ evmrpc/watermark_manager_test.go | 42 ++++++++++ .../ledger_db/receipt/litt_receipt_store.go | 7 +- sei-db/ledger_db/receipt/littidx_test.go | 2 +- sei-db/ledger_db/receipt/receipt_store.go | 6 +- 11 files changed, 240 insertions(+), 59 deletions(-) diff --git a/evmrpc/historical_debug_trace_test.go b/evmrpc/historical_debug_trace_test.go index a9a97d8271..399a2ca427 100644 --- a/evmrpc/historical_debug_trace_test.go +++ b/evmrpc/historical_debug_trace_test.go @@ -3,8 +3,10 @@ package evmrpc import ( "context" "testing" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" "github.com/stretchr/testify/require" ) @@ -93,7 +95,7 @@ func TestGuardHistoricalDebugTraceHeight(t *testing.T) { require.Contains(t, err.Error(), "block number 9 is beyond max lookback of 0") } -func TestGuardHistoricalDebugTraceByHashUsesTendermintHeight(t *testing.T) { +func TestGuardTraceRequestByHashUsesTendermintHeight(t *testing.T) { latestHeight := int64(10) latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) tmClient := newHeightTestClient(8, 1, latestHeight) @@ -107,10 +109,84 @@ func TestGuardHistoricalDebugTraceByHashUsesTendermintHeight(t *testing.T) { }, } - err := api.guardHistoricalDebugTraceByHash(context.Background(), "debug_traceBlockByHash", common.HexToHash(highBlockHashHex)) + err := api.guardTraceRequestByHash(t.Context(), "debug_traceBlockByHash", common.HexToHash(highBlockHashHex)) require.Error(t, err) require.Contains(t, err.Error(), "block number 8 is beyond max lookback of 1") - err = api.guardHistoricalDebugTraceByHash(context.Background(), "debug_traceCall", common.HexToHash("0x1")) + err = api.guardTraceRequestByHash(t.Context(), "debug_traceCall", common.HexToHash("0x1")) require.NoError(t, err) } + +func TestEnsureTraceHeightAvailableReceiptsPruned(t *testing.T) { + t.Parallel() + + client := newHeightTestClient(100, 1, 200) + rs := &fakeReceiptStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(client, watermarkTestCtxProvider(200), nil, rs) + + err := wm.EnsureTraceHeightAvailable(t.Context(), 100) + require.Error(t, err) + require.Contains(t, err.Error(), "receipts have been pruned") +} + +func TestEnsureTraceHeightAvailableStatePruned(t *testing.T) { + t.Parallel() + + client := newHeightTestClient(100, 1, 200) + stateStore := &fakeStateStore{latest: 200, earliest: 150} + rs := &fakeReceiptStore{latest: 200, earliest: 1} + wm := NewWatermarkManager(client, watermarkTestCtxProvider(200), stateStore, rs) + + err := wm.EnsureTraceHeightAvailable(t.Context(), 100) + require.Error(t, err) + require.Contains(t, err.Error(), "has been pruned") +} + +func TestTraceReceiptFloorBoundary(t *testing.T) { + t.Parallel() + + // Receipt retention starts at 150; block 149 still exists in Tendermint but + // its receipts are gone. Trace guard applies at the requested height only — + // parent fetches at height-1 for state replay must not re-run it. + parentHeight := int64(149) + receiptFloor := int64(150) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(parentHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: receiptFloor} + stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), parentHeight), "receipts have been pruned") + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), receiptFloor)) +} + +func TestTraceBlockByNumberReceiptPrunedBeforeSemaphore(t *testing.T) { + t.Parallel() + + prunedHeight := int64(100) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(prunedHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 150} + api := &DebugAPI{ + tmClient: tmClient, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + connectionType: ConnectionTypeHTTP, + maxBlockLookback: -1, + traceCallSemaphore: make(chan struct{}, 1), + traceTimeout: time.Second, + backend: &Backend{ + tmClient: tmClient, + watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs), + }, + } + + api.traceCallSemaphore <- struct{}{} + defer func() { <-api.traceCallSemaphore }() + + _, err := api.TraceBlockByNumber(t.Context(), rpc.BlockNumber(prunedHeight), nil) + require.Error(t, err) + require.Contains(t, err.Error(), "receipts have been pruned") + require.NotErrorIs(t, err, errTraceConcurrencyLimit) +} diff --git a/evmrpc/trace_db_reader_test.go b/evmrpc/trace_db_reader_test.go index b557077821..3252052c5e 100644 --- a/evmrpc/trace_db_reader_test.go +++ b/evmrpc/trace_db_reader_test.go @@ -82,10 +82,10 @@ func TestBlockTraceCacheGet(t *testing.T) { require.Nil(t, got) }) - t.Run("empty block -> empty hit", func(t *testing.T) { + t.Run("empty tx list is not a cache hit", func(t *testing.T) { got, ok := blockTraceCacheGet(c, 5, []common.Hash{}, cfg) - require.True(t, ok) - require.Empty(t, got) + require.False(t, ok, "empty tx list must not report cache hit") + require.Nil(t, got) }) } diff --git a/evmrpc/trace_profile.go b/evmrpc/trace_profile.go index 244836c5c6..2a2df33214 100644 --- a/evmrpc/trace_profile.go +++ b/evmrpc/trace_profile.go @@ -51,7 +51,7 @@ func (api *DebugAPI) TraceTransactionProfile(ctx context.Context, hash common.Ha if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } - if returnErr = api.guardHistoricalDebugTraceByTxHash(ctx, "debug_traceTransactionProfile", hash); returnErr != nil { + if returnErr = api.guardTraceRequestByTxHash(ctx, "debug_traceTransactionProfile", hash); returnErr != nil { return nil, returnErr } diff --git a/evmrpc/tracers.go b/evmrpc/tracers.go index 9afaf9ccfc..9e0b754d37 100644 --- a/evmrpc/tracers.go +++ b/evmrpc/tracers.go @@ -26,6 +26,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/baseapp" "github.com/sei-protocol/sei-chain/sei-cosmos/client" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" "github.com/sei-protocol/sei-chain/x/evm/keeper" "github.com/sei-protocol/sei-chain/x/evm/state" ) @@ -101,44 +102,59 @@ func (api *DebugAPI) prepareTraceContext(ctx context.Context) (context.Context, }, nil } -func (api *DebugAPI) guardHistoricalDebugTraceByTxHash(ctx context.Context, endpoint string, hash common.Hash) error { - if api.keeper == nil { - return nil +func (api *DebugAPI) guardTraceRequest(ctx context.Context, endpoint string, height int64) error { + if api.backend != nil && api.backend.watermarks != nil { + if err := api.backend.watermarks.EnsureTraceHeightAvailable(ctx, height); err != nil { + return err + } } - receipt, err := api.keeper.GetReceipt(api.ctxProvider(LatestCtxHeight), hash) - if err != nil || receipt == nil { - return nil + return api.guardHistoricalDebugTraceHeight(ctx, endpoint, height) +} + +func (api *DebugAPI) guardTraceRequestByTxHash(ctx context.Context, endpoint string, hash common.Hash) error { + if api.keeper != nil { + rcpt, err := api.keeper.GetReceipt(api.ctxProvider(LatestCtxHeight), hash) + if err != nil { + if errors.Is(err, receipt.ErrReceiptPruned) { + return err + } + } else if rcpt != nil { + return api.guardTraceRequest(ctx, endpoint, int64(rcpt.BlockNumber)) //nolint:gosec + } } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, int64(receipt.BlockNumber)) //nolint:gosec + return api.guardHistoricalDebugTraceHeight(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) } -func (api *DebugAPI) guardHistoricalDebugTraceByNumber(ctx context.Context, endpoint string, number rpc.BlockNumber) error { +func (api *DebugAPI) guardTraceRequestByNumber(ctx context.Context, endpoint string, number rpc.BlockNumber) error { height, err := api.resolveDebugTraceBlockNumber(ctx, number) if err != nil { return err } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, height) + return api.guardTraceRequest(ctx, endpoint, height) } -func (api *DebugAPI) guardHistoricalDebugTraceByHash(ctx context.Context, endpoint string, hash common.Hash) error { +func (api *DebugAPI) guardTraceRequestByHash(ctx context.Context, endpoint string, hash common.Hash) error { if api.backend == nil || api.tmClient == nil { - return nil + return api.guardTraceRequest(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) } block, err := blockByHashRespectingWatermarks(ctx, api.tmClient, api.backend.watermarks, hash.Bytes(), 1) - if err != nil || block == nil || block.Block == nil { - return nil + if err != nil { + return err } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, block.Block.Height) + if block == nil || block.Block == nil { + return fmt.Errorf("block %s not found", hash.Hex()) + } + return api.guardTraceRequest(ctx, endpoint, block.Block.Height) } -func (api *DebugAPI) guardHistoricalDebugTraceByNumberOrHash(ctx context.Context, endpoint string, blockNrOrHash rpc.BlockNumberOrHash) error { +func (api *DebugAPI) guardTraceRequestByNumberOrHash(ctx context.Context, endpoint string, blockNrOrHash rpc.BlockNumberOrHash) error { if number, ok := blockNrOrHash.Number(); ok { - return api.guardHistoricalDebugTraceByNumber(ctx, endpoint, number) + return api.guardTraceRequestByNumber(ctx, endpoint, number) } if hash, ok := blockNrOrHash.Hash(); ok { - return api.guardHistoricalDebugTraceByHash(ctx, endpoint, hash) + return api.guardTraceRequestByHash(ctx, endpoint, hash) } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) + return api.guardTraceRequest(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) } func (api *DebugAPI) resolveDebugTraceBlockNumber(ctx context.Context, number rpc.BlockNumber) (int64, error) { @@ -331,7 +347,7 @@ func (api *DebugAPI) TraceTransaction(ctx context.Context, hash common.Hash, con if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } - if returnErr = api.guardHistoricalDebugTraceByTxHash(ctx, "debug_traceTransaction", hash); returnErr != nil { + if returnErr = api.guardTraceRequestByTxHash(ctx, "debug_traceTransaction", hash); returnErr != nil { return nil, returnErr } @@ -374,7 +390,7 @@ func (api *DebugAPI) tryTraceCache(hash common.Hash, config *tracers.TraceConfig // blockTraceCacheGet assembles a per-tx hit; returns (nil, false) if any miss. func blockTraceCacheGet(cache *keeper.TraceDB, height int64, txHashes []common.Hash, config *tracers.TraceConfig) ([]*tracers.TxTraceResult, bool) { - if cache == nil { + if cache == nil || len(txHashes) == 0 { return nil, false } name := bakeableTracerName(config) @@ -544,7 +560,7 @@ func (api *DebugAPI) TraceBlockByNumber(ctx context.Context, number rpc.BlockNum if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } - if returnErr = api.guardHistoricalDebugTraceByNumber(ctx, "debug_traceBlockByNumber", number); returnErr != nil { + if returnErr = api.guardTraceRequestByNumber(ctx, "debug_traceBlockByNumber", number); returnErr != nil { return nil, returnErr } @@ -579,6 +595,9 @@ func (api *DebugAPI) TraceBlockByHash(ctx context.Context, hash common.Hash, con if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } + if returnErr = api.guardTraceRequestByHash(ctx, "debug_traceBlockByHash", hash); returnErr != nil { + return nil, returnErr + } ctx, done, err := api.prepareTraceContext(ctx) if err != nil { @@ -586,10 +605,6 @@ func (api *DebugAPI) TraceBlockByHash(ctx context.Context, hash common.Hash, con } defer done() - if returnErr = api.guardHistoricalDebugTraceByHash(ctx, "debug_traceBlockByHash", hash); returnErr != nil { - return nil, returnErr - } - if cached, ok := api.tryBlockTraceCacheByHash(ctx, hash, config); ok { return cached, nil } @@ -618,6 +633,9 @@ func (api *DebugAPI) TraceCall(ctx context.Context, args export.TransactionArgs, if returnErr = api.validateTraceTracer(&config.TraceConfig); returnErr != nil { return nil, returnErr } + if returnErr = api.guardTraceRequestByNumberOrHash(ctx, "debug_traceCall", blockNrOrHash); returnErr != nil { + return nil, returnErr + } ctx, done, err := api.prepareTraceContext(ctx) if err != nil { @@ -625,10 +643,6 @@ func (api *DebugAPI) TraceCall(ctx context.Context, args export.TransactionArgs, } defer done() - if returnErr = api.guardHistoricalDebugTraceByNumberOrHash(ctx, "debug_traceCall", blockNrOrHash); returnErr != nil { - return nil, returnErr - } - if returnErr = validateStateOverrides(config.StateOverrides, api.backend.MaxStateOverrideAccounts(), api.backend.MaxStateOverrideSlots()); returnErr != nil { return nil, returnErr } @@ -683,7 +697,7 @@ func (api *DebugAPI) TraceStateAccess(ctx context.Context, hash common.Hash) (re returnErr = fmt.Errorf("panic occurred: %v, could not trace tx state: %s", r, hash.Hex()) } }() - if returnErr = api.guardHistoricalDebugTraceByTxHash(ctx, "debug_traceStateAccess", hash); returnErr != nil { + if returnErr = api.guardTraceRequestByTxHash(ctx, "debug_traceStateAccess", hash); returnErr != nil { return nil, returnErr } diff --git a/evmrpc/tracers_semaphore_test.go b/evmrpc/tracers_semaphore_test.go index efd2db45fe..39c1be7192 100644 --- a/evmrpc/tracers_semaphore_test.go +++ b/evmrpc/tracers_semaphore_test.go @@ -6,11 +6,8 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/export" "github.com/ethereum/go-ethereum/rpc" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" - tmbytes "github.com/sei-protocol/sei-chain/sei-tendermint/libs/bytes" - "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes" "github.com/stretchr/testify/require" ) @@ -89,20 +86,12 @@ func TestAcquireTraceSemaphoreCanceledContextDoesNotConsumeSlot(t *testing.T) { } } -type panicHashLookupClient struct { - *heightTestClient -} - -func (c *panicHashLookupClient) BlockByHash(context.Context, tmbytes.HexBytes) (*coretypes.ResultBlock, error) { - panic("hash lookup should not happen before trace context setup") -} +func TestTraceBlockByNumberRejectsConcurrencyLimitAfterGuard(t *testing.T) { + t.Parallel() -func TestHashBasedTraceEndpointsAcquireSemaphoreBeforeHashLookup(t *testing.T) { latestHeight := int64(10) latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) - tmClient := &panicHashLookupClient{ - heightTestClient: newHeightTestClient(8, 1, latestHeight), - } + tmClient := newHeightTestClient(8, 1, latestHeight) watermarks := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, &fakeReceiptStore{latest: latestHeight}) api := &DebugAPI{ tmClient: tmClient, @@ -119,11 +108,37 @@ func TestHashBasedTraceEndpointsAcquireSemaphoreBeforeHashLookup(t *testing.T) { api.traceCallSemaphore <- struct{}{} defer func() { <-api.traceCallSemaphore }() - hash := common.HexToHash(highBlockHashHex) - _, err := api.TraceBlockByHash(context.Background(), hash, nil) + _, err := api.TraceBlockByNumber(context.Background(), rpc.LatestBlockNumber, nil) require.ErrorIs(t, err, errTraceConcurrencyLimit) +} - blockNrOrHash := rpc.BlockNumberOrHashWithHash(hash, false) - _, err = api.TraceCall(context.Background(), export.TransactionArgs{}, blockNrOrHash, nil) - require.ErrorIs(t, err, errTraceConcurrencyLimit) +func TestTraceBlockByHashChecksAvailabilityBeforeSemaphore(t *testing.T) { + t.Parallel() + + prunedHeight := int64(100) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(prunedHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 150} + watermarks := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs) + api := &DebugAPI{ + tmClient: tmClient, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + connectionType: ConnectionTypeHTTP, + traceCallSemaphore: make(chan struct{}, 1), + traceTimeout: time.Second, + backend: &Backend{ + tmClient: tmClient, + watermarks: watermarks, + }, + } + + api.traceCallSemaphore <- struct{}{} + defer func() { <-api.traceCallSemaphore }() + + hash := common.HexToHash(highBlockHashHex) + _, err := api.TraceBlockByHash(context.Background(), hash, nil) + require.Error(t, err) + require.ErrorContains(t, err, "receipts have been pruned") + require.NotErrorIs(t, err, errTraceConcurrencyLimit) } diff --git a/evmrpc/utils.go b/evmrpc/utils.go index 2037fdb447..c868bbe502 100644 --- a/evmrpc/utils.go +++ b/evmrpc/utils.go @@ -229,6 +229,9 @@ func filterTransactions( continue } ethtx, _ := m.AsTransaction() + if ethtx == nil { + continue + } hash := ethtx.Hash() sender, _ := rpcutils.RecoverEVMSender(ethtx, block.Block.Height, block.Block.Time.Unix()) receipt, found := getOrSetCachedReceipt(cacheCreationMutex, globalBlockCache, latestCtx, k, block, hash) diff --git a/evmrpc/watermark_manager.go b/evmrpc/watermark_manager.go index 959946d09d..5262d9fd26 100644 --- a/evmrpc/watermark_manager.go +++ b/evmrpc/watermark_manager.go @@ -184,6 +184,28 @@ func (m *WatermarkManager) EnsureReceiptHeightAvailable(height int64) error { return nil } +// EnsureStateHeightAvailable verifies that historical state for the given block +// height has not been pruned from the state store. +func (m *WatermarkManager) EnsureStateHeightAvailable(ctx context.Context, height int64) error { + _, stateEarliest, latest, err := m.Watermarks(ctx) + if err != nil { + return err + } + return ensureWithinWatermarks(height, stateEarliest, latest) +} + +// EnsureTraceHeightAvailable verifies block, receipt, and state availability +// for debug_trace* endpoints. All three stores must retain the height. +func (m *WatermarkManager) EnsureTraceHeightAvailable(ctx context.Context, height int64) error { + if err := m.EnsureBlockHeightAvailable(ctx, height); err != nil { + return err + } + if err := m.EnsureReceiptHeightAvailable(height); err != nil { + return err + } + return m.EnsureStateHeightAvailable(ctx, height) +} + func ensureWithinWatermarks(height, earliest, latest int64) error { if height > latest { return fmt.Errorf("requested height %d is not yet available; safe latest is %d: %w", height, latest, ErrBlockHeightNotYetAvailable) diff --git a/evmrpc/watermark_manager_test.go b/evmrpc/watermark_manager_test.go index 7cc3f633fa..aa419ddc8a 100644 --- a/evmrpc/watermark_manager_test.go +++ b/evmrpc/watermark_manager_test.go @@ -134,6 +134,48 @@ func TestEnsureReceiptHeightAvailable(t *testing.T) { }) } +func TestEnsureStateHeightAvailable(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + + t.Run("pruned state height returns error", func(t *testing.T) { + stateStore := &fakeStateStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, &fakeReceiptStore{latest: 200}) + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 100), "has been pruned") + }) + + t.Run("height within state retention succeeds", func(t *testing.T) { + stateStore := &fakeStateStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, &fakeReceiptStore{latest: 200}) + require.NoError(t, wm.EnsureStateHeightAvailable(t.Context(), 150)) + }) + + t.Run("nil state store uses latest as earliest from Watermarks", func(t *testing.T) { + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), nil, &fakeReceiptStore{latest: 200}) + require.NoError(t, wm.EnsureStateHeightAvailable(t.Context(), 200)) + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 199), "has been pruned") + }) +} + +func TestEnsureTraceHeightAvailable(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + stateStore := &fakeStateStore{latest: 200, earliest: 1} + rs := &fakeReceiptStore{latest: 200, earliest: 1} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, rs) + + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), 175)) + + rs.earliest = 150 + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 100), "receipts have been pruned") + + rs.earliest = 1 + stateStore.earliest = 150 + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 100), "has been pruned") +} + func TestLatestAndEarliestHeightHelpers(t *testing.T) { tmClient := &fakeTMClient{ status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 22, EarliestBlockHeight: 11}}, diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index 069bc127a5..3bc125ad4f 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -213,6 +213,9 @@ func (s *littReceiptStore) GetReceipt(ctx sdk.Context, txHash common.Hash) (*typ if err == nil { return receipt, nil } + if errors.Is(err, ErrReceiptPruned) { + return nil, err + } if !errors.Is(err, ErrNotFound) { return nil, err } @@ -242,7 +245,9 @@ func (s *littReceiptStore) GetReceiptFromStore(_ sdk.Context, txHash common.Hash // Enforce the KeepRecent floor: litt expires values lazily via TTL, so a // pruned block may still be physically present. if s.belowRetentionFloor(r.BlockNumber) { - return nil, ErrNotFound + earliest := s.earliestVersion.Load() + return nil, fmt.Errorf("requested height %d receipts have been pruned; earliest available is %d: %w", + r.BlockNumber, earliest, ErrReceiptPruned) } return &r, nil } diff --git a/sei-db/ledger_db/receipt/littidx_test.go b/sei-db/ledger_db/receipt/littidx_test.go index d4f5fa23ad..72a1632867 100644 --- a/sei-db/ledger_db/receipt/littidx_test.go +++ b/sei-db/ledger_db/receipt/littidx_test.go @@ -332,7 +332,7 @@ func TestLittIdxPrune(t *testing.T) { // Pruned blocks are invisible (tag entries deleted, read floor enforced). for block := uint64(1); block <= 5; block++ { _, err := store.GetReceiptFromStore(ctx, litTxHash(block, 0)) - require.ErrorIs(t, err, receipt.ErrNotFound, "block %d should be pruned", block) + require.ErrorIs(t, err, receipt.ErrReceiptPruned, "block %d should be pruned", block) } logs, err := store.FilterLogs(ctx, 1, 5, filters.FilterCriteria{Addresses: []common.Address{addr}}, nil) require.NoError(t, err) diff --git a/sei-db/ledger_db/receipt/receipt_store.go b/sei-db/ledger_db/receipt/receipt_store.go index 25bdeded24..357cd1e710 100644 --- a/sei-db/ledger_db/receipt/receipt_store.go +++ b/sei-db/ledger_db/receipt/receipt_store.go @@ -27,7 +27,11 @@ var logger = seilog.NewLogger("db", "ledger-db", "receipt") // Sentinel errors for consistent error checking. var ( - ErrNotFound = errors.New("receipt not found") + ErrNotFound = errors.New("receipt not found") + // ErrReceiptPruned is returned when a receipt existed but has fallen below + // the store's retention floor. Distinct from ErrNotFound so RPC callers can + // surface an explicit pruned error instead of treating the tx as missing. + ErrReceiptPruned = errors.New("receipt pruned") ErrNotConfigured = errors.New("receipt store not configured") ErrRangeQueryNotSupported = errors.New("range query not supported by this backend") // ErrTooManyLogs is returned by FilterLogs when a query matches more logs From 19a24a552f22d51acd5841a20796fb3dbc76be18 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Mon, 10 Aug 2026 18:36:01 +0200 Subject: [PATCH 2/7] fix(evmrpc): address AI review blockers on trace height guards (PLT-975) - Resolve latest/pending/safe/finalized trace tags via the watermark's safe latest instead of the raw app tip, so debug_trace* no longer intermittently errors while receipts/state lag the tip. - Check the parent height (height-1) against state retention, matching how initializeBlock actually replays a traced block. - Wrap ErrReceiptPruned around ErrNotFound so eth_getTransactionReceipt and friends keep returning null for pruned receipts instead of an RPC error, while trace guards can still react to it specifically. Co-Authored-By: Claude Sonnet 5 --- evmrpc/tracers.go | 21 +++++++++++++++++---- evmrpc/watermark_manager.go | 9 ++++++++- sei-db/ledger_db/receipt/receipt_store.go | 7 ++++--- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/evmrpc/tracers.go b/evmrpc/tracers.go index 9e0b754d37..d744148f45 100644 --- a/evmrpc/tracers.go +++ b/evmrpc/tracers.go @@ -122,7 +122,20 @@ func (api *DebugAPI) guardTraceRequestByTxHash(ctx context.Context, endpoint str return api.guardTraceRequest(ctx, endpoint, int64(rcpt.BlockNumber)) //nolint:gosec } } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) + return api.guardHistoricalDebugTraceHeight(ctx, endpoint, api.latestTraceHeight(ctx)) +} + +// latestTraceHeight resolves the height debug_trace* should use for latest-ish +// tags. It prefers the watermark's safe latest over the raw app tip, since the +// tip can outrun the receipt/state stores by a block or so and would otherwise +// make EnsureTraceHeightAvailable reject the most common trace requests. +func (api *DebugAPI) latestTraceHeight(ctx context.Context) int64 { + if api.backend != nil && api.backend.watermarks != nil { + if latest, err := api.backend.watermarks.LatestHeight(ctx); err == nil { + return latest + } + } + return api.ctxProvider(LatestCtxHeight).BlockHeight() } func (api *DebugAPI) guardTraceRequestByNumber(ctx context.Context, endpoint string, number rpc.BlockNumber) error { @@ -135,7 +148,7 @@ func (api *DebugAPI) guardTraceRequestByNumber(ctx context.Context, endpoint str func (api *DebugAPI) guardTraceRequestByHash(ctx context.Context, endpoint string, hash common.Hash) error { if api.backend == nil || api.tmClient == nil { - return api.guardTraceRequest(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) + return api.guardTraceRequest(ctx, endpoint, api.latestTraceHeight(ctx)) } block, err := blockByHashRespectingWatermarks(ctx, api.tmClient, api.backend.watermarks, hash.Bytes(), 1) if err != nil { @@ -154,13 +167,13 @@ func (api *DebugAPI) guardTraceRequestByNumberOrHash(ctx context.Context, endpoi if hash, ok := blockNrOrHash.Hash(); ok { return api.guardTraceRequestByHash(ctx, endpoint, hash) } - return api.guardTraceRequest(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) + return api.guardTraceRequest(ctx, endpoint, api.latestTraceHeight(ctx)) } func (api *DebugAPI) resolveDebugTraceBlockNumber(ctx context.Context, number rpc.BlockNumber) (int64, error) { switch number { case rpc.SafeBlockNumber, rpc.FinalizedBlockNumber, rpc.LatestBlockNumber, rpc.PendingBlockNumber: - return api.ctxProvider(LatestCtxHeight).BlockHeight(), nil + return api.latestTraceHeight(ctx), nil case rpc.EarliestBlockNumber: if api.tmClient == nil { return 0, errors.New("tendermint client is not configured") diff --git a/evmrpc/watermark_manager.go b/evmrpc/watermark_manager.go index 5262d9fd26..bee0764de2 100644 --- a/evmrpc/watermark_manager.go +++ b/evmrpc/watermark_manager.go @@ -203,7 +203,14 @@ func (m *WatermarkManager) EnsureTraceHeightAvailable(ctx context.Context, heigh if err := m.EnsureReceiptHeightAvailable(height); err != nil { return err } - return m.EnsureStateHeightAvailable(ctx, height) + // Tracing replays from the parent state (height-1), not height itself. With + // no historical state store, state only ever extends to the tip anyway, so + // the parent-height distinction doesn't apply. + stateHeight := height + if m.stateStore != nil { + stateHeight = max(height-1, m.genesisInitialHeight()) + } + return m.EnsureStateHeightAvailable(ctx, stateHeight) } func ensureWithinWatermarks(height, earliest, latest int64) error { diff --git a/sei-db/ledger_db/receipt/receipt_store.go b/sei-db/ledger_db/receipt/receipt_store.go index 357cd1e710..6152b81267 100644 --- a/sei-db/ledger_db/receipt/receipt_store.go +++ b/sei-db/ledger_db/receipt/receipt_store.go @@ -29,9 +29,10 @@ var logger = seilog.NewLogger("db", "ledger-db", "receipt") var ( ErrNotFound = errors.New("receipt not found") // ErrReceiptPruned is returned when a receipt existed but has fallen below - // the store's retention floor. Distinct from ErrNotFound so RPC callers can - // surface an explicit pruned error instead of treating the tx as missing. - ErrReceiptPruned = errors.New("receipt pruned") + // the store's retention floor. It wraps ErrNotFound so existing not-found + // handling (e.g. eth_getTransactionReceipt returning null) still applies, + // while errors.Is(err, ErrReceiptPruned) lets trace guards react to it specifically. + ErrReceiptPruned = fmt.Errorf("receipt pruned: %w", ErrNotFound) ErrNotConfigured = errors.New("receipt store not configured") ErrRangeQueryNotSupported = errors.New("range query not supported by this backend") // ErrTooManyLogs is returned by FilterLogs when a query matches more logs From 6bc998dcf682da39309744ef6cfb41ae90112998 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 11 Aug 2026 15:19:12 +0200 Subject: [PATCH 3/7] Fixes Latest-tag guard vs execution mismatch --- evmrpc/historical_debug_trace_test.go | 59 +++++++++++++++++++++++++++ evmrpc/simulate.go | 24 +++-------- evmrpc/simulate_test.go | 21 ---------- 3 files changed, 64 insertions(+), 40 deletions(-) diff --git a/evmrpc/historical_debug_trace_test.go b/evmrpc/historical_debug_trace_test.go index 399a2ca427..522b4b7145 100644 --- a/evmrpc/historical_debug_trace_test.go +++ b/evmrpc/historical_debug_trace_test.go @@ -190,3 +190,62 @@ func TestTraceBlockByNumberReceiptPrunedBeforeSemaphore(t *testing.T) { require.Contains(t, err.Error(), "receipts have been pruned") require.NotErrorIs(t, err, errTraceConcurrencyLimit) } + +func TestBlockByNumberLatestUsesSafeLatestWatermark(t *testing.T) { + t.Parallel() + + const ( + safeLatest = int64(99) + ctxTip = int64(100) + ) + latestCtx := sdk.Context{}.WithBlockHeight(ctxTip) + tmClient := newHeightTestClient(safeLatest, 1, ctxTip) + rs := &fakeReceiptStore{latest: safeLatest, earliest: 1} + stateStore := &fakeStateStore{latest: safeLatest, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + blockNumberPtr, err := getBlockNumber(t.Context(), tmClient, rpc.LatestBlockNumber) + require.NoError(t, err) + require.Nil(t, blockNumberPtr) + + tmBlock, err := blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, blockNumberPtr, 1) + require.NoError(t, err) + require.Equal(t, safeLatest, tmBlock.Block.Height) + + rawTip := ctxTip + _, err = blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, &rawTip, 1) + require.Error(t, err) + require.ErrorIs(t, err, ErrBlockHeightNotYetAvailable) +} + +func TestTraceLatestTagGuardMatchesBlockResolution(t *testing.T) { + t.Parallel() + + const ( + safeLatest = int64(99) + ctxTip = int64(100) + ) + latestCtx := sdk.Context{}.WithBlockHeight(ctxTip) + tmClient := newHeightTestClient(safeLatest, 1, ctxTip) + rs := &fakeReceiptStore{latest: safeLatest, earliest: 1} + stateStore := &fakeStateStore{latest: safeLatest, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + api := &DebugAPI{ + tmClient: tmClient, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + backend: &Backend{ + tmClient: tmClient, + watermarks: wm, + }, + } + + guardHeight, err := api.resolveDebugTraceBlockNumber(t.Context(), rpc.LatestBlockNumber) + require.NoError(t, err) + require.Equal(t, safeLatest, guardHeight) + + blockNumberPtr, err := getBlockNumber(t.Context(), tmClient, rpc.LatestBlockNumber) + require.NoError(t, err) + tmBlock, err := blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, blockNumberPtr, 1) + require.NoError(t, err) + require.Equal(t, guardHeight, tmBlock.Block.Height) +} diff --git a/evmrpc/simulate.go b/evmrpc/simulate.go index dc05696a2c..b58736cb3e 100644 --- a/evmrpc/simulate.go +++ b/evmrpc/simulate.go @@ -410,26 +410,12 @@ func (b *Backend) ChainDb() ethdb.Database { panic("implement me") } -func (b Backend) ConvertBlockNumber(bn rpc.BlockNumber) int64 { - blockNum := bn.Int64() - switch blockNum { - case rpc.SafeBlockNumber.Int64(), rpc.FinalizedBlockNumber.Int64(), rpc.LatestBlockNumber.Int64(): - blockNum = b.ctxProvider(LatestCtxHeight).BlockHeight() - case rpc.EarliestBlockNumber.Int64(): - genesisRes, err := b.tmClient.Genesis(context.Background()) - if err != nil { - panic("could not get genesis info from tendermint") - } - blockNum = genesisRes.Genesis.InitialHeight - case rpc.PendingBlockNumber.Int64(): - panic("tracing on pending block is not supported") - } - return blockNum -} - func (b Backend) BlockByNumber(ctx context.Context, bn rpc.BlockNumber) (*ethtypes.Block, []tracersutils.TraceBlockMetadata, error) { - blockNum := b.ConvertBlockNumber(bn) - tmBlock, err := blockByNumberRespectingWatermarks(ctx, b.tmClient, b.watermarks, &blockNum, 1) + blockNumberPtr, err := getBlockNumber(ctx, b.tmClient, bn) + if err != nil { + return nil, nil, err + } + tmBlock, err := blockByNumberRespectingWatermarks(ctx, b.tmClient, b.watermarks, blockNumberPtr, 1) if err != nil { return nil, nil, err } diff --git a/evmrpc/simulate_test.go b/evmrpc/simulate_test.go index 27f19c98d5..9b5d0a0909 100644 --- a/evmrpc/simulate_test.go +++ b/evmrpc/simulate_test.go @@ -379,27 +379,6 @@ func TestNewRevertError(t *testing.T) { require.Equal(t, "0x", err.ErrorData()) } -func TestConvertBlockNumber(t *testing.T) { - tmClient := &MockClient{} - watermarks := evmrpc.NewWatermarkManager(tmClient, func(i int64) sdk.Context { - if i == evmrpc.LatestCtxHeight { - return sdk.Context{}.WithBlockHeight(1000) - } - return sdk.Context{} - }, nil, nil) - backend := evmrpc.NewBackend(func(i int64) sdk.Context { - if i == evmrpc.LatestCtxHeight { - return sdk.Context{}.WithBlockHeight(1000) - } - return sdk.Context{} - }, nil, legacyabci.BeginBlockKeepers{}, nil, &MockClient{}, nil, nil, nil, evmrpc.NewBlockCache(3000), &sync.Mutex{}, watermarks) - require.Equal(t, int64(10), backend.ConvertBlockNumber(10)) - require.Equal(t, int64(1), backend.ConvertBlockNumber(0)) - require.Equal(t, int64(1000), backend.ConvertBlockNumber(-2)) - require.Equal(t, int64(1000), backend.ConvertBlockNumber(-3)) - require.Equal(t, int64(1000), backend.ConvertBlockNumber(-4)) -} - func TestPreV620UpgradeUsesBaseFeeNil(t *testing.T) { // Set up a test context with a height before v6.2.0 upgrade // For pacific-1 chain, we need to set a height that's before the v6.2.0 upgrade From c964dcd8ec295f6dceeb3f01c9326329be0f668a Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 11 Aug 2026 15:31:59 +0200 Subject: [PATCH 4/7] fixed debug trace call overgaurd issue --- evmrpc/historical_debug_trace_test.go | 37 +++++++++++++++++++++-- evmrpc/tracers.go | 43 ++++++++++++++++++++++++++- evmrpc/watermark_manager.go | 13 +++++++- evmrpc/watermark_manager_test.go | 21 +++++++++++++ 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/evmrpc/historical_debug_trace_test.go b/evmrpc/historical_debug_trace_test.go index 522b4b7145..f3af4b1fda 100644 --- a/evmrpc/historical_debug_trace_test.go +++ b/evmrpc/historical_debug_trace_test.go @@ -99,13 +99,15 @@ func TestGuardTraceRequestByHashUsesTendermintHeight(t *testing.T) { latestHeight := int64(10) latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) tmClient := newHeightTestClient(8, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 1} + stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} api := &DebugAPI{ tmClient: tmClient, ctxProvider: func(int64) sdk.Context { return latestCtx }, connectionType: ConnectionTypeHTTP, maxBlockLookback: 1, backend: &Backend{ - watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, &fakeReceiptStore{latest: latestHeight}), + watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs), }, } @@ -113,10 +115,41 @@ func TestGuardTraceRequestByHashUsesTendermintHeight(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "block number 8 is beyond max lookback of 1") - err = api.guardTraceRequestByHash(t.Context(), "debug_traceCall", common.HexToHash("0x1")) + err = api.guardTraceCallRequestByHash(t.Context(), "debug_traceCall", common.HexToHash("0x1")) require.NoError(t, err) } +func TestEnsureTraceCallHeightAvailableIgnoresReceipts(t *testing.T) { + t.Parallel() + + parentHeight := int64(149) + receiptFloor := int64(150) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(parentHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: receiptFloor} + stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), parentHeight), "receipts have been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), parentHeight)) +} + +func TestEnsureTraceCallHeightAvailableUsesStateAtHeight(t *testing.T) { + t.Parallel() + + stateFloor := int64(150) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(stateFloor, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 1} + stateStore := &fakeStateStore{latest: latestHeight, earliest: stateFloor} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), stateFloor), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), stateFloor)) +} + func TestEnsureTraceHeightAvailableReceiptsPruned(t *testing.T) { t.Parallel() diff --git a/evmrpc/tracers.go b/evmrpc/tracers.go index d744148f45..d41f36ec5a 100644 --- a/evmrpc/tracers.go +++ b/evmrpc/tracers.go @@ -170,6 +170,47 @@ func (api *DebugAPI) guardTraceRequestByNumberOrHash(ctx context.Context, endpoi return api.guardTraceRequest(ctx, endpoint, api.latestTraceHeight(ctx)) } +func (api *DebugAPI) guardTraceCallRequest(ctx context.Context, endpoint string, height int64) error { + if api.backend != nil && api.backend.watermarks != nil { + if err := api.backend.watermarks.EnsureTraceCallHeightAvailable(ctx, height); err != nil { + return err + } + } + return api.guardHistoricalDebugTraceHeight(ctx, endpoint, height) +} + +func (api *DebugAPI) guardTraceCallRequestByNumber(ctx context.Context, endpoint string, number rpc.BlockNumber) error { + height, err := api.resolveDebugTraceBlockNumber(ctx, number) + if err != nil { + return err + } + return api.guardTraceCallRequest(ctx, endpoint, height) +} + +func (api *DebugAPI) guardTraceCallRequestByHash(ctx context.Context, endpoint string, hash common.Hash) error { + if api.backend == nil || api.tmClient == nil { + return api.guardTraceCallRequest(ctx, endpoint, api.latestTraceHeight(ctx)) + } + block, err := blockByHashRespectingWatermarks(ctx, api.tmClient, api.backend.watermarks, hash.Bytes(), 1) + if err != nil { + return err + } + if block == nil || block.Block == nil { + return fmt.Errorf("block %s not found", hash.Hex()) + } + return api.guardTraceCallRequest(ctx, endpoint, block.Block.Height) +} + +func (api *DebugAPI) guardTraceCallRequestByNumberOrHash(ctx context.Context, endpoint string, blockNrOrHash rpc.BlockNumberOrHash) error { + if number, ok := blockNrOrHash.Number(); ok { + return api.guardTraceCallRequestByNumber(ctx, endpoint, number) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return api.guardTraceCallRequestByHash(ctx, endpoint, hash) + } + return api.guardTraceCallRequest(ctx, endpoint, api.latestTraceHeight(ctx)) +} + func (api *DebugAPI) resolveDebugTraceBlockNumber(ctx context.Context, number rpc.BlockNumber) (int64, error) { switch number { case rpc.SafeBlockNumber, rpc.FinalizedBlockNumber, rpc.LatestBlockNumber, rpc.PendingBlockNumber: @@ -646,7 +687,7 @@ func (api *DebugAPI) TraceCall(ctx context.Context, args export.TransactionArgs, if returnErr = api.validateTraceTracer(&config.TraceConfig); returnErr != nil { return nil, returnErr } - if returnErr = api.guardTraceRequestByNumberOrHash(ctx, "debug_traceCall", blockNrOrHash); returnErr != nil { + if returnErr = api.guardTraceCallRequestByNumberOrHash(ctx, "debug_traceCall", blockNrOrHash); returnErr != nil { return nil, returnErr } diff --git a/evmrpc/watermark_manager.go b/evmrpc/watermark_manager.go index bee0764de2..ea7109ad04 100644 --- a/evmrpc/watermark_manager.go +++ b/evmrpc/watermark_manager.go @@ -194,8 +194,19 @@ func (m *WatermarkManager) EnsureStateHeightAvailable(ctx context.Context, heigh return ensureWithinWatermarks(height, stateEarliest, latest) } +// EnsureTraceCallHeightAvailable verifies block and state availability for +// debug_traceCall. TraceCall loads state at the requested height via +// StateAndHeaderByNumberOrHash and never reads receipts. +func (m *WatermarkManager) EnsureTraceCallHeightAvailable(ctx context.Context, height int64) error { + if err := m.EnsureBlockHeightAvailable(ctx, height); err != nil { + return err + } + return m.EnsureStateHeightAvailable(ctx, height) +} + // EnsureTraceHeightAvailable verifies block, receipt, and state availability -// for debug_trace* endpoints. All three stores must retain the height. +// for debug_trace* replay endpoints (transaction/block). Replay loads parent +// state (height-1) and reads receipts at the requested height. func (m *WatermarkManager) EnsureTraceHeightAvailable(ctx context.Context, height int64) error { if err := m.EnsureBlockHeightAvailable(ctx, height); err != nil { return err diff --git a/evmrpc/watermark_manager_test.go b/evmrpc/watermark_manager_test.go index aa419ddc8a..531f006411 100644 --- a/evmrpc/watermark_manager_test.go +++ b/evmrpc/watermark_manager_test.go @@ -158,6 +158,27 @@ func TestEnsureStateHeightAvailable(t *testing.T) { }) } +func TestEnsureTraceCallHeightAvailable(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + stateStore := &fakeStateStore{latest: 200, earliest: 150} + rs := &fakeReceiptStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, rs) + + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 175)) + + // Receipts pruned below 150; replay guard fails, TraceCall guard does not check receipts. + rs.earliest = 150 + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 149), "receipts have been pruned") + stateStore.earliest = 1 + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 149)) + + stateStore.earliest = 175 + require.ErrorContains(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 100), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 175)) +} + func TestEnsureTraceHeightAvailable(t *testing.T) { tmClient := &fakeTMClient{ status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, From 1845b87ab92017a7531128846f0d75cc0a65448f Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Tue, 11 Aug 2026 15:41:10 +0200 Subject: [PATCH 5/7] Fixed edge case for state store nil --- evmrpc/historical_debug_trace_test.go | 3 +-- evmrpc/watermark_manager.go | 14 ++++++++------ evmrpc/watermark_manager_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/evmrpc/historical_debug_trace_test.go b/evmrpc/historical_debug_trace_test.go index f3af4b1fda..26c8554fb5 100644 --- a/evmrpc/historical_debug_trace_test.go +++ b/evmrpc/historical_debug_trace_test.go @@ -100,14 +100,13 @@ func TestGuardTraceRequestByHashUsesTendermintHeight(t *testing.T) { latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) tmClient := newHeightTestClient(8, 1, latestHeight) rs := &fakeReceiptStore{latest: latestHeight, earliest: 1} - stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} api := &DebugAPI{ tmClient: tmClient, ctxProvider: func(int64) sdk.Context { return latestCtx }, connectionType: ConnectionTypeHTTP, maxBlockLookback: 1, backend: &Backend{ - watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs), + watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs), }, } diff --git a/evmrpc/watermark_manager.go b/evmrpc/watermark_manager.go index ea7109ad04..cc8d061e79 100644 --- a/evmrpc/watermark_manager.go +++ b/evmrpc/watermark_manager.go @@ -201,6 +201,10 @@ func (m *WatermarkManager) EnsureTraceCallHeightAvailable(ctx context.Context, h if err := m.EnsureBlockHeightAvailable(ctx, height); err != nil { return err } + if m.stateStore == nil { + // SS disabled: trace replay uses SC via ctxProvider, not SS retention. + return nil + } return m.EnsureStateHeightAvailable(ctx, height) } @@ -214,13 +218,11 @@ func (m *WatermarkManager) EnsureTraceHeightAvailable(ctx context.Context, heigh if err := m.EnsureReceiptHeightAvailable(height); err != nil { return err } - // Tracing replays from the parent state (height-1), not height itself. With - // no historical state store, state only ever extends to the tip anyway, so - // the parent-height distinction doesn't apply. - stateHeight := height - if m.stateStore != nil { - stateHeight = max(height-1, m.genesisInitialHeight()) + if m.stateStore == nil { + // SS disabled: trace replay uses SC via ctxProvider, not SS retention. + return nil } + stateHeight := max(height-1, m.genesisInitialHeight()) return m.EnsureStateHeightAvailable(ctx, stateHeight) } diff --git a/evmrpc/watermark_manager_test.go b/evmrpc/watermark_manager_test.go index 531f006411..31a231564e 100644 --- a/evmrpc/watermark_manager_test.go +++ b/evmrpc/watermark_manager_test.go @@ -197,6 +197,30 @@ func TestEnsureTraceHeightAvailable(t *testing.T) { require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 100), "has been pruned") } +func TestEnsureTraceHeightAvailableSkipsStateWhenSSDisabled(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + rs := &fakeReceiptStore{latest: 200, earliest: 1} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), nil, rs) + + // EnsureStateHeightAvailable rejects below tip when SS is disabled. + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 199), "has been pruned") + // Trace guards skip the SS leg and defer state to SC/ctxProvider. + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), 199)) + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 199)) +} + +func TestEnsureTraceCallHeightAvailableSkipsStateWhenSSDisabled(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), nil, &fakeReceiptStore{latest: 200}) + + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 100), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 100)) +} + func TestLatestAndEarliestHeightHelpers(t *testing.T) { tmClient := &fakeTMClient{ status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 22, EarliestBlockHeight: 11}}, From f26224e6bab8e73d676b1aaea41622b48208d261 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 12 Aug 2026 12:57:01 +0200 Subject: [PATCH 6/7] Fix parent block retention boundary --- evmrpc/watermark_manager.go | 13 ++++++++++++- evmrpc/watermark_manager_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/evmrpc/watermark_manager.go b/evmrpc/watermark_manager.go index cc8d061e79..dadddc7114 100644 --- a/evmrpc/watermark_manager.go +++ b/evmrpc/watermark_manager.go @@ -210,11 +210,15 @@ func (m *WatermarkManager) EnsureTraceCallHeightAvailable(ctx context.Context, h // EnsureTraceHeightAvailable verifies block, receipt, and state availability // for debug_trace* replay endpoints (transaction/block). Replay loads parent -// state (height-1) and reads receipts at the requested height. +// block validators and parent state (height-1) and reads receipts at the +// requested height. func (m *WatermarkManager) EnsureTraceHeightAvailable(ctx context.Context, height int64) error { if err := m.EnsureBlockHeightAvailable(ctx, height); err != nil { return err } + if err := m.ensureReplayParentBlockAvailable(ctx, height); err != nil { + return err + } if err := m.EnsureReceiptHeightAvailable(height); err != nil { return err } @@ -226,6 +230,13 @@ func (m *WatermarkManager) EnsureTraceHeightAvailable(ctx context.Context, heigh return m.EnsureStateHeightAvailable(ctx, stateHeight) } +// ensureReplayParentBlockAvailable verifies the parent block height replay +// tracing loads for validator set lookup in initializeBlock. +func (m *WatermarkManager) ensureReplayParentBlockAvailable(ctx context.Context, height int64) error { + parentBlockHeight := max(height-1, 0) + return m.EnsureBlockHeightAvailable(ctx, parentBlockHeight) +} + func ensureWithinWatermarks(height, earliest, latest int64) error { if height > latest { return fmt.Errorf("requested height %d is not yet available; safe latest is %d: %w", height, latest, ErrBlockHeightNotYetAvailable) diff --git a/evmrpc/watermark_manager_test.go b/evmrpc/watermark_manager_test.go index 31a231564e..2c393e8cc7 100644 --- a/evmrpc/watermark_manager_test.go +++ b/evmrpc/watermark_manager_test.go @@ -197,6 +197,31 @@ func TestEnsureTraceHeightAvailable(t *testing.T) { require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 100), "has been pruned") } +func TestEnsureTraceHeightAvailableParentBlockFloor(t *testing.T) { + t.Parallel() + + const ( + blockFloor = int64(150) + latestHeight = int64(200) + ) + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{ + SyncInfo: coretypes.SyncInfo{ + LatestBlockHeight: latestHeight, + EarliestBlockHeight: blockFloor, + }, + }, + } + stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} + rs := &fakeReceiptStore{latest: latestHeight, earliest: 1} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(latestHeight), stateStore, rs) + + require.NoError(t, wm.EnsureBlockHeightAvailable(t.Context(), blockFloor)) + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), blockFloor), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), blockFloor)) + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), blockFloor+1)) +} + func TestEnsureTraceHeightAvailableSkipsStateWhenSSDisabled(t *testing.T) { tmClient := &fakeTMClient{ status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, From 25f0527abd520d598ea40ca0267669a9671ae8a8 Mon Sep 17 00:00:00 2001 From: Amir Deris Date: Wed, 12 Aug 2026 14:09:31 +0200 Subject: [PATCH 7/7] Fixed error propogation issue --- evmrpc/historical_debug_trace_test.go | 65 +++++++++++++++++++++++++++ evmrpc/tracers.go | 2 +- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/evmrpc/historical_debug_trace_test.go b/evmrpc/historical_debug_trace_test.go index 26c8554fb5..731894269a 100644 --- a/evmrpc/historical_debug_trace_test.go +++ b/evmrpc/historical_debug_trace_test.go @@ -2,12 +2,17 @@ package evmrpc import ( "context" + "errors" + "fmt" "testing" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rpc" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" + "github.com/sei-protocol/sei-chain/x/evm/keeper" + evmtypes "github.com/sei-protocol/sei-chain/x/evm/types" "github.com/stretchr/testify/require" ) @@ -281,3 +286,63 @@ func TestTraceLatestTagGuardMatchesBlockResolution(t *testing.T) { require.NoError(t, err) require.Equal(t, guardHeight, tmBlock.Block.Height) } + +type traceGuardReceiptStore struct { + fakeReceiptStore + getReceiptErr error +} + +func (s *traceGuardReceiptStore) GetReceipt(_ sdk.Context, _ common.Hash) (*evmtypes.Receipt, error) { + if s.getReceiptErr != nil { + return nil, s.getReceiptErr + } + return nil, receipt.ErrNotFound +} + +func TestGuardTraceRequestByTxHashReceiptLookupErrors(t *testing.T) { + t.Parallel() + + const latestHeight = int64(10) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + txHash := common.HexToHash("0xabc") + storeErr := errors.New("receipt store unavailable") + + newAPI := func(store receipt.ReceiptStore) *DebugAPI { + k := &keeper.Keeper{} + k.SetReceiptStoreForTesting(store) + return &DebugAPI{ + keeper: k, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + maxBlockLookback: -1, + } + } + + t.Run("ErrReceiptPruned", func(t *testing.T) { + t.Parallel() + prunedErr := fmt.Errorf("requested height 100 receipts have been pruned; earliest available is 150: %w", receipt.ErrReceiptPruned) + api := newAPI(&traceGuardReceiptStore{getReceiptErr: prunedErr}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.ErrorIs(t, err, receipt.ErrReceiptPruned) + }) + + t.Run("store error", func(t *testing.T) { + t.Parallel() + api := newAPI(&traceGuardReceiptStore{getReceiptErr: storeErr}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.ErrorIs(t, err, storeErr) + }) + + t.Run("ErrNotFound falls through to lookback", func(t *testing.T) { + t.Parallel() + api := newAPI(&traceGuardReceiptStore{getReceiptErr: receipt.ErrNotFound}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.NoError(t, err) + }) + + t.Run("ErrNotConfigured", func(t *testing.T) { + t.Parallel() + api := newAPI(&traceGuardReceiptStore{getReceiptErr: receipt.ErrNotConfigured}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.ErrorIs(t, err, receipt.ErrNotConfigured) + }) +} diff --git a/evmrpc/tracers.go b/evmrpc/tracers.go index d41f36ec5a..2ffddbae76 100644 --- a/evmrpc/tracers.go +++ b/evmrpc/tracers.go @@ -115,7 +115,7 @@ func (api *DebugAPI) guardTraceRequestByTxHash(ctx context.Context, endpoint str if api.keeper != nil { rcpt, err := api.keeper.GetReceipt(api.ctxProvider(LatestCtxHeight), hash) if err != nil { - if errors.Is(err, receipt.ErrReceiptPruned) { + if errors.Is(err, receipt.ErrReceiptPruned) || !errors.Is(err, receipt.ErrNotFound) { return err } } else if rcpt != nil {