diff --git a/evmrpc/historical_debug_trace_test.go b/evmrpc/historical_debug_trace_test.go index a9a97d8271..731894269a 100644 --- a/evmrpc/historical_debug_trace_test.go +++ b/evmrpc/historical_debug_trace_test.go @@ -2,10 +2,17 @@ package evmrpc import ( "context" + "errors" + "fmt" "testing" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" + "github.com/sei-protocol/sei-chain/x/evm/keeper" + evmtypes "github.com/sei-protocol/sei-chain/x/evm/types" "github.com/stretchr/testify/require" ) @@ -93,24 +100,249 @@ func TestGuardHistoricalDebugTraceHeight(t *testing.T) { require.Contains(t, err.Error(), "block number 9 is beyond max lookback of 0") } -func TestGuardHistoricalDebugTraceByHashUsesTendermintHeight(t *testing.T) { +func TestGuardTraceRequestByHashUsesTendermintHeight(t *testing.T) { latestHeight := int64(10) latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) tmClient := newHeightTestClient(8, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 1} api := &DebugAPI{ tmClient: tmClient, ctxProvider: func(int64) sdk.Context { return latestCtx }, connectionType: ConnectionTypeHTTP, maxBlockLookback: 1, backend: &Backend{ - watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, &fakeReceiptStore{latest: latestHeight}), + watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs), }, } - err := api.guardHistoricalDebugTraceByHash(context.Background(), "debug_traceBlockByHash", common.HexToHash(highBlockHashHex)) + err := api.guardTraceRequestByHash(t.Context(), "debug_traceBlockByHash", common.HexToHash(highBlockHashHex)) require.Error(t, err) require.Contains(t, err.Error(), "block number 8 is beyond max lookback of 1") - err = api.guardHistoricalDebugTraceByHash(context.Background(), "debug_traceCall", common.HexToHash("0x1")) + err = api.guardTraceCallRequestByHash(t.Context(), "debug_traceCall", common.HexToHash("0x1")) require.NoError(t, err) } + +func TestEnsureTraceCallHeightAvailableIgnoresReceipts(t *testing.T) { + t.Parallel() + + parentHeight := int64(149) + receiptFloor := int64(150) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(parentHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: receiptFloor} + stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), parentHeight), "receipts have been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), parentHeight)) +} + +func TestEnsureTraceCallHeightAvailableUsesStateAtHeight(t *testing.T) { + t.Parallel() + + stateFloor := int64(150) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(stateFloor, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 1} + stateStore := &fakeStateStore{latest: latestHeight, earliest: stateFloor} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), stateFloor), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), stateFloor)) +} + +func TestEnsureTraceHeightAvailableReceiptsPruned(t *testing.T) { + t.Parallel() + + client := newHeightTestClient(100, 1, 200) + rs := &fakeReceiptStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(client, watermarkTestCtxProvider(200), nil, rs) + + err := wm.EnsureTraceHeightAvailable(t.Context(), 100) + require.Error(t, err) + require.Contains(t, err.Error(), "receipts have been pruned") +} + +func TestEnsureTraceHeightAvailableStatePruned(t *testing.T) { + t.Parallel() + + client := newHeightTestClient(100, 1, 200) + stateStore := &fakeStateStore{latest: 200, earliest: 150} + rs := &fakeReceiptStore{latest: 200, earliest: 1} + wm := NewWatermarkManager(client, watermarkTestCtxProvider(200), stateStore, rs) + + err := wm.EnsureTraceHeightAvailable(t.Context(), 100) + require.Error(t, err) + require.Contains(t, err.Error(), "has been pruned") +} + +func TestTraceReceiptFloorBoundary(t *testing.T) { + t.Parallel() + + // Receipt retention starts at 150; block 149 still exists in Tendermint but + // its receipts are gone. Trace guard applies at the requested height only — + // parent fetches at height-1 for state replay must not re-run it. + parentHeight := int64(149) + receiptFloor := int64(150) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(parentHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: receiptFloor} + stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), parentHeight), "receipts have been pruned") + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), receiptFloor)) +} + +func TestTraceBlockByNumberReceiptPrunedBeforeSemaphore(t *testing.T) { + t.Parallel() + + prunedHeight := int64(100) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(prunedHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 150} + api := &DebugAPI{ + tmClient: tmClient, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + connectionType: ConnectionTypeHTTP, + maxBlockLookback: -1, + traceCallSemaphore: make(chan struct{}, 1), + traceTimeout: time.Second, + backend: &Backend{ + tmClient: tmClient, + watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs), + }, + } + + api.traceCallSemaphore <- struct{}{} + defer func() { <-api.traceCallSemaphore }() + + _, err := api.TraceBlockByNumber(t.Context(), rpc.BlockNumber(prunedHeight), nil) + require.Error(t, err) + require.Contains(t, err.Error(), "receipts have been pruned") + require.NotErrorIs(t, err, errTraceConcurrencyLimit) +} + +func TestBlockByNumberLatestUsesSafeLatestWatermark(t *testing.T) { + t.Parallel() + + const ( + safeLatest = int64(99) + ctxTip = int64(100) + ) + latestCtx := sdk.Context{}.WithBlockHeight(ctxTip) + tmClient := newHeightTestClient(safeLatest, 1, ctxTip) + rs := &fakeReceiptStore{latest: safeLatest, earliest: 1} + stateStore := &fakeStateStore{latest: safeLatest, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + + blockNumberPtr, err := getBlockNumber(t.Context(), tmClient, rpc.LatestBlockNumber) + require.NoError(t, err) + require.Nil(t, blockNumberPtr) + + tmBlock, err := blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, blockNumberPtr, 1) + require.NoError(t, err) + require.Equal(t, safeLatest, tmBlock.Block.Height) + + rawTip := ctxTip + _, err = blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, &rawTip, 1) + require.Error(t, err) + require.ErrorIs(t, err, ErrBlockHeightNotYetAvailable) +} + +func TestTraceLatestTagGuardMatchesBlockResolution(t *testing.T) { + t.Parallel() + + const ( + safeLatest = int64(99) + ctxTip = int64(100) + ) + latestCtx := sdk.Context{}.WithBlockHeight(ctxTip) + tmClient := newHeightTestClient(safeLatest, 1, ctxTip) + rs := &fakeReceiptStore{latest: safeLatest, earliest: 1} + stateStore := &fakeStateStore{latest: safeLatest, earliest: 1} + wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs) + api := &DebugAPI{ + tmClient: tmClient, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + backend: &Backend{ + tmClient: tmClient, + watermarks: wm, + }, + } + + guardHeight, err := api.resolveDebugTraceBlockNumber(t.Context(), rpc.LatestBlockNumber) + require.NoError(t, err) + require.Equal(t, safeLatest, guardHeight) + + blockNumberPtr, err := getBlockNumber(t.Context(), tmClient, rpc.LatestBlockNumber) + require.NoError(t, err) + tmBlock, err := blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, blockNumberPtr, 1) + require.NoError(t, err) + require.Equal(t, guardHeight, tmBlock.Block.Height) +} + +type traceGuardReceiptStore struct { + fakeReceiptStore + getReceiptErr error +} + +func (s *traceGuardReceiptStore) GetReceipt(_ sdk.Context, _ common.Hash) (*evmtypes.Receipt, error) { + if s.getReceiptErr != nil { + return nil, s.getReceiptErr + } + return nil, receipt.ErrNotFound +} + +func TestGuardTraceRequestByTxHashReceiptLookupErrors(t *testing.T) { + t.Parallel() + + const latestHeight = int64(10) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + txHash := common.HexToHash("0xabc") + storeErr := errors.New("receipt store unavailable") + + newAPI := func(store receipt.ReceiptStore) *DebugAPI { + k := &keeper.Keeper{} + k.SetReceiptStoreForTesting(store) + return &DebugAPI{ + keeper: k, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + maxBlockLookback: -1, + } + } + + t.Run("ErrReceiptPruned", func(t *testing.T) { + t.Parallel() + prunedErr := fmt.Errorf("requested height 100 receipts have been pruned; earliest available is 150: %w", receipt.ErrReceiptPruned) + api := newAPI(&traceGuardReceiptStore{getReceiptErr: prunedErr}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.ErrorIs(t, err, receipt.ErrReceiptPruned) + }) + + t.Run("store error", func(t *testing.T) { + t.Parallel() + api := newAPI(&traceGuardReceiptStore{getReceiptErr: storeErr}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.ErrorIs(t, err, storeErr) + }) + + t.Run("ErrNotFound falls through to lookback", func(t *testing.T) { + t.Parallel() + api := newAPI(&traceGuardReceiptStore{getReceiptErr: receipt.ErrNotFound}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.NoError(t, err) + }) + + t.Run("ErrNotConfigured", func(t *testing.T) { + t.Parallel() + api := newAPI(&traceGuardReceiptStore{getReceiptErr: receipt.ErrNotConfigured}) + err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash) + require.ErrorIs(t, err, receipt.ErrNotConfigured) + }) +} diff --git a/evmrpc/simulate.go b/evmrpc/simulate.go index dc05696a2c..b58736cb3e 100644 --- a/evmrpc/simulate.go +++ b/evmrpc/simulate.go @@ -410,26 +410,12 @@ func (b *Backend) ChainDb() ethdb.Database { panic("implement me") } -func (b Backend) ConvertBlockNumber(bn rpc.BlockNumber) int64 { - blockNum := bn.Int64() - switch blockNum { - case rpc.SafeBlockNumber.Int64(), rpc.FinalizedBlockNumber.Int64(), rpc.LatestBlockNumber.Int64(): - blockNum = b.ctxProvider(LatestCtxHeight).BlockHeight() - case rpc.EarliestBlockNumber.Int64(): - genesisRes, err := b.tmClient.Genesis(context.Background()) - if err != nil { - panic("could not get genesis info from tendermint") - } - blockNum = genesisRes.Genesis.InitialHeight - case rpc.PendingBlockNumber.Int64(): - panic("tracing on pending block is not supported") - } - return blockNum -} - func (b Backend) BlockByNumber(ctx context.Context, bn rpc.BlockNumber) (*ethtypes.Block, []tracersutils.TraceBlockMetadata, error) { - blockNum := b.ConvertBlockNumber(bn) - tmBlock, err := blockByNumberRespectingWatermarks(ctx, b.tmClient, b.watermarks, &blockNum, 1) + blockNumberPtr, err := getBlockNumber(ctx, b.tmClient, bn) + if err != nil { + return nil, nil, err + } + tmBlock, err := blockByNumberRespectingWatermarks(ctx, b.tmClient, b.watermarks, blockNumberPtr, 1) if err != nil { return nil, nil, err } diff --git a/evmrpc/simulate_test.go b/evmrpc/simulate_test.go index 27f19c98d5..9b5d0a0909 100644 --- a/evmrpc/simulate_test.go +++ b/evmrpc/simulate_test.go @@ -379,27 +379,6 @@ func TestNewRevertError(t *testing.T) { require.Equal(t, "0x", err.ErrorData()) } -func TestConvertBlockNumber(t *testing.T) { - tmClient := &MockClient{} - watermarks := evmrpc.NewWatermarkManager(tmClient, func(i int64) sdk.Context { - if i == evmrpc.LatestCtxHeight { - return sdk.Context{}.WithBlockHeight(1000) - } - return sdk.Context{} - }, nil, nil) - backend := evmrpc.NewBackend(func(i int64) sdk.Context { - if i == evmrpc.LatestCtxHeight { - return sdk.Context{}.WithBlockHeight(1000) - } - return sdk.Context{} - }, nil, legacyabci.BeginBlockKeepers{}, nil, &MockClient{}, nil, nil, nil, evmrpc.NewBlockCache(3000), &sync.Mutex{}, watermarks) - require.Equal(t, int64(10), backend.ConvertBlockNumber(10)) - require.Equal(t, int64(1), backend.ConvertBlockNumber(0)) - require.Equal(t, int64(1000), backend.ConvertBlockNumber(-2)) - require.Equal(t, int64(1000), backend.ConvertBlockNumber(-3)) - require.Equal(t, int64(1000), backend.ConvertBlockNumber(-4)) -} - func TestPreV620UpgradeUsesBaseFeeNil(t *testing.T) { // Set up a test context with a height before v6.2.0 upgrade // For pacific-1 chain, we need to set a height that's before the v6.2.0 upgrade diff --git a/evmrpc/trace_db_reader_test.go b/evmrpc/trace_db_reader_test.go index b557077821..3252052c5e 100644 --- a/evmrpc/trace_db_reader_test.go +++ b/evmrpc/trace_db_reader_test.go @@ -82,10 +82,10 @@ func TestBlockTraceCacheGet(t *testing.T) { require.Nil(t, got) }) - t.Run("empty block -> empty hit", func(t *testing.T) { + t.Run("empty tx list is not a cache hit", func(t *testing.T) { got, ok := blockTraceCacheGet(c, 5, []common.Hash{}, cfg) - require.True(t, ok) - require.Empty(t, got) + require.False(t, ok, "empty tx list must not report cache hit") + require.Nil(t, got) }) } diff --git a/evmrpc/trace_profile.go b/evmrpc/trace_profile.go index 244836c5c6..2a2df33214 100644 --- a/evmrpc/trace_profile.go +++ b/evmrpc/trace_profile.go @@ -51,7 +51,7 @@ func (api *DebugAPI) TraceTransactionProfile(ctx context.Context, hash common.Ha if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } - if returnErr = api.guardHistoricalDebugTraceByTxHash(ctx, "debug_traceTransactionProfile", hash); returnErr != nil { + if returnErr = api.guardTraceRequestByTxHash(ctx, "debug_traceTransactionProfile", hash); returnErr != nil { return nil, returnErr } diff --git a/evmrpc/tracers.go b/evmrpc/tracers.go index 9afaf9ccfc..2ffddbae76 100644 --- a/evmrpc/tracers.go +++ b/evmrpc/tracers.go @@ -26,6 +26,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-cosmos/baseapp" "github.com/sei-protocol/sei-chain/sei-cosmos/client" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" + "github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt" "github.com/sei-protocol/sei-chain/x/evm/keeper" "github.com/sei-protocol/sei-chain/x/evm/state" ) @@ -101,50 +102,119 @@ func (api *DebugAPI) prepareTraceContext(ctx context.Context) (context.Context, }, nil } -func (api *DebugAPI) guardHistoricalDebugTraceByTxHash(ctx context.Context, endpoint string, hash common.Hash) error { - if api.keeper == nil { - return nil +func (api *DebugAPI) guardTraceRequest(ctx context.Context, endpoint string, height int64) error { + if api.backend != nil && api.backend.watermarks != nil { + if err := api.backend.watermarks.EnsureTraceHeightAvailable(ctx, height); err != nil { + return err + } } - receipt, err := api.keeper.GetReceipt(api.ctxProvider(LatestCtxHeight), hash) - if err != nil || receipt == nil { - return nil + return api.guardHistoricalDebugTraceHeight(ctx, endpoint, height) +} + +func (api *DebugAPI) guardTraceRequestByTxHash(ctx context.Context, endpoint string, hash common.Hash) error { + if api.keeper != nil { + rcpt, err := api.keeper.GetReceipt(api.ctxProvider(LatestCtxHeight), hash) + if err != nil { + if errors.Is(err, receipt.ErrReceiptPruned) || !errors.Is(err, receipt.ErrNotFound) { + return err + } + } else if rcpt != nil { + return api.guardTraceRequest(ctx, endpoint, int64(rcpt.BlockNumber)) //nolint:gosec + } } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, int64(receipt.BlockNumber)) //nolint:gosec + return api.guardHistoricalDebugTraceHeight(ctx, endpoint, api.latestTraceHeight(ctx)) } -func (api *DebugAPI) guardHistoricalDebugTraceByNumber(ctx context.Context, endpoint string, number rpc.BlockNumber) error { +// latestTraceHeight resolves the height debug_trace* should use for latest-ish +// tags. It prefers the watermark's safe latest over the raw app tip, since the +// tip can outrun the receipt/state stores by a block or so and would otherwise +// make EnsureTraceHeightAvailable reject the most common trace requests. +func (api *DebugAPI) latestTraceHeight(ctx context.Context) int64 { + if api.backend != nil && api.backend.watermarks != nil { + if latest, err := api.backend.watermarks.LatestHeight(ctx); err == nil { + return latest + } + } + return api.ctxProvider(LatestCtxHeight).BlockHeight() +} + +func (api *DebugAPI) guardTraceRequestByNumber(ctx context.Context, endpoint string, number rpc.BlockNumber) error { height, err := api.resolveDebugTraceBlockNumber(ctx, number) if err != nil { return err } + return api.guardTraceRequest(ctx, endpoint, height) +} + +func (api *DebugAPI) guardTraceRequestByHash(ctx context.Context, endpoint string, hash common.Hash) error { + if api.backend == nil || api.tmClient == nil { + return api.guardTraceRequest(ctx, endpoint, api.latestTraceHeight(ctx)) + } + block, err := blockByHashRespectingWatermarks(ctx, api.tmClient, api.backend.watermarks, hash.Bytes(), 1) + if err != nil { + return err + } + if block == nil || block.Block == nil { + return fmt.Errorf("block %s not found", hash.Hex()) + } + return api.guardTraceRequest(ctx, endpoint, block.Block.Height) +} + +func (api *DebugAPI) guardTraceRequestByNumberOrHash(ctx context.Context, endpoint string, blockNrOrHash rpc.BlockNumberOrHash) error { + if number, ok := blockNrOrHash.Number(); ok { + return api.guardTraceRequestByNumber(ctx, endpoint, number) + } + if hash, ok := blockNrOrHash.Hash(); ok { + return api.guardTraceRequestByHash(ctx, endpoint, hash) + } + return api.guardTraceRequest(ctx, endpoint, api.latestTraceHeight(ctx)) +} + +func (api *DebugAPI) guardTraceCallRequest(ctx context.Context, endpoint string, height int64) error { + if api.backend != nil && api.backend.watermarks != nil { + if err := api.backend.watermarks.EnsureTraceCallHeightAvailable(ctx, height); err != nil { + return err + } + } return api.guardHistoricalDebugTraceHeight(ctx, endpoint, height) } -func (api *DebugAPI) guardHistoricalDebugTraceByHash(ctx context.Context, endpoint string, hash common.Hash) error { +func (api *DebugAPI) guardTraceCallRequestByNumber(ctx context.Context, endpoint string, number rpc.BlockNumber) error { + height, err := api.resolveDebugTraceBlockNumber(ctx, number) + if err != nil { + return err + } + return api.guardTraceCallRequest(ctx, endpoint, height) +} + +func (api *DebugAPI) guardTraceCallRequestByHash(ctx context.Context, endpoint string, hash common.Hash) error { if api.backend == nil || api.tmClient == nil { - return nil + return api.guardTraceCallRequest(ctx, endpoint, api.latestTraceHeight(ctx)) } block, err := blockByHashRespectingWatermarks(ctx, api.tmClient, api.backend.watermarks, hash.Bytes(), 1) - if err != nil || block == nil || block.Block == nil { - return nil + if err != nil { + return err } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, block.Block.Height) + if block == nil || block.Block == nil { + return fmt.Errorf("block %s not found", hash.Hex()) + } + return api.guardTraceCallRequest(ctx, endpoint, block.Block.Height) } -func (api *DebugAPI) guardHistoricalDebugTraceByNumberOrHash(ctx context.Context, endpoint string, blockNrOrHash rpc.BlockNumberOrHash) error { +func (api *DebugAPI) guardTraceCallRequestByNumberOrHash(ctx context.Context, endpoint string, blockNrOrHash rpc.BlockNumberOrHash) error { if number, ok := blockNrOrHash.Number(); ok { - return api.guardHistoricalDebugTraceByNumber(ctx, endpoint, number) + return api.guardTraceCallRequestByNumber(ctx, endpoint, number) } if hash, ok := blockNrOrHash.Hash(); ok { - return api.guardHistoricalDebugTraceByHash(ctx, endpoint, hash) + return api.guardTraceCallRequestByHash(ctx, endpoint, hash) } - return api.guardHistoricalDebugTraceHeight(ctx, endpoint, api.ctxProvider(LatestCtxHeight).BlockHeight()) + return api.guardTraceCallRequest(ctx, endpoint, api.latestTraceHeight(ctx)) } func (api *DebugAPI) resolveDebugTraceBlockNumber(ctx context.Context, number rpc.BlockNumber) (int64, error) { switch number { case rpc.SafeBlockNumber, rpc.FinalizedBlockNumber, rpc.LatestBlockNumber, rpc.PendingBlockNumber: - return api.ctxProvider(LatestCtxHeight).BlockHeight(), nil + return api.latestTraceHeight(ctx), nil case rpc.EarliestBlockNumber: if api.tmClient == nil { return 0, errors.New("tendermint client is not configured") @@ -331,7 +401,7 @@ func (api *DebugAPI) TraceTransaction(ctx context.Context, hash common.Hash, con if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } - if returnErr = api.guardHistoricalDebugTraceByTxHash(ctx, "debug_traceTransaction", hash); returnErr != nil { + if returnErr = api.guardTraceRequestByTxHash(ctx, "debug_traceTransaction", hash); returnErr != nil { return nil, returnErr } @@ -374,7 +444,7 @@ func (api *DebugAPI) tryTraceCache(hash common.Hash, config *tracers.TraceConfig // blockTraceCacheGet assembles a per-tx hit; returns (nil, false) if any miss. func blockTraceCacheGet(cache *keeper.TraceDB, height int64, txHashes []common.Hash, config *tracers.TraceConfig) ([]*tracers.TxTraceResult, bool) { - if cache == nil { + if cache == nil || len(txHashes) == 0 { return nil, false } name := bakeableTracerName(config) @@ -544,7 +614,7 @@ func (api *DebugAPI) TraceBlockByNumber(ctx context.Context, number rpc.BlockNum if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } - if returnErr = api.guardHistoricalDebugTraceByNumber(ctx, "debug_traceBlockByNumber", number); returnErr != nil { + if returnErr = api.guardTraceRequestByNumber(ctx, "debug_traceBlockByNumber", number); returnErr != nil { return nil, returnErr } @@ -579,6 +649,9 @@ func (api *DebugAPI) TraceBlockByHash(ctx context.Context, hash common.Hash, con if returnErr = api.validateTraceTracer(config); returnErr != nil { return nil, returnErr } + if returnErr = api.guardTraceRequestByHash(ctx, "debug_traceBlockByHash", hash); returnErr != nil { + return nil, returnErr + } ctx, done, err := api.prepareTraceContext(ctx) if err != nil { @@ -586,10 +659,6 @@ func (api *DebugAPI) TraceBlockByHash(ctx context.Context, hash common.Hash, con } defer done() - if returnErr = api.guardHistoricalDebugTraceByHash(ctx, "debug_traceBlockByHash", hash); returnErr != nil { - return nil, returnErr - } - if cached, ok := api.tryBlockTraceCacheByHash(ctx, hash, config); ok { return cached, nil } @@ -618,6 +687,9 @@ func (api *DebugAPI) TraceCall(ctx context.Context, args export.TransactionArgs, if returnErr = api.validateTraceTracer(&config.TraceConfig); returnErr != nil { return nil, returnErr } + if returnErr = api.guardTraceCallRequestByNumberOrHash(ctx, "debug_traceCall", blockNrOrHash); returnErr != nil { + return nil, returnErr + } ctx, done, err := api.prepareTraceContext(ctx) if err != nil { @@ -625,10 +697,6 @@ func (api *DebugAPI) TraceCall(ctx context.Context, args export.TransactionArgs, } defer done() - if returnErr = api.guardHistoricalDebugTraceByNumberOrHash(ctx, "debug_traceCall", blockNrOrHash); returnErr != nil { - return nil, returnErr - } - if returnErr = validateStateOverrides(config.StateOverrides, api.backend.MaxStateOverrideAccounts(), api.backend.MaxStateOverrideSlots()); returnErr != nil { return nil, returnErr } @@ -683,7 +751,7 @@ func (api *DebugAPI) TraceStateAccess(ctx context.Context, hash common.Hash) (re returnErr = fmt.Errorf("panic occurred: %v, could not trace tx state: %s", r, hash.Hex()) } }() - if returnErr = api.guardHistoricalDebugTraceByTxHash(ctx, "debug_traceStateAccess", hash); returnErr != nil { + if returnErr = api.guardTraceRequestByTxHash(ctx, "debug_traceStateAccess", hash); returnErr != nil { return nil, returnErr } diff --git a/evmrpc/tracers_semaphore_test.go b/evmrpc/tracers_semaphore_test.go index efd2db45fe..39c1be7192 100644 --- a/evmrpc/tracers_semaphore_test.go +++ b/evmrpc/tracers_semaphore_test.go @@ -6,11 +6,8 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/export" "github.com/ethereum/go-ethereum/rpc" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" - tmbytes "github.com/sei-protocol/sei-chain/sei-tendermint/libs/bytes" - "github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes" "github.com/stretchr/testify/require" ) @@ -89,20 +86,12 @@ func TestAcquireTraceSemaphoreCanceledContextDoesNotConsumeSlot(t *testing.T) { } } -type panicHashLookupClient struct { - *heightTestClient -} - -func (c *panicHashLookupClient) BlockByHash(context.Context, tmbytes.HexBytes) (*coretypes.ResultBlock, error) { - panic("hash lookup should not happen before trace context setup") -} +func TestTraceBlockByNumberRejectsConcurrencyLimitAfterGuard(t *testing.T) { + t.Parallel() -func TestHashBasedTraceEndpointsAcquireSemaphoreBeforeHashLookup(t *testing.T) { latestHeight := int64(10) latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) - tmClient := &panicHashLookupClient{ - heightTestClient: newHeightTestClient(8, 1, latestHeight), - } + tmClient := newHeightTestClient(8, 1, latestHeight) watermarks := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, &fakeReceiptStore{latest: latestHeight}) api := &DebugAPI{ tmClient: tmClient, @@ -119,11 +108,37 @@ func TestHashBasedTraceEndpointsAcquireSemaphoreBeforeHashLookup(t *testing.T) { api.traceCallSemaphore <- struct{}{} defer func() { <-api.traceCallSemaphore }() - hash := common.HexToHash(highBlockHashHex) - _, err := api.TraceBlockByHash(context.Background(), hash, nil) + _, err := api.TraceBlockByNumber(context.Background(), rpc.LatestBlockNumber, nil) require.ErrorIs(t, err, errTraceConcurrencyLimit) +} - blockNrOrHash := rpc.BlockNumberOrHashWithHash(hash, false) - _, err = api.TraceCall(context.Background(), export.TransactionArgs{}, blockNrOrHash, nil) - require.ErrorIs(t, err, errTraceConcurrencyLimit) +func TestTraceBlockByHashChecksAvailabilityBeforeSemaphore(t *testing.T) { + t.Parallel() + + prunedHeight := int64(100) + latestHeight := int64(200) + latestCtx := sdk.Context{}.WithBlockHeight(latestHeight) + tmClient := newHeightTestClient(prunedHeight, 1, latestHeight) + rs := &fakeReceiptStore{latest: latestHeight, earliest: 150} + watermarks := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs) + api := &DebugAPI{ + tmClient: tmClient, + ctxProvider: func(int64) sdk.Context { return latestCtx }, + connectionType: ConnectionTypeHTTP, + traceCallSemaphore: make(chan struct{}, 1), + traceTimeout: time.Second, + backend: &Backend{ + tmClient: tmClient, + watermarks: watermarks, + }, + } + + api.traceCallSemaphore <- struct{}{} + defer func() { <-api.traceCallSemaphore }() + + hash := common.HexToHash(highBlockHashHex) + _, err := api.TraceBlockByHash(context.Background(), hash, nil) + require.Error(t, err) + require.ErrorContains(t, err, "receipts have been pruned") + require.NotErrorIs(t, err, errTraceConcurrencyLimit) } diff --git a/evmrpc/utils.go b/evmrpc/utils.go index 2037fdb447..c868bbe502 100644 --- a/evmrpc/utils.go +++ b/evmrpc/utils.go @@ -229,6 +229,9 @@ func filterTransactions( continue } ethtx, _ := m.AsTransaction() + if ethtx == nil { + continue + } hash := ethtx.Hash() sender, _ := rpcutils.RecoverEVMSender(ethtx, block.Block.Height, block.Block.Time.Unix()) receipt, found := getOrSetCachedReceipt(cacheCreationMutex, globalBlockCache, latestCtx, k, block, hash) diff --git a/evmrpc/watermark_manager.go b/evmrpc/watermark_manager.go index 959946d09d..dadddc7114 100644 --- a/evmrpc/watermark_manager.go +++ b/evmrpc/watermark_manager.go @@ -184,6 +184,59 @@ func (m *WatermarkManager) EnsureReceiptHeightAvailable(height int64) error { return nil } +// EnsureStateHeightAvailable verifies that historical state for the given block +// height has not been pruned from the state store. +func (m *WatermarkManager) EnsureStateHeightAvailable(ctx context.Context, height int64) error { + _, stateEarliest, latest, err := m.Watermarks(ctx) + if err != nil { + return err + } + return ensureWithinWatermarks(height, stateEarliest, latest) +} + +// EnsureTraceCallHeightAvailable verifies block and state availability for +// debug_traceCall. TraceCall loads state at the requested height via +// StateAndHeaderByNumberOrHash and never reads receipts. +func (m *WatermarkManager) EnsureTraceCallHeightAvailable(ctx context.Context, height int64) error { + if err := m.EnsureBlockHeightAvailable(ctx, height); err != nil { + return err + } + if m.stateStore == nil { + // SS disabled: trace replay uses SC via ctxProvider, not SS retention. + return nil + } + return m.EnsureStateHeightAvailable(ctx, height) +} + +// EnsureTraceHeightAvailable verifies block, receipt, and state availability +// for debug_trace* replay endpoints (transaction/block). Replay loads parent +// block validators and parent state (height-1) and reads receipts at the +// requested height. +func (m *WatermarkManager) EnsureTraceHeightAvailable(ctx context.Context, height int64) error { + if err := m.EnsureBlockHeightAvailable(ctx, height); err != nil { + return err + } + if err := m.ensureReplayParentBlockAvailable(ctx, height); err != nil { + return err + } + if err := m.EnsureReceiptHeightAvailable(height); err != nil { + return err + } + if m.stateStore == nil { + // SS disabled: trace replay uses SC via ctxProvider, not SS retention. + return nil + } + stateHeight := max(height-1, m.genesisInitialHeight()) + return m.EnsureStateHeightAvailable(ctx, stateHeight) +} + +// ensureReplayParentBlockAvailable verifies the parent block height replay +// tracing loads for validator set lookup in initializeBlock. +func (m *WatermarkManager) ensureReplayParentBlockAvailable(ctx context.Context, height int64) error { + parentBlockHeight := max(height-1, 0) + return m.EnsureBlockHeightAvailable(ctx, parentBlockHeight) +} + func ensureWithinWatermarks(height, earliest, latest int64) error { if height > latest { return fmt.Errorf("requested height %d is not yet available; safe latest is %d: %w", height, latest, ErrBlockHeightNotYetAvailable) diff --git a/evmrpc/watermark_manager_test.go b/evmrpc/watermark_manager_test.go index 7cc3f633fa..2c393e8cc7 100644 --- a/evmrpc/watermark_manager_test.go +++ b/evmrpc/watermark_manager_test.go @@ -134,6 +134,118 @@ func TestEnsureReceiptHeightAvailable(t *testing.T) { }) } +func TestEnsureStateHeightAvailable(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + + t.Run("pruned state height returns error", func(t *testing.T) { + stateStore := &fakeStateStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, &fakeReceiptStore{latest: 200}) + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 100), "has been pruned") + }) + + t.Run("height within state retention succeeds", func(t *testing.T) { + stateStore := &fakeStateStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, &fakeReceiptStore{latest: 200}) + require.NoError(t, wm.EnsureStateHeightAvailable(t.Context(), 150)) + }) + + t.Run("nil state store uses latest as earliest from Watermarks", func(t *testing.T) { + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), nil, &fakeReceiptStore{latest: 200}) + require.NoError(t, wm.EnsureStateHeightAvailable(t.Context(), 200)) + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 199), "has been pruned") + }) +} + +func TestEnsureTraceCallHeightAvailable(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + stateStore := &fakeStateStore{latest: 200, earliest: 150} + rs := &fakeReceiptStore{latest: 200, earliest: 150} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, rs) + + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 175)) + + // Receipts pruned below 150; replay guard fails, TraceCall guard does not check receipts. + rs.earliest = 150 + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 149), "receipts have been pruned") + stateStore.earliest = 1 + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 149)) + + stateStore.earliest = 175 + require.ErrorContains(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 100), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 175)) +} + +func TestEnsureTraceHeightAvailable(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + stateStore := &fakeStateStore{latest: 200, earliest: 1} + rs := &fakeReceiptStore{latest: 200, earliest: 1} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), stateStore, rs) + + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), 175)) + + rs.earliest = 150 + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 100), "receipts have been pruned") + + rs.earliest = 1 + stateStore.earliest = 150 + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), 100), "has been pruned") +} + +func TestEnsureTraceHeightAvailableParentBlockFloor(t *testing.T) { + t.Parallel() + + const ( + blockFloor = int64(150) + latestHeight = int64(200) + ) + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{ + SyncInfo: coretypes.SyncInfo{ + LatestBlockHeight: latestHeight, + EarliestBlockHeight: blockFloor, + }, + }, + } + stateStore := &fakeStateStore{latest: latestHeight, earliest: 1} + rs := &fakeReceiptStore{latest: latestHeight, earliest: 1} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(latestHeight), stateStore, rs) + + require.NoError(t, wm.EnsureBlockHeightAvailable(t.Context(), blockFloor)) + require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), blockFloor), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), blockFloor)) + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), blockFloor+1)) +} + +func TestEnsureTraceHeightAvailableSkipsStateWhenSSDisabled(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + rs := &fakeReceiptStore{latest: 200, earliest: 1} + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), nil, rs) + + // EnsureStateHeightAvailable rejects below tip when SS is disabled. + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 199), "has been pruned") + // Trace guards skip the SS leg and defer state to SC/ctxProvider. + require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), 199)) + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 199)) +} + +func TestEnsureTraceCallHeightAvailableSkipsStateWhenSSDisabled(t *testing.T) { + tmClient := &fakeTMClient{ + status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 200, EarliestBlockHeight: 1}}, + } + wm := NewWatermarkManager(tmClient, watermarkTestCtxProvider(200), nil, &fakeReceiptStore{latest: 200}) + + require.ErrorContains(t, wm.EnsureStateHeightAvailable(t.Context(), 100), "has been pruned") + require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), 100)) +} + func TestLatestAndEarliestHeightHelpers(t *testing.T) { tmClient := &fakeTMClient{ status: &coretypes.ResultStatus{SyncInfo: coretypes.SyncInfo{LatestBlockHeight: 22, EarliestBlockHeight: 11}}, diff --git a/sei-db/ledger_db/receipt/litt_receipt_store.go b/sei-db/ledger_db/receipt/litt_receipt_store.go index 7ff821ddd7..bcace94621 100644 --- a/sei-db/ledger_db/receipt/litt_receipt_store.go +++ b/sei-db/ledger_db/receipt/litt_receipt_store.go @@ -244,6 +244,9 @@ func (s *littReceiptStore) GetReceipt(ctx sdk.Context, txHash common.Hash) (*typ if err == nil { return receipt, nil } + if errors.Is(err, ErrReceiptPruned) { + return nil, err + } if !errors.Is(err, ErrNotFound) { return nil, err } @@ -273,7 +276,9 @@ func (s *littReceiptStore) GetReceiptFromStore(_ sdk.Context, txHash common.Hash // Enforce the KeepRecent floor: litt expires values lazily via TTL, so a // pruned block may still be physically present. if s.belowRetentionFloor(r.BlockNumber) { - return nil, ErrNotFound + earliest := s.earliestVersion.Load() + return nil, fmt.Errorf("requested height %d receipts have been pruned; earliest available is %d: %w", + r.BlockNumber, earliest, ErrReceiptPruned) } return &r, nil } diff --git a/sei-db/ledger_db/receipt/littidx_test.go b/sei-db/ledger_db/receipt/littidx_test.go index d4f5fa23ad..72a1632867 100644 --- a/sei-db/ledger_db/receipt/littidx_test.go +++ b/sei-db/ledger_db/receipt/littidx_test.go @@ -332,7 +332,7 @@ func TestLittIdxPrune(t *testing.T) { // Pruned blocks are invisible (tag entries deleted, read floor enforced). for block := uint64(1); block <= 5; block++ { _, err := store.GetReceiptFromStore(ctx, litTxHash(block, 0)) - require.ErrorIs(t, err, receipt.ErrNotFound, "block %d should be pruned", block) + require.ErrorIs(t, err, receipt.ErrReceiptPruned, "block %d should be pruned", block) } logs, err := store.FilterLogs(ctx, 1, 5, filters.FilterCriteria{Addresses: []common.Address{addr}}, nil) require.NoError(t, err) diff --git a/sei-db/ledger_db/receipt/receipt_store.go b/sei-db/ledger_db/receipt/receipt_store.go index 378106a222..86779b1597 100644 --- a/sei-db/ledger_db/receipt/receipt_store.go +++ b/sei-db/ledger_db/receipt/receipt_store.go @@ -27,7 +27,12 @@ var logger = seilog.NewLogger("db", "ledger-db", "receipt") // Sentinel errors for consistent error checking. var ( - ErrNotFound = errors.New("receipt not found") + ErrNotFound = errors.New("receipt not found") + // ErrReceiptPruned is returned when a receipt existed but has fallen below + // the store's retention floor. It wraps ErrNotFound so existing not-found + // handling (e.g. eth_getTransactionReceipt returning null) still applies, + // while errors.Is(err, ErrReceiptPruned) lets trace guards react to it specifically. + ErrReceiptPruned = fmt.Errorf("receipt pruned: %w", ErrNotFound) ErrNotConfigured = errors.New("receipt store not configured") ErrRangeQueryNotSupported = errors.New("range query not supported by this backend") // ErrTooManyLogs is returned by FilterLogs when a query matches more logs