fix(evmrpc): fail fast on pruned heights for debug_trace* (PLT-975)

evmrpc/historical_debug_trace_test.go

@@ -2,10 +2,17 @@ package evmrpc
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
 	sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
+	"github.com/sei-protocol/sei-chain/sei-db/ledger_db/receipt"
+	"github.com/sei-protocol/sei-chain/x/evm/keeper"
+	evmtypes "github.com/sei-protocol/sei-chain/x/evm/types"
 	"github.com/stretchr/testify/require"
 )
 
@@ -93,24 +100,254 @@ func TestGuardHistoricalDebugTraceHeight(t *testing.T) {
 	require.Contains(t, err.Error(), "block number 9 is beyond max lookback of 0")
 }
 
-func TestGuardHistoricalDebugTraceByHashUsesTendermintHeight(t *testing.T) {
+func TestGuardTraceRequestByHashUsesTendermintHeight(t *testing.T) {
 	latestHeight := int64(10)
 	latestCtx := sdk.Context{}.WithBlockHeight(latestHeight)
 	tmClient := newHeightTestClient(8, 1, latestHeight)
+	rs := &fakeReceiptStore{latest: latestHeight, earliest: 1}
 	api := &DebugAPI{
 		tmClient:         tmClient,
 		ctxProvider:      func(int64) sdk.Context { return latestCtx },
 		connectionType:   ConnectionTypeHTTP,
 		maxBlockLookback: 1,
 		backend: &Backend{
-			watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, &fakeReceiptStore{latest: latestHeight}),
+			watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs),
 		},
 	}
 
-	err := api.guardHistoricalDebugTraceByHash(context.Background(), "debug_traceBlockByHash", common.HexToHash(highBlockHashHex))
+	err := api.guardTraceRequestByHash(t.Context(), "debug_traceBlockByHash", common.HexToHash(highBlockHashHex))
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "block number 8 is beyond max lookback of 1")
 
-	err = api.guardHistoricalDebugTraceByHash(context.Background(), "debug_traceCall", common.HexToHash("0x1"))
+	err = api.guardTraceCallRequestByHash(t.Context(), "debug_traceCall", common.HexToHash("0x1"))
 	require.NoError(t, err)
 }
+
+func TestEnsureTraceCallHeightAvailableIgnoresReceipts(t *testing.T) {
+	t.Parallel()
+
+	parentHeight := int64(149)
+	receiptFloor := int64(150)
+	latestHeight := int64(200)
+	latestCtx := sdk.Context{}.WithBlockHeight(latestHeight)
+	tmClient := newHeightTestClient(parentHeight, 1, latestHeight)
+	rs := &fakeReceiptStore{latest: latestHeight, earliest: receiptFloor}
+	stateStore := &fakeStateStore{latest: latestHeight, earliest: 1}
+	wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs)
+
+	require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), parentHeight), "receipts have been pruned")
+	require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), parentHeight))
+}
+
+func TestEnsureTraceCallHeightAvailableUsesStateAtHeight(t *testing.T) {
+	t.Parallel()
+
+	stateFloor := int64(150)
+	latestHeight := int64(200)
+	latestCtx := sdk.Context{}.WithBlockHeight(latestHeight)
+	tmClient := newHeightTestClient(stateFloor, 1, latestHeight)
+	rs := &fakeReceiptStore{latest: latestHeight, earliest: 1}
+	stateStore := &fakeStateStore{latest: latestHeight, earliest: stateFloor}
+	wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs)
+
+	require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), stateFloor), "has been pruned")
+	require.NoError(t, wm.EnsureTraceCallHeightAvailable(t.Context(), stateFloor))
+}
+
+func TestEnsureTraceHeightAvailableReceiptsPruned(t *testing.T) {
+	t.Parallel()
+
+	client := newHeightTestClient(100, 1, 200)
+	rs := &fakeReceiptStore{latest: 200, earliest: 150}
+	wm := NewWatermarkManager(client, watermarkTestCtxProvider(200), nil, rs)
+
+	err := wm.EnsureTraceHeightAvailable(t.Context(), 100)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "receipts have been pruned")
+}
+
+func TestEnsureTraceHeightAvailableStatePruned(t *testing.T) {
+	t.Parallel()
+
+	client := newHeightTestClient(100, 1, 200)
+	stateStore := &fakeStateStore{latest: 200, earliest: 150}
+	rs := &fakeReceiptStore{latest: 200, earliest: 1}
+	wm := NewWatermarkManager(client, watermarkTestCtxProvider(200), stateStore, rs)
+
+	err := wm.EnsureTraceHeightAvailable(t.Context(), 100)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "has been pruned")
+}
+
+func TestTraceReceiptFloorBoundary(t *testing.T) {
+	t.Parallel()
+
+	// Receipt retention starts at 150; block 149 still exists in Tendermint but
+	// its receipts are gone. Trace guard applies at the requested height only —
+	// parent fetches at height-1 for state replay must not re-run it.
+	parentHeight := int64(149)
+	receiptFloor := int64(150)
+	latestHeight := int64(200)
+	latestCtx := sdk.Context{}.WithBlockHeight(latestHeight)
+	tmClient := newHeightTestClient(parentHeight, 1, latestHeight)
+	rs := &fakeReceiptStore{latest: latestHeight, earliest: receiptFloor}
+	stateStore := &fakeStateStore{latest: latestHeight, earliest: 1}
+	wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs)
+
+	require.ErrorContains(t, wm.EnsureTraceHeightAvailable(t.Context(), parentHeight), "receipts have been pruned")
+	require.NoError(t, wm.EnsureTraceHeightAvailable(t.Context(), receiptFloor))
+}
+
+func TestTraceBlockByNumberReceiptPrunedBeforeSemaphore(t *testing.T) {
+	t.Parallel()
+
+	prunedHeight := int64(100)
+	latestHeight := int64(200)
+	latestCtx := sdk.Context{}.WithBlockHeight(latestHeight)
+	tmClient := newHeightTestClient(prunedHeight, 1, latestHeight)
+	rs := &fakeReceiptStore{latest: latestHeight, earliest: 150}
+	api := &DebugAPI{
+		tmClient:           tmClient,
+		ctxProvider:        func(int64) sdk.Context { return latestCtx },
+		connectionType:     ConnectionTypeHTTP,
+		maxBlockLookback:   -1,
+		traceCallSemaphore: make(chan struct{}, 1),
+		traceTimeout:       time.Second,
+		backend: &Backend{
+			tmClient:   tmClient,
+			watermarks: NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, nil, rs),
+		},
+	}
+
+	api.traceCallSemaphore <- struct{}{}
+	defer func() { <-api.traceCallSemaphore }()
+
+	_, err := api.TraceBlockByNumber(t.Context(), rpc.BlockNumber(prunedHeight), nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "receipts have been pruned")
+	require.NotErrorIs(t, err, errTraceConcurrencyLimit)
+}
+
+func TestBlockByNumberLatestUsesSafeLatestWatermark(t *testing.T) {
+	t.Parallel()
+
+	const (
+		safeLatest = int64(99)
+		ctxTip     = int64(100)
+	)
+	latestCtx := sdk.Context{}.WithBlockHeight(ctxTip)
+	tmClient := newHeightTestClient(safeLatest, 1, ctxTip)
+	rs := &fakeReceiptStore{latest: safeLatest, earliest: 1}
+	stateStore := &fakeStateStore{latest: safeLatest, earliest: 1}
+	wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs)
+
+	blockNumberPtr, err := getBlockNumber(t.Context(), tmClient, rpc.LatestBlockNumber)
+	require.NoError(t, err)
+	require.Nil(t, blockNumberPtr)
+
+	tmBlock, err := blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, blockNumberPtr, 1)
+	require.NoError(t, err)
+	require.Equal(t, safeLatest, tmBlock.Block.Height)
+
+	rawTip := ctxTip
+	_, err = blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, &rawTip, 1)
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrBlockHeightNotYetAvailable)
+}
+
+func TestTraceLatestTagGuardMatchesBlockResolution(t *testing.T) {
+	t.Parallel()
+
+	const (
+		safeLatest = int64(99)
+		ctxTip     = int64(100)
+	)
+	latestCtx := sdk.Context{}.WithBlockHeight(ctxTip)
+	tmClient := newHeightTestClient(safeLatest, 1, ctxTip)
+	rs := &fakeReceiptStore{latest: safeLatest, earliest: 1}
+	stateStore := &fakeStateStore{latest: safeLatest, earliest: 1}
+	wm := NewWatermarkManager(tmClient, func(int64) sdk.Context { return latestCtx }, stateStore, rs)
+	api := &DebugAPI{
+		tmClient:         tmClient,
+		ctxProvider:      func(int64) sdk.Context { return latestCtx },
+		maxBlockLookback: 0,
+		backend: &Backend{
+			tmClient:   tmClient,
+			watermarks: wm,
+		},
+	}
+
+	guardHeight, err := api.resolveDebugTraceBlockNumber(t.Context(), rpc.LatestBlockNumber)
+	require.NoError(t, err)
+	require.Equal(t, safeLatest, guardHeight)
+
+	blockNumberPtr, err := getBlockNumber(t.Context(), tmClient, rpc.LatestBlockNumber)
+	require.NoError(t, err)
+	tmBlock, err := blockByNumberRespectingWatermarks(t.Context(), tmClient, wm, blockNumberPtr, 1)
+	require.NoError(t, err)
+	require.Equal(t, guardHeight, tmBlock.Block.Height)
+
+	// Lookback must measure against the same safe latest, not the raw app tip.
+	err = api.guardTraceRequestByNumber(t.Context(), "debug_traceBlockByNumber", rpc.LatestBlockNumber)
+	require.NoError(t, err)
+}
+
+type traceGuardReceiptStore struct {
+	fakeReceiptStore
+	getReceiptErr error
+}
+
+func (s *traceGuardReceiptStore) GetReceipt(_ sdk.Context, _ common.Hash) (*evmtypes.Receipt, error) {
+	if s.getReceiptErr != nil {
+		return nil, s.getReceiptErr
+	}
+	return nil, receipt.ErrNotFound
+}
+
+func TestGuardTraceRequestByTxHashReceiptLookupErrors(t *testing.T) {
+	t.Parallel()
+
+	const latestHeight = int64(10)
+	latestCtx := sdk.Context{}.WithBlockHeight(latestHeight)
+	txHash := common.HexToHash("0xabc")
+	storeErr := errors.New("receipt store unavailable")
+
+	newAPI := func(store receipt.ReceiptStore) *DebugAPI {
+		k := &keeper.Keeper{}
+		k.SetReceiptStoreForTesting(store)
+		return &DebugAPI{
+			keeper:           k,
+			ctxProvider:      func(int64) sdk.Context { return latestCtx },
+			maxBlockLookback: -1,
+		}
+	}
+
+	t.Run("ErrReceiptPruned", func(t *testing.T) {
+		t.Parallel()
+		prunedErr := fmt.Errorf("requested height 100 receipts have been pruned; earliest available is 150: %w", receipt.ErrReceiptPruned)
+		api := newAPI(&traceGuardReceiptStore{getReceiptErr: prunedErr})
+		err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash)
+		require.ErrorIs(t, err, receipt.ErrReceiptPruned)
+	})
+
+	t.Run("store error", func(t *testing.T) {
+		t.Parallel()
+		api := newAPI(&traceGuardReceiptStore{getReceiptErr: storeErr})
+		err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash)
+		require.ErrorIs(t, err, storeErr)
+	})
+
+	t.Run("ErrNotFound falls through to lookback", func(t *testing.T) {
+		t.Parallel()
+		api := newAPI(&traceGuardReceiptStore{getReceiptErr: receipt.ErrNotFound})
+		err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash)
+		require.NoError(t, err)
+	})
+
+	t.Run("ErrNotConfigured", func(t *testing.T) {
+		t.Parallel()
+		api := newAPI(&traceGuardReceiptStore{getReceiptErr: receipt.ErrNotConfigured})
+		err := api.guardTraceRequestByTxHash(t.Context(), "debug_traceTransaction", txHash)
+		require.ErrorIs(t, err, receipt.ErrNotConfigured)
+	})
+}

evmrpc/simulate.go

@@ -410,26 +410,12 @@ func (b *Backend) ChainDb() ethdb.Database {
 	panic("implement me")
 }
 
-func (b Backend) ConvertBlockNumber(bn rpc.BlockNumber) int64 {
-	blockNum := bn.Int64()
-	switch blockNum {
-	case rpc.SafeBlockNumber.Int64(), rpc.FinalizedBlockNumber.Int64(), rpc.LatestBlockNumber.Int64():
-		blockNum = b.ctxProvider(LatestCtxHeight).BlockHeight()
-	case rpc.EarliestBlockNumber.Int64():
-		genesisRes, err := b.tmClient.Genesis(context.Background())
-		if err != nil {
-			panic("could not get genesis info from tendermint")
-		}
-		blockNum = genesisRes.Genesis.InitialHeight
-	case rpc.PendingBlockNumber.Int64():
-		panic("tracing on pending block is not supported")
-	}
-	return blockNum
-}
-
 func (b Backend) BlockByNumber(ctx context.Context, bn rpc.BlockNumber) (*ethtypes.Block, []tracersutils.TraceBlockMetadata, error) {
-	blockNum := b.ConvertBlockNumber(bn)
-	tmBlock, err := blockByNumberRespectingWatermarks(ctx, b.tmClient, b.watermarks, &blockNum, 1)
+	blockNumberPtr, err := getBlockNumber(ctx, b.tmClient, bn)
+	if err != nil {
+		return nil, nil, err
+	}
+	tmBlock, err := blockByNumberRespectingWatermarks(ctx, b.tmClient, b.watermarks, blockNumberPtr, 1)
 	if err != nil {
 		return nil, nil, err
 	}

evmrpc/simulate_test.go

@@ -378,27 +378,6 @@ func TestNewRevertError(t *testing.T) {
 	require.Equal(t, "0x", err.ErrorData())
 }
 
-func TestConvertBlockNumber(t *testing.T) {
-	tmClient := &MockClient{}
-	watermarks := evmrpc.NewWatermarkManager(tmClient, func(i int64) sdk.Context {
-		if i == evmrpc.LatestCtxHeight {
-			return sdk.Context{}.WithBlockHeight(1000)
-		}
-		return sdk.Context{}
-	}, nil, nil)
-	backend := evmrpc.NewBackend(func(i int64) sdk.Context {
-		if i == evmrpc.LatestCtxHeight {
-			return sdk.Context{}.WithBlockHeight(1000)
-		}
-		return sdk.Context{}
-	}, nil, legacyabci.BeginBlockKeepers{}, nil, &MockClient{}, nil, nil, nil, evmrpc.NewBlockCache(3000), &sync.Mutex{}, watermarks)
-	require.Equal(t, int64(10), backend.ConvertBlockNumber(10))
-	require.Equal(t, int64(1), backend.ConvertBlockNumber(0))
-	require.Equal(t, int64(1000), backend.ConvertBlockNumber(-2))
-	require.Equal(t, int64(1000), backend.ConvertBlockNumber(-3))
-	require.Equal(t, int64(1000), backend.ConvertBlockNumber(-4))
-}
-
 func TestPreV620UpgradeUsesBaseFeeNil(t *testing.T) {
 	// Set up a test context with a height before v6.2.0 upgrade
 	// For pacific-1 chain, we need to set a height that's before the v6.2.0 upgrade

evmrpc/trace_db_reader_test.go

@@ -82,10 +82,10 @@ func TestBlockTraceCacheGet(t *testing.T) {
 		require.Nil(t, got)
 	})
 
-	t.Run("empty block -> empty hit", func(t *testing.T) {
+	t.Run("empty tx list is not a cache hit", func(t *testing.T) {
 		got, ok := blockTraceCacheGet(c, 5, []common.Hash{}, cfg)
-		require.True(t, ok)
-		require.Empty(t, got)
+		require.False(t, ok, "empty tx list must not report cache hit")
+		require.Nil(t, got)
 	})
 }
 

evmrpc/trace_profile.go

@@ -51,7 +51,7 @@ func (api *DebugAPI) TraceTransactionProfile(ctx context.Context, hash common.Ha
 	if returnErr = api.validateTraceTracer(config); returnErr != nil {
 		return nil, returnErr
 	}
-	if returnErr = api.guardHistoricalDebugTraceByTxHash(ctx, "debug_traceTransactionProfile", hash); returnErr != nil {
+	if returnErr = api.guardTraceRequestByTxHash(ctx, "debug_traceTransactionProfile", hash); returnErr != nil {
 		return nil, returnErr
 	}