Skip to content

Commit ffa72b4

Browse files
authored
Return an error instead of panicking when RPC state is unavailable (#4430)
On a node with the state store disabled, `RPCContextProvider` can only build a context for the latest committed height and panics for any other height. `EncodeTmBlock` and `filterTransactions` always ask for the parent height to compute the base fee and nonces, and the log fetcher asks for the queried heights, so on such a node `eth_getBlockByNumber` (even for `latest`) and `eth_getLogs` hit that panic on every call. go-ethereum's RPC server recovers it and the client gets a bare `-32603 method handler crashed`, while seid logs a stack trace for each request. atlantic-2's sei-node-1 currently logs roughly 130 of these an hour from a single external client. This adds `ctxAtHeight`, which calls the context provider and turns an error it panics with into a returned error naming the height. Non-error panics and runtime errors such as nil dereferences are re-raised, so real bugs keep their stack. `EncodeTmBlock`, `filterTransactions`, `tryFilterLogsRange`, `collectLogs` and the bloom read in `readUncachedBlock` use it for their non-latest contexts, so these paths now return an error such as `state at height N is unavailable: failed to load state at height N; unable to load historical state with SS disabled ...` instead of panicking. The same applies to heights pruned from the state store on SS-enabled nodes. Nothing changes for heights the node can serve. One caveat: the block-by-block fallback in `GetLogsByFilters` already drops per-block errors from `GetLogsForBlockPooled` on main, so an unfiltered `eth_getLogs` that takes that path still answers with the affected blocks' logs missing rather than an error. That is pre-existing and left for a follow-up. This only touches the EVM RPC read path, with no consensus or state impact. New unit tests cover `ctxAtHeight`, `EncodeTmBlock` with an unavailable parent context, and the log fetcher's `tryFilterLogsRange`, `collectLogs` and `readUncachedBlock` with unavailable state.
1 parent 7a2ec6e commit ffa72b4

5 files changed

Lines changed: 145 additions & 7 deletions

File tree

‎evmrpc/block.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -345,10 +345,17 @@ func EncodeTmBlock(
345345
txHash := common.HexToHash(block.Block.DataHash.String())
346346
resultHash := common.HexToHash(block.Block.LastResultsHash.String())
347347
miner := common.HexToAddress(block.Block.ProposerAddress.String())
348-
ctx := ctxProvider(block.Block.Height)
348+
ctx, err := ctxAtHeight(ctxProvider, block.Block.Height)
349+
if err != nil {
350+
return nil, err
351+
}
349352
var baseFeePerGas *big.Int
350353
if block.Block.Height > 1 {
351-
baseFeePerGas = k.GetNextBaseFeePerGas(ctxProvider(block.Block.Height - 1)).TruncateInt().BigInt()
354+
prevCtx, err := ctxAtHeight(ctxProvider, block.Block.Height-1)
355+
if err != nil {
356+
return nil, err
357+
}
358+
baseFeePerGas = k.GetNextBaseFeePerGas(prevCtx).TruncateInt().BigInt()
352359
} else {
353360
baseFeePerGas = types.DefaultMinFeePerGas.TruncateInt().BigInt()
354361
}

‎evmrpc/block_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package evmrpc_test
22

33
import (
44
"crypto/sha256"
5+
"errors"
56
"math/big"
67
"sync"
78
"testing"
@@ -46,6 +47,30 @@ func TestEncodeTmBlock_EmptyTransactions(t *testing.T) {
4647
require.Equal(t, ethtypes.EmptyTxsHash, result["transactionsRoot"])
4748
}
4849

50+
func TestEncodeTmBlockReturnsErrorWhenParentStateUnavailable(t *testing.T) {
51+
k := &testkeeper.EVMTestApp.EvmKeeper
52+
ctx := testkeeper.EVMTestApp.GetContextForDeliverTx([]byte{}).WithBlockTime(time.Now())
53+
block := &coretypes.ResultBlock{
54+
BlockID: MockBlockID,
55+
Block: &tmtypes.Block{
56+
Header: mockBlockHeader(MockHeight8),
57+
Data: tmtypes.Data{},
58+
LastCommit: &tmtypes.Commit{Height: MockHeight8 - 1},
59+
},
60+
}
61+
cause := errors.New("unable to load historical state with SS disabled")
62+
ctxProvider := func(h int64) sdk.Context {
63+
if h == MockHeight8-1 {
64+
panic(cause)
65+
}
66+
return ctx
67+
}
68+
69+
result, err := evmrpc.EncodeTmBlock(ctxProvider, func(int64) client.TxConfig { return TxConfig }, block, k, true, false, evmrpc.NewBlockCache(3000), &sync.Mutex{})
70+
require.ErrorIs(t, err, cause)
71+
require.Nil(t, result)
72+
}
73+
4974
// Sei commits blocks more often than once a second, so the seconds-only
5075
// Ethereum "timestamp" repeats across consecutive blocks. milliTimestamp carries
5176
// the sub-second part the Tendermint header already holds.

‎evmrpc/filter.go‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1159,7 +1159,11 @@ func (f *LogFetcher) tryFilterLogsRange(ctx context.Context, fromBlock, toBlock
11591159

11601160
// Use a context at the window's toBlock height for the query
11611161
// #nosec G115 -- windowTo is a block height which fits in int64
1162-
sdkCtx := f.ctxProvider(int64(windowTo)).WithContext(ctx)
1162+
sdkCtx, err := ctxAtHeight(f.ctxProvider, int64(windowTo))
1163+
if err != nil {
1164+
return nil, err
1165+
}
1166+
sdkCtx = sdkCtx.WithContext(ctx)
11631167

11641168
candidates, err := store.FilterLogs(sdkCtx, windowFrom, windowTo, crit, f.storeCandidateBudget())
11651169
if err != nil {
@@ -1316,7 +1320,10 @@ func (f *LogFetcher) GetLogsForBlockPooled(block *coretypes.ResultBlock, crit fi
13161320

13171321
// Unified log collection logic - fallback path that fetches receipts individually
13181322
func (f *LogFetcher) collectLogs(block *coretypes.ResultBlock, crit filters.FilterCriteria, collector logCollector) error {
1319-
ctx := f.ctxProvider(block.Block.Height)
1323+
ctx, err := ctxAtHeight(f.ctxProvider, block.Block.Height)
1324+
if err != nil {
1325+
return err
1326+
}
13201327

13211328
txHashes, err := getTxHashesFromBlock(f.ctxProvider, f.txConfigProvider, f.k, block, f.includeSyntheticReceipts, f.cacheCreationMutex, f.globalBlockCache)
13221329
if err != nil {
@@ -1554,7 +1561,10 @@ func (f *LogFetcher) readUncachedBlock(ctx context.Context, height int64, crit f
15541561
var blockBloom ethtypes.Bloom
15551562
if len(crit.Addresses) != 0 || len(crit.Topics) != 0 {
15561563
// Bloom cache miss - read from database
1557-
providerCtx := f.ctxProvider(height)
1564+
providerCtx, err := ctxAtHeight(f.ctxProvider, height)
1565+
if err != nil {
1566+
return nil, false, err
1567+
}
15581568
if f.includeSyntheticReceipts {
15591569
blockBloom = f.k.GetBlockBloom(providerCtx)
15601570
} else {

‎evmrpc/utils.go‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"errors"
99
"fmt"
1010
"math/big"
11+
"runtime"
1112
"runtime/debug"
1213
"strings"
1314
"sync"
@@ -227,8 +228,14 @@ func filterTransactions(
227228
startOfBlockNonce := make(map[string]uint64)
228229
txConfig := txConfigProvider(block.Block.Height)
229230
latestCtx := ctxProvider(LatestCtxHeight)
230-
ctx := ctxProvider(block.Block.Height)
231-
prevCtx := ctxProvider(block.Block.Height - 1)
231+
ctx, err := ctxAtHeight(ctxProvider, block.Block.Height)
232+
if err != nil {
233+
return nil, err
234+
}
235+
prevCtx, err := ctxAtHeight(ctxProvider, block.Block.Height-1)
236+
if err != nil {
237+
return nil, err
238+
}
232239
for i, tx := range block.Block.Txs {
233240
sdkTx, err := txConfig.TxDecoder()(tx)
234241
if err != nil {
@@ -411,6 +418,23 @@ func recoverAndLog() {
411418
}
412419
}
413420

421+
// ctxAtHeight returns the context ctxProvider builds for height, or the error it
422+
// panics with when it cannot, e.g. historical state the node does not retain.
423+
// Non-error panics and runtime errors are re-raised.
424+
func ctxAtHeight(ctxProvider func(int64) sdk.Context, height int64) (ctx sdk.Context, err error) {
425+
defer func() {
426+
if r := recover(); r != nil {
427+
e, ok := r.(error)
428+
var runtimeErr runtime.Error
429+
if !ok || errors.As(e, &runtimeErr) {
430+
panic(r)
431+
}
432+
err = fmt.Errorf("state at height %d is unavailable: %w", height, e)
433+
}
434+
}()
435+
return ctxProvider(height), nil
436+
}
437+
414438
func must[V any](v V, err error) V {
415439
if err != nil {
416440
panic(err)

‎evmrpc/utils_internal_test.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
package evmrpc
22

33
import (
4+
"context"
5+
"errors"
46
"sync"
57
"testing"
68
"time"
9+
10+
"github.com/ethereum/go-ethereum/common"
11+
"github.com/ethereum/go-ethereum/eth/filters"
12+
sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
13+
"github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes"
14+
tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types"
15+
"github.com/sei-protocol/sei-chain/x/evm/keeper"
16+
"github.com/stretchr/testify/require"
717
)
818

919
func TestRunWithRecoveryHandlesPanic(t *testing.T) {
@@ -24,3 +34,65 @@ func TestRunWithRecoveryHandlesPanic(t *testing.T) {
2434
t.Fatal("expected panic to be recovered")
2535
}
2636
}
37+
38+
func TestCtxAtHeightReturnsProviderPanicAsError(t *testing.T) {
39+
cause := errors.New("unable to load historical state with SS disabled for version: 7")
40+
_, err := ctxAtHeight(func(int64) sdk.Context { panic(cause) }, 7)
41+
require.ErrorIs(t, err, cause)
42+
require.Contains(t, err.Error(), "state at height 7 is unavailable")
43+
}
44+
45+
func TestCtxAtHeightRepanicsNonErrorValues(t *testing.T) {
46+
require.PanicsWithValue(t, "boom", func() {
47+
_, _ = ctxAtHeight(func(int64) sdk.Context { panic("boom") }, 7)
48+
})
49+
}
50+
51+
func TestCtxAtHeightRepanicsRuntimeErrors(t *testing.T) {
52+
require.Panics(t, func() {
53+
_, _ = ctxAtHeight(func(int64) sdk.Context {
54+
var heights []int64
55+
_ = heights[1]
56+
return sdk.Context{}
57+
}, 7)
58+
})
59+
}
60+
61+
func TestCtxAtHeightReturnsProviderContext(t *testing.T) {
62+
ctx, err := ctxAtHeight(func(h int64) sdk.Context { return sdk.Context{}.WithBlockHeight(h) }, 7)
63+
require.NoError(t, err)
64+
require.Equal(t, int64(7), ctx.BlockHeight())
65+
}
66+
67+
func unavailableStateProvider(cause error) func(int64) sdk.Context {
68+
return func(h int64) sdk.Context {
69+
if h == LatestCtxHeight {
70+
return sdk.Context{}
71+
}
72+
panic(cause)
73+
}
74+
}
75+
76+
func TestLogFetcherReturnsErrorWhenStateUnavailable(t *testing.T) {
77+
cause := errors.New("unable to load historical state with SS disabled")
78+
k := &keeper.Keeper{}
79+
k.SetReceiptStoreForTesting(&fakeReceiptStore{})
80+
f := &LogFetcher{
81+
k: k,
82+
ctxProvider: unavailableStateProvider(cause),
83+
filterConfig: &FilterConfig{},
84+
dbReadSemaphore: make(chan struct{}, 1),
85+
globalBlockCache: NewBlockCache(10),
86+
}
87+
88+
_, err := f.tryFilterLogsRange(context.Background(), 5, 5, filters.FilterCriteria{}, 10)
89+
require.ErrorIs(t, err, cause)
90+
91+
block := &coretypes.ResultBlock{Block: &tmtypes.Block{Header: tmtypes.Header{Height: 5}}}
92+
require.ErrorIs(t, f.collectLogs(block, filters.FilterCriteria{}, nil), cause)
93+
94+
crit := filters.FilterCriteria{Addresses: []common.Address{{0x1}}}
95+
_, ok, err := f.readUncachedBlock(context.Background(), 5, crit, EncodeFilters(crit.Addresses, crit.Topics))
96+
require.ErrorIs(t, err, cause)
97+
require.False(t, ok)
98+
}

0 commit comments

Comments
 (0)