Skip to content

Commit 714ecac

Browse files
fix(evmrpc): release eth_getLogs DB-read slots when a block read panics (#4388)
After the atlantic-2 v6.7 upgrade, RPC nodes began rejecting every `eth_getLogs` call with `server I/O saturated, rejecting new request (semaphore: 26/32 in use)` until they were restarted (PLT-1332). `LogFetcher.processBatch` took a `dbReadSemaphore` slot for each uncached block and released it by hand on each exit path, so a panic in `BlockStore.LoadBlock` kept both the slot and the `DBSemaphoreAcquired` gauge. The worker pool recovered the panic, and `GetLogs` refuses new requests once 80% of slots are held, so the leaks added up to an outage. The panicking batch also sent nothing to `errChan`, so the request that hit it could succeed with that batch's logs missing. The part of the read that holds a slot moves into `readUncachedBlock`, which takes the slot through `acquireDBReadSlot` and defers the release it returns, so the slot and the gauge come back on every exit, panics included. `processBatch` now recovers a panic and reports it on `errChan` like any other failed read, so the request fails instead of returning partial logs. Because the worker pool no longer sees that panic, `reportBatchPanic` logs it with its stack and counts it in `TasksPanicked`. Freeing the slot alone would have ended the outage but kept the partial results. The order of the cache check, bloom check and block fetch is unchanged, and blocks still go to `res` after the slot is freed. No consensus or state impact. This is the block-by-block path of `GetLogsByFilters`, which serves `eth_getLogs`, `eth_getFilterLogs`, `eth_getFilterChanges` and log subscriptions on nodes whose receipt store can't answer range queries, such as the pebble backend. The panic itself will keep happening, since v6.7 can't load blocks stored before the v6.4 commit-hash change: ranges over those heights now fail with an error, and each panicking batch logs one error with a stack trace, which may be noisy on archive nodes. The hand-release pattern dates back to #2371, so `release/v6.7` and `release/v6.6` have the same leak, and this likely wants a backport to `release/v6.7` before mainnet upgrades. Both tests in `filter_db_read_slot_test.go` fail on `main` (the panic escapes `processBatch`, and a three-batch request hangs on the leaked slot) and pass with this change, and `go test -race ./evmrpc/...` passes. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent e822b8f commit 714ecac

2 files changed

Lines changed: 194 additions & 51 deletions

File tree

‎evmrpc/filter.go‎

Lines changed: 89 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"log/slog"
1010
"math/big"
11+
"runtime/debug"
1112
"sort"
1213
"sync"
1314
"time"
@@ -1500,11 +1501,17 @@ func (f *LogFetcher) fetchBlocksByCrit(ctx context.Context, crit filters.FilterC
15001501
return res, end, nil
15011502
}
15021503

1503-
// Batch processing function for blocks
1504+
// processBatch sends to res each block in [start, end] that crit can match, and reports on errChan
1505+
// each block it cannot read. A panic is reported the same way and ends the batch.
15041506
func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit filters.FilterCriteria, bloomIndexes [][]BloomIndexes, res chan *coretypes.ResultBlock, errChan chan error) {
1505-
wpMetrics := GetGlobalMetrics()
1507+
var height int64
1508+
defer func() {
1509+
if r := recover(); r != nil {
1510+
reportBatchPanic(errChan, height, r)
1511+
}
1512+
}()
15061513

1507-
for height := start; height <= end; height++ {
1514+
for height = start; height <= end; height++ {
15081515
if height == 0 {
15091516
continue
15101517
}
@@ -1520,65 +1527,96 @@ func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit fi
15201527
continue
15211528
}
15221529

1523-
// Block cache miss, acquire semaphore for I/O operations
1524-
semWaitStart := time.Now()
1525-
f.dbReadSemaphore <- struct{}{}
1526-
wpMetrics.RecordDBSemaphoreWait(time.Since(semWaitStart))
1527-
wpMetrics.RecordDBSemaphoreAcquire()
1528-
1529-
// Re-check cache after acquiring semaphore, in case another worker cached it.
1530-
if cachedEntry, found := f.globalBlockCache.Get(height); found {
1531-
<-f.dbReadSemaphore
1532-
wpMetrics.RecordDBSemaphoreRelease()
1533-
if cachedEntry.Block != nil {
1534-
if err := f.watermarks.EnsureBlockHeightAvailable(ctx, cachedEntry.Block.Block.Height); err != nil {
1535-
continue
1536-
}
1537-
}
1538-
res <- cachedEntry.Block
1530+
// Block cache miss, read the block from the store
1531+
block, ok, err := f.readUncachedBlock(ctx, height, crit, bloomIndexes)
1532+
if err != nil {
1533+
reportBatchError(errChan, fmt.Errorf("failed to fetch block at height %d: %w", height, err))
15391534
continue
15401535
}
1536+
if ok {
1537+
res <- block
1538+
}
1539+
}
1540+
}
15411541

1542-
// check bloom filter if cache miss AND we have filters
1543-
var blockBloom ethtypes.Bloom
1544-
if len(crit.Addresses) != 0 || len(crit.Topics) != 0 {
1545-
// Bloom cache miss - read from database
1546-
providerCtx := f.ctxProvider(height)
1547-
if f.includeSyntheticReceipts {
1548-
blockBloom = f.k.GetBlockBloom(providerCtx)
1549-
} else {
1550-
blockBloom = f.k.GetEvmOnlyBlockBloom(providerCtx)
1551-
}
1542+
// reportBatchPanic logs and counts a panic raised while processBatch handled height, and reports it
1543+
// on errChan.
1544+
func reportBatchPanic(errChan chan<- error, height int64, panicValue any) {
1545+
logger.Error("panic while fetching block for logs", "height", height, "panic", panicValue, "stack", string(debug.Stack()))
1546+
// Recovered in processBatch, the panic never reaches the worker pool, which counts task panics.
1547+
GetGlobalMetrics().RecordTaskPanicked()
1548+
reportBatchError(errChan, fmt.Errorf("failed to fetch block at height %d: panic: %v", height, panicValue))
1549+
}
15521550

1553-
// When we cannot retrieve a bloom for the EVM-only view (all zeroes),
1554-
// skip the bloom pre-filter instead of short-circuiting the block.
1555-
if blockBloom != (ethtypes.Bloom{}) && !MatchFilters(blockBloom, bloomIndexes) {
1556-
<-f.dbReadSemaphore
1557-
wpMetrics.RecordDBSemaphoreRelease()
1558-
continue // skip the block if bloom filter does not match
1551+
// reportBatchError sends err on errChan without blocking, dropping it when errChan is full.
1552+
func reportBatchError(errChan chan<- error, err error) {
1553+
select {
1554+
case errChan <- err:
1555+
default: // fetchBlocksByCrit reports only the first failure
1556+
}
1557+
}
1558+
1559+
// readUncachedBlock reads the block at height under a DB-read slot and adds it to the block cache.
1560+
// It returns ok=false, with no error, when the block's bloom rules out crit or when a copy another
1561+
// worker cached meanwhile is outside the available range.
1562+
func (f *LogFetcher) readUncachedBlock(ctx context.Context, height int64, crit filters.FilterCriteria, bloomIndexes [][]BloomIndexes) (block *coretypes.ResultBlock, ok bool, err error) {
1563+
release := f.acquireDBReadSlot()
1564+
defer release()
1565+
1566+
// Re-check cache after acquiring semaphore, in case another worker cached it.
1567+
if cachedEntry, found := f.globalBlockCache.Get(height); found {
1568+
if cachedEntry.Block != nil {
1569+
if err := f.watermarks.EnsureBlockHeightAvailable(ctx, cachedEntry.Block.Block.Height); err != nil {
1570+
return nil, false, nil
15591571
}
15601572
}
1573+
return cachedEntry.Block, true, nil
1574+
}
15611575

1562-
// fetch block from network
1563-
block, err := blockByNumberRespectingWatermarks(ctx, f.tmClient, f.watermarks, &height, 1)
1564-
if err != nil {
1565-
select {
1566-
case errChan <- fmt.Errorf("failed to fetch block at height %d: %w", height, err):
1567-
default:
1568-
}
1569-
<-f.dbReadSemaphore
1570-
wpMetrics.RecordDBSemaphoreRelease()
1571-
continue
1576+
// check bloom filter if cache miss AND we have filters
1577+
var blockBloom ethtypes.Bloom
1578+
if len(crit.Addresses) != 0 || len(crit.Topics) != 0 {
1579+
// Bloom cache miss - read from database
1580+
providerCtx := f.ctxProvider(height)
1581+
if f.includeSyntheticReceipts {
1582+
blockBloom = f.k.GetBlockBloom(providerCtx)
1583+
} else {
1584+
blockBloom = f.k.GetEvmOnlyBlockBloom(providerCtx)
15721585
}
15731586

1574-
// Use LoadOrStore to create/get cache entry atomically
1575-
entry := loadOrStoreCacheEntry(f.cacheCreationMutex, f.globalBlockCache, height, block)
1576-
// Fill bloom if we have it and it's missing
1577-
if blockBloom != (ethtypes.Bloom{}) {
1578-
fillMissingFields(entry, block, blockBloom)
1587+
// When we cannot retrieve a bloom for the EVM-only view (all zeroes),
1588+
// skip the bloom pre-filter instead of short-circuiting the block.
1589+
if blockBloom != (ethtypes.Bloom{}) && !MatchFilters(blockBloom, bloomIndexes) {
1590+
return nil, false, nil // skip the block if bloom filter does not match
15791591
}
1592+
}
1593+
1594+
// fetch block from network
1595+
block, err = blockByNumberRespectingWatermarks(ctx, f.tmClient, f.watermarks, &height, 1)
1596+
if err != nil {
1597+
return nil, false, err
1598+
}
1599+
1600+
// Use LoadOrStore to create/get cache entry atomically
1601+
entry := loadOrStoreCacheEntry(f.cacheCreationMutex, f.globalBlockCache, height, block)
1602+
// Fill bloom if we have it and it's missing
1603+
if blockBloom != (ethtypes.Bloom{}) {
1604+
fillMissingFields(entry, block, blockBloom)
1605+
}
1606+
return block, true, nil
1607+
}
1608+
1609+
// acquireDBReadSlot blocks until a DB-read slot is free and returns the function that frees it.
1610+
// Callers must defer the returned function so the slot is freed on every exit, panics included:
1611+
// GetLogs turns requests away while too many slots are held.
1612+
func (f *LogFetcher) acquireDBReadSlot() (release func()) {
1613+
wpMetrics := GetGlobalMetrics()
1614+
semWaitStart := time.Now()
1615+
f.dbReadSemaphore <- struct{}{}
1616+
wpMetrics.RecordDBSemaphoreWait(time.Since(semWaitStart))
1617+
wpMetrics.RecordDBSemaphoreAcquire()
1618+
return func() {
15801619
<-f.dbReadSemaphore
15811620
wpMetrics.RecordDBSemaphoreRelease()
1582-
res <- block
15831621
}
15841622
}

‎evmrpc/filter_db_read_slot_test.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package evmrpc
2+
3+
import (
4+
"context"
5+
"math/big"
6+
"sync"
7+
"testing"
8+
"time"
9+
10+
"github.com/ethereum/go-ethereum/eth/filters"
11+
evmrpcconfig "github.com/sei-protocol/sei-chain/evmrpc/config"
12+
"github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
const blockStorePanic = "error from proto block: wrong Header.LastCommitHash"
17+
18+
// panickingBlockTestClient panics when asked for panicHeight, the way the block store does when it
19+
// cannot decode a stored block.
20+
type panickingBlockTestClient struct {
21+
*heightTestClient
22+
panicHeight int64
23+
}
24+
25+
func (c *panickingBlockTestClient) Block(ctx context.Context, height *int64) (*coretypes.ResultBlock, error) {
26+
if height != nil && *height == c.panicHeight {
27+
panic(blockStorePanic)
28+
}
29+
return c.heightTestClient.Block(ctx, height)
30+
}
31+
32+
// newPanickingBlockFetcher returns a LogFetcher over blocks 1 to latest whose block store panics at
33+
// panicHeight. It has a single DB-read slot, so a leaked slot stalls every later read.
34+
func newPanickingBlockFetcher(latest, panicHeight int64) *LogFetcher {
35+
client := &panickingBlockTestClient{heightTestClient: newHeightTestClient(latest+10, 1, latest), panicHeight: panicHeight}
36+
return &LogFetcher{
37+
tmClient: client,
38+
dbReadSemaphore: make(chan struct{}, 1),
39+
globalBlockCache: NewBlockCache(int(latest)),
40+
cacheCreationMutex: &sync.Mutex{},
41+
watermarks: newHeightTestWatermarks(client, latest),
42+
}
43+
}
44+
45+
// A block read that panics must free its DB-read slot, or GetLogs ends up rejecting every request as
46+
// I/O saturated, and must come back as an error rather than escape the batch.
47+
func TestProcessBatchReportsPanickingBlockReadAndFreesDBReadSlot(t *testing.T) {
48+
t.Parallel()
49+
50+
const panicHeight = int64(10)
51+
fetcher := newPanickingBlockFetcher(90, panicHeight)
52+
res := make(chan *coretypes.ResultBlock, 1)
53+
errChan := make(chan error, 1)
54+
55+
require.NotPanics(t, func() {
56+
fetcher.processBatch(context.Background(), panicHeight, panicHeight, filters.FilterCriteria{}, nil, res, errChan)
57+
})
58+
59+
require.Empty(t, fetcher.dbReadSemaphore)
60+
require.Empty(t, res)
61+
require.Len(t, errChan, 1)
62+
err := <-errChan
63+
require.ErrorContains(t, err, "height 10")
64+
require.ErrorContains(t, err, blockStorePanic)
65+
}
66+
67+
// A panic in one batch of a multi-batch request must fail the request. Left to the worker pool, which
68+
// recovers it silently, the request would return the other batches' blocks as if they were the answer.
69+
//
70+
// Not parallel: it reads the global worker pool metrics, which parallel tests also move.
71+
func TestFetchBlocksByCritFailsWhenABlockReadPanics(t *testing.T) {
72+
const (
73+
batchSize = int64(evmrpcconfig.WorkerBatchSize)
74+
latest = 3 * batchSize
75+
panicHeight = batchSize + batchSize/2 // inside the second of three batches
76+
)
77+
metrics := GetGlobalMetrics()
78+
slotsHeldBefore := metrics.DBSemaphoreAcquired.Load()
79+
panicsBefore := metrics.TasksPanicked.Load()
80+
fetcher := newPanickingBlockFetcher(latest, panicHeight)
81+
crit := filters.FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(latest)}
82+
83+
type fetchResult struct {
84+
blocks chan *coretypes.ResultBlock
85+
err error
86+
}
87+
done := make(chan fetchResult, 1)
88+
go func() {
89+
blocks, _, err := fetcher.fetchBlocksByCrit(context.Background(), crit, 0, nil)
90+
done <- fetchResult{blocks: blocks, err: err}
91+
}()
92+
93+
var got fetchResult
94+
select {
95+
case got = <-done:
96+
case <-time.After(10 * time.Second):
97+
t.Fatal("fetchBlocksByCrit did not return, most likely because a DB-read slot leaked")
98+
}
99+
100+
require.ErrorContains(t, got.err, blockStorePanic)
101+
require.Nil(t, got.blocks)
102+
require.Empty(t, fetcher.dbReadSemaphore)
103+
require.Equal(t, slotsHeldBefore, metrics.DBSemaphoreAcquired.Load())
104+
require.Equal(t, panicsBefore+1, metrics.TasksPanicked.Load())
105+
}

0 commit comments

Comments
 (0)