Skip to content

Commit 0adcd5b

Browse files
seidroid[bot]alexander-seicursoragent
authored
Backport release/v6.7: fix(evmrpc): release eth_getLogs DB-read slots when a block read panics (#4397)
Backport of #4388 to `release/v6.7`. Co-authored-by: alexander-sei <alexanderh@seinetwork.io> Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 8a4dff3 commit 0adcd5b

2 files changed

Lines changed: 195 additions & 52 deletions

File tree

‎evmrpc/filter.go‎

Lines changed: 90 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"fmt"
99
"log/slog"
1010
"math/big"
11+
"runtime/debug"
1112
"sort"
1213
"sync"
1314
"time"
@@ -1415,11 +1416,17 @@ func (f *LogFetcher) fetchBlocksByCrit(ctx context.Context, crit filters.FilterC
14151416
return res, end, nil
14161417
}
14171418

1418-
// Batch processing function for blocks
1419+
// processBatch sends to res each block in [start, end] that crit can match, and reports on errChan
1420+
// each block it cannot read. A panic is reported the same way and ends the batch.
14191421
func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit filters.FilterCriteria, bloomIndexes [][]BloomIndexes, res chan *coretypes.ResultBlock, errChan chan error) {
1420-
wpMetrics := GetGlobalMetrics()
1422+
var height int64
1423+
defer func() {
1424+
if r := recover(); r != nil {
1425+
reportBatchPanic(errChan, height, r)
1426+
}
1427+
}()
14211428

1422-
for height := start; height <= end; height++ {
1429+
for height = start; height <= end; height++ {
14231430
if height == 0 {
14241431
continue
14251432
}
@@ -1435,65 +1442,96 @@ func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit fi
14351442
continue
14361443
}
14371444

1438-
// Block cache miss, acquire semaphore for I/O operations
1439-
semWaitStart := time.Now()
1440-
f.dbReadSemaphore <- struct{}{}
1441-
wpMetrics.RecordDBSemaphoreWait(time.Since(semWaitStart))
1442-
wpMetrics.RecordDBSemaphoreAcquire()
1443-
1444-
// Re-check cache after acquiring semaphore, in case another worker cached it.
1445-
if cachedEntry, found := f.globalBlockCache.Get(height); found {
1446-
<-f.dbReadSemaphore
1447-
wpMetrics.RecordDBSemaphoreRelease()
1448-
if cachedEntry.Block != nil {
1449-
if err := f.watermarks.EnsureBlockHeightAvailable(ctx, cachedEntry.Block.Block.Height); err != nil {
1450-
continue
1451-
}
1452-
}
1453-
res <- cachedEntry.Block
1445+
// Block cache miss, read the block from the store
1446+
block, ok, err := f.readUncachedBlock(ctx, height, crit, bloomIndexes)
1447+
if err != nil {
1448+
reportBatchError(errChan, fmt.Errorf("failed to fetch block at height %d: %w", height, err))
14541449
continue
14551450
}
1451+
if ok {
1452+
res <- block
1453+
}
1454+
}
1455+
}
14561456

1457-
// check bloom filter if cache miss AND we have filters
1458-
var blockBloom ethtypes.Bloom
1459-
if len(crit.Addresses) != 0 || len(crit.Topics) != 0 {
1460-
// Bloom cache miss - read from database
1461-
providerCtx := f.ctxProvider(height)
1462-
if f.includeSyntheticReceipts {
1463-
blockBloom = f.k.GetBlockBloom(providerCtx)
1464-
} else {
1465-
blockBloom = f.k.GetEvmOnlyBlockBloom(providerCtx)
1466-
}
1457+
// reportBatchPanic logs and counts a panic raised while processBatch handled height, and reports it
1458+
// on errChan.
1459+
func reportBatchPanic(errChan chan<- error, height int64, panicValue any) {
1460+
logger.Error("panic while fetching block for logs", "height", height, "panic", panicValue, "stack", string(debug.Stack()))
1461+
// Recovered in processBatch, the panic never reaches the worker pool, which counts task panics.
1462+
GetGlobalMetrics().RecordTaskPanicked()
1463+
reportBatchError(errChan, fmt.Errorf("failed to fetch block at height %d: panic: %v", height, panicValue))
1464+
}
14671465

1468-
// When we cannot retrieve a bloom for the EVM-only view (all zeroes),
1469-
// skip the bloom pre-filter instead of short-circuiting the block.
1470-
if blockBloom != (ethtypes.Bloom{}) && !MatchFilters(blockBloom, bloomIndexes) {
1471-
<-f.dbReadSemaphore
1472-
wpMetrics.RecordDBSemaphoreRelease()
1473-
continue // skip the block if bloom filter does not match
1474-
}
1475-
}
1466+
// reportBatchError sends err on errChan without blocking, dropping it when errChan is full.
1467+
func reportBatchError(errChan chan<- error, err error) {
1468+
select {
1469+
case errChan <- err:
1470+
default: // fetchBlocksByCrit reports only the first failure
1471+
}
1472+
}
14761473

1477-
// fetch block from network
1478-
block, err := blockByNumberRespectingWatermarks(ctx, f.tmClient, f.watermarks, &height, 1)
1479-
if err != nil {
1480-
select {
1481-
case errChan <- fmt.Errorf("failed to fetch block at height %d: %w", height, err):
1482-
default:
1474+
// readUncachedBlock reads the block at height under a DB-read slot and adds it to the block cache.
1475+
// It returns ok=false, with no error, when the block's bloom rules out crit or when a copy another
1476+
// worker cached meanwhile is outside the available range.
1477+
func (f *LogFetcher) readUncachedBlock(ctx context.Context, height int64, crit filters.FilterCriteria, bloomIndexes [][]BloomIndexes) (block *coretypes.ResultBlock, ok bool, err error) {
1478+
release := f.acquireDBReadSlot()
1479+
defer release()
1480+
1481+
// Re-check cache after acquiring semaphore, in case another worker cached it.
1482+
if cachedEntry, found := f.globalBlockCache.Get(height); found {
1483+
if cachedEntry.Block != nil {
1484+
if err := f.watermarks.EnsureBlockHeightAvailable(ctx, cachedEntry.Block.Block.Height); err != nil {
1485+
return nil, false, nil
14831486
}
1484-
<-f.dbReadSemaphore
1485-
wpMetrics.RecordDBSemaphoreRelease()
1486-
continue
1487+
}
1488+
return cachedEntry.Block, true, nil
1489+
}
1490+
1491+
// check bloom filter if cache miss AND we have filters
1492+
var blockBloom ethtypes.Bloom
1493+
if len(crit.Addresses) != 0 || len(crit.Topics) != 0 {
1494+
// Bloom cache miss - read from database
1495+
providerCtx := f.ctxProvider(height)
1496+
if f.includeSyntheticReceipts {
1497+
blockBloom = f.k.GetBlockBloom(providerCtx)
1498+
} else {
1499+
blockBloom = f.k.GetEvmOnlyBlockBloom(providerCtx)
14871500
}
14881501

1489-
// Use LoadOrStore to create/get cache entry atomically
1490-
entry := loadOrStoreCacheEntry(f.cacheCreationMutex, f.globalBlockCache, height, block)
1491-
// Fill bloom if we have it and it's missing
1492-
if blockBloom != (ethtypes.Bloom{}) {
1493-
fillMissingFields(entry, block, blockBloom)
1502+
// When we cannot retrieve a bloom for the EVM-only view (all zeroes),
1503+
// skip the bloom pre-filter instead of short-circuiting the block.
1504+
if blockBloom != (ethtypes.Bloom{}) && !MatchFilters(blockBloom, bloomIndexes) {
1505+
return nil, false, nil // skip the block if bloom filter does not match
14941506
}
1507+
}
1508+
1509+
// fetch block from network
1510+
block, err = blockByNumberRespectingWatermarks(ctx, f.tmClient, f.watermarks, &height, 1)
1511+
if err != nil {
1512+
return nil, false, err
1513+
}
1514+
1515+
// Use LoadOrStore to create/get cache entry atomically
1516+
entry := loadOrStoreCacheEntry(f.cacheCreationMutex, f.globalBlockCache, height, block)
1517+
// Fill bloom if we have it and it's missing
1518+
if blockBloom != (ethtypes.Bloom{}) {
1519+
fillMissingFields(entry, block, blockBloom)
1520+
}
1521+
return block, true, nil
1522+
}
1523+
1524+
// acquireDBReadSlot blocks until a DB-read slot is free and returns the function that frees it.
1525+
// Callers must defer the returned function so the slot is freed on every exit, panics included:
1526+
// GetLogs turns requests away while too many slots are held.
1527+
func (f *LogFetcher) acquireDBReadSlot() (release func()) {
1528+
wpMetrics := GetGlobalMetrics()
1529+
semWaitStart := time.Now()
1530+
f.dbReadSemaphore <- struct{}{}
1531+
wpMetrics.RecordDBSemaphoreWait(time.Since(semWaitStart))
1532+
wpMetrics.RecordDBSemaphoreAcquire()
1533+
return func() {
14951534
<-f.dbReadSemaphore
14961535
wpMetrics.RecordDBSemaphoreRelease()
1497-
res <- block
14981536
}
14991537
}

‎evmrpc/filter_db_read_slot_test.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package evmrpc
2+
3+
import (
4+
"context"
5+
"math/big"
6+
"sync"
7+
"testing"
8+
"time"
9+
10+
"github.com/ethereum/go-ethereum/eth/filters"
11+
evmrpcconfig "github.com/sei-protocol/sei-chain/evmrpc/config"
12+
"github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
const blockStorePanic = "error from proto block: wrong Header.LastCommitHash"
17+
18+
// panickingBlockTestClient panics when asked for panicHeight, the way the block store does when it
19+
// cannot decode a stored block.
20+
type panickingBlockTestClient struct {
21+
*heightTestClient
22+
panicHeight int64
23+
}
24+
25+
func (c *panickingBlockTestClient) Block(ctx context.Context, height *int64) (*coretypes.ResultBlock, error) {
26+
if height != nil && *height == c.panicHeight {
27+
panic(blockStorePanic)
28+
}
29+
return c.heightTestClient.Block(ctx, height)
30+
}
31+
32+
// newPanickingBlockFetcher returns a LogFetcher over blocks 1 to latest whose block store panics at
33+
// panicHeight. It has a single DB-read slot, so a leaked slot stalls every later read.
34+
func newPanickingBlockFetcher(latest, panicHeight int64) *LogFetcher {
35+
client := &panickingBlockTestClient{heightTestClient: newHeightTestClient(latest+10, 1, latest), panicHeight: panicHeight}
36+
return &LogFetcher{
37+
tmClient: client,
38+
dbReadSemaphore: make(chan struct{}, 1),
39+
globalBlockCache: NewBlockCache(int(latest)),
40+
cacheCreationMutex: &sync.Mutex{},
41+
watermarks: newHeightTestWatermarks(client, latest),
42+
}
43+
}
44+
45+
// A block read that panics must free its DB-read slot, or GetLogs ends up rejecting every request as
46+
// I/O saturated, and must come back as an error rather than escape the batch.
47+
func TestProcessBatchReportsPanickingBlockReadAndFreesDBReadSlot(t *testing.T) {
48+
t.Parallel()
49+
50+
const panicHeight = int64(10)
51+
fetcher := newPanickingBlockFetcher(90, panicHeight)
52+
res := make(chan *coretypes.ResultBlock, 1)
53+
errChan := make(chan error, 1)
54+
55+
require.NotPanics(t, func() {
56+
fetcher.processBatch(context.Background(), panicHeight, panicHeight, filters.FilterCriteria{}, nil, res, errChan)
57+
})
58+
59+
require.Empty(t, fetcher.dbReadSemaphore)
60+
require.Empty(t, res)
61+
require.Len(t, errChan, 1)
62+
err := <-errChan
63+
require.ErrorContains(t, err, "height 10")
64+
require.ErrorContains(t, err, blockStorePanic)
65+
}
66+
67+
// A panic in one batch of a multi-batch request must fail the request. Left to the worker pool, which
68+
// recovers it silently, the request would return the other batches' blocks as if they were the answer.
69+
//
70+
// Not parallel: it reads the global worker pool metrics, which parallel tests also move.
71+
func TestFetchBlocksByCritFailsWhenABlockReadPanics(t *testing.T) {
72+
const (
73+
batchSize = int64(evmrpcconfig.WorkerBatchSize)
74+
latest = 3 * batchSize
75+
panicHeight = batchSize + batchSize/2 // inside the second of three batches
76+
)
77+
metrics := GetGlobalMetrics()
78+
slotsHeldBefore := metrics.DBSemaphoreAcquired.Load()
79+
panicsBefore := metrics.TasksPanicked.Load()
80+
fetcher := newPanickingBlockFetcher(latest, panicHeight)
81+
crit := filters.FilterCriteria{FromBlock: big.NewInt(1), ToBlock: big.NewInt(latest)}
82+
83+
type fetchResult struct {
84+
blocks chan *coretypes.ResultBlock
85+
err error
86+
}
87+
done := make(chan fetchResult, 1)
88+
go func() {
89+
blocks, _, err := fetcher.fetchBlocksByCrit(context.Background(), crit, 0, nil)
90+
done <- fetchResult{blocks: blocks, err: err}
91+
}()
92+
93+
var got fetchResult
94+
select {
95+
case got = <-done:
96+
case <-time.After(10 * time.Second):
97+
t.Fatal("fetchBlocksByCrit did not return, most likely because a DB-read slot leaked")
98+
}
99+
100+
require.ErrorContains(t, got.err, blockStorePanic)
101+
require.Nil(t, got.blocks)
102+
require.Empty(t, fetcher.dbReadSemaphore)
103+
require.Equal(t, slotsHeldBefore, metrics.DBSemaphoreAcquired.Load())
104+
require.Equal(t, panicsBefore+1, metrics.TasksPanicked.Load())
105+
}

0 commit comments

Comments
 (0)