Skip to content

Commit fc7cf5f

Browse files
Cherry pick eth_getLogs performance with early rejection and backpressure (#2606)
## Describe your changes and provide context Cherry picks #2591 into v6.2.0 ## Testing performed to validate your change Tested with RPC providers --------- Co-authored-by: blindchaser <zengyiren0@gmail.com>
1 parent c18a74c commit fc7cf5f

5 files changed

Lines changed: 1028 additions & 13 deletions

File tree

‎evmrpc/filter.go‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ import (
2929
const TxSearchPerPage = 10
3030

3131
const (
32-
// DB Concurrency Read Limit
32+
// MaxDBReadConcurrency is the default DB read concurrency limit.
33+
// In practice, this is aligned with WorkerPoolSize at runtime.
34+
// This constant is only used as a fallback for metrics initialization.
3335
MaxDBReadConcurrency = 16
3436

3537
// Default request limits (used as fallback values)
@@ -530,7 +532,9 @@ func (a *FilterAPI) GetFilterLogs(
530532
}
531533

532534
func (a *FilterAPI) GetLogs(ctx context.Context, crit filters.FilterCriteria) (res []*ethtypes.Log, err error) {
533-
defer recordMetrics(fmt.Sprintf("%s_getLogs", a.namespace), a.connectionType, time.Now())
535+
startTime := time.Now()
536+
defer recordMetrics(fmt.Sprintf("%s_getLogs", a.namespace), a.connectionType, startTime)
537+
534538
// Calculate block range
535539
latest := a.logFetcher.ctxProvider(LatestCtxHeight).BlockHeight()
536540
begin, end := latest, latest
@@ -546,16 +550,44 @@ func (a *FilterAPI) GetLogs(ctx context.Context, crit filters.FilterCriteria) (r
546550

547551
blockRange := end - begin + 1
548552

553+
// Record metrics for eth_getLogs
554+
defer func() {
555+
GetGlobalMetrics().RecordGetLogsRequest(blockRange, time.Since(startTime), startTime, err)
556+
}()
557+
549558
// Use config value instead of hardcoded constant
550559
if blockRange > a.filterConfig.maxBlock {
551560
return nil, fmt.Errorf("block range too large (%d), maximum allowed is %d blocks", blockRange, a.filterConfig.maxBlock)
552561
}
553562

563+
// Early rejection for pruned blocks - avoid wasting resources on blocks that don't exist
564+
earliest := a.logFetcher.earliestVersion()
565+
if earliest > 0 && begin < earliest {
566+
return nil, fmt.Errorf("requested block range [%d, %d] includes pruned blocks, earliest available block is %d", begin, end, earliest)
567+
}
568+
554569
// Only apply rate limiting for large queries (> RPSLimitThreshold blocks)
555570
if blockRange > RPSLimitThreshold && !a.globalRPSLimiter.Allow() {
556571
return nil, fmt.Errorf("log query rate limit exceeded for large queries, please try again later")
557572
}
558573

574+
// Backpressure: early rejection based on system load
575+
m := GetGlobalMetrics()
576+
577+
// Check 1: Too many pending tasks (queue backlog)
578+
pending := m.TasksSubmitted.Load() - m.TasksCompleted.Load()
579+
maxPending := int64(float64(m.QueueCapacity) * 0.8) // 80% threshold
580+
if pending > maxPending {
581+
return nil, fmt.Errorf("server too busy, rejecting new request (pending: %d, threshold: %d)", pending, maxPending)
582+
}
583+
584+
// Check 2: I/O saturated (semaphore exhausted)
585+
semInUse := m.DBSemaphoreAcquired.Load()
586+
semCapacity := m.DBSemaphoreCapacity
587+
if semCapacity > 0 && float64(semInUse)/float64(semCapacity) >= 0.8 {
588+
return nil, fmt.Errorf("server I/O saturated, rejecting new request (semaphore: %d/%d in use)", semInUse, semCapacity)
589+
}
590+
559591
logs, _, err := a.logFetcher.GetLogsByFilters(ctx, crit, 0)
560592
if err != nil {
561593
return nil, err
@@ -741,7 +773,7 @@ func (f *LogFetcher) GetLogsByFilters(ctx context.Context, crit filters.FilterCr
741773
batch := blockBatch
742774
wg.Add(1)
743775

744-
if err := runner.Submit(func() { processBatch(batch) }); err != nil {
776+
if err := runner.SubmitWithMetrics(func() { processBatch(batch) }); err != nil {
745777
wg.Done()
746778
submitError = fmt.Errorf("system overloaded, please reduce request frequency: %w", err)
747779
break
@@ -757,7 +789,7 @@ func (f *LogFetcher) GetLogsByFilters(ctx context.Context, crit filters.FilterCr
757789
// Process remaining blocks
758790
if len(blockBatch) > 0 {
759791
wg.Add(1)
760-
if err := runner.Submit(func() { processBatch(blockBatch) }); err != nil {
792+
if err := runner.SubmitWithMetrics(func() { processBatch(blockBatch) }); err != nil {
761793
wg.Done()
762794
return nil, 0, fmt.Errorf("system overloaded, please reduce request frequency: %w", err)
763795
}
@@ -977,7 +1009,7 @@ func (f *LogFetcher) fetchBlocksByCrit(ctx context.Context, crit filters.FilterC
9771009
}
9781010

9791011
wg.Add(1)
980-
if err := runner.Submit(func(start, endHeight int64) func() {
1012+
if err := runner.SubmitWithMetrics(func(start, endHeight int64) func() {
9811013
return func() {
9821014
defer wg.Done()
9831015
f.processBatch(ctx, start, endHeight, crit, bloomIndexes, res, errChan)
@@ -1014,6 +1046,9 @@ func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit fi
10141046
defer func() {
10151047
metrics.IncrementRpcRequestCounter("num_blocks_fetched", "blocks", true)
10161048
}()
1049+
1050+
wpMetrics := GetGlobalMetrics()
1051+
10171052
for height := start; height <= end; height++ {
10181053
if height == 0 {
10191054
continue
@@ -1026,11 +1061,15 @@ func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit fi
10261061
}
10271062

10281063
// Block cache miss, acquire semaphore for I/O operations
1064+
semWaitStart := time.Now()
10291065
f.dbReadSemaphore <- struct{}{}
1066+
wpMetrics.RecordDBSemaphoreWait(time.Since(semWaitStart))
1067+
wpMetrics.RecordDBSemaphoreAcquire()
10301068

10311069
// Re-check cache after acquiring semaphore, in case another worker cached it.
10321070
if cachedEntry, found := f.globalBlockCache.Get(height); found {
10331071
<-f.dbReadSemaphore
1072+
wpMetrics.RecordDBSemaphoreRelease()
10341073
res <- cachedEntry.Block
10351074
continue
10361075
}
@@ -1046,6 +1085,7 @@ func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit fi
10461085
// skip the bloom pre-filter instead of short-circuiting the block.
10471086
if blockBloom != (ethtypes.Bloom{}) && !MatchFilters(blockBloom, bloomIndexes) {
10481087
<-f.dbReadSemaphore
1088+
wpMetrics.RecordDBSemaphoreRelease()
10491089
continue // skip the block if bloom filter does not match
10501090
}
10511091
}
@@ -1058,6 +1098,7 @@ func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit fi
10581098
default:
10591099
}
10601100
<-f.dbReadSemaphore
1101+
wpMetrics.RecordDBSemaphoreRelease()
10611102
continue
10621103
}
10631104

@@ -1068,6 +1109,7 @@ func (f *LogFetcher) processBatch(ctx context.Context, start, end int64, crit fi
10681109
fillMissingFields(entry, block, blockBloom)
10691110
}
10701111
<-f.dbReadSemaphore
1112+
wpMetrics.RecordDBSemaphoreRelease()
10711113
res <- block
10721114
}
10731115
}

‎evmrpc/server.go‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package evmrpc
22

33
import (
44
"context"
5+
"runtime"
56
"strings"
67
"sync"
8+
"time"
79

810
"github.com/cosmos/cosmos-sdk/baseapp"
911
"github.com/cosmos/cosmos-sdk/client"
@@ -47,8 +49,27 @@ func NewEVMHTTPServer(
4749
// Initialize global worker pool with configuration
4850
InitGlobalWorkerPool(config.WorkerPoolSize, config.WorkerQueueSize)
4951

50-
// Initialize RPC tracker (stats package not available in release branch)
51-
// stats.InitRPCTracker(ctxProvider(LatestCtxHeight).Context(), logger, config.RPCStatsInterval)
52+
// Initialize global metrics with worker pool and DB semaphore configuration
53+
workerCount := config.WorkerPoolSize
54+
if workerCount <= 0 {
55+
workerCount = min(MaxWorkerPoolSize, runtime.NumCPU()*2)
56+
}
57+
queueSize := config.WorkerQueueSize
58+
if queueSize <= 0 {
59+
queueSize = DefaultWorkerQueueSize
60+
}
61+
// Align DB semaphore with worker count - each worker gets one I/O slot
62+
dbSemaphoreSize := workerCount
63+
InitGlobalMetrics(workerCount, queueSize, dbSemaphoreSize)
64+
65+
// Start metrics printer (every 5 seconds)
66+
// Prometheus metrics are always exported; stdout printing requires EVM_DEBUG_METRICS=true
67+
StartMetricsPrinter(5 * time.Second)
68+
debugEnabled := IsDebugMetricsEnabled()
69+
logger.Info("Started EVM RPC metrics exporter (interval: 5s)", "workers", workerCount, "queue", queueSize, "db_semaphore", dbSemaphoreSize, "debug_stdout", debugEnabled)
70+
if !debugEnabled {
71+
logger.Info("To enable debug metrics output to stdout, set EVM_DEBUG_METRICS=true")
72+
}
5273

5374
httpServer := NewHTTPServer(logger, rpc.HTTPTimeouts{
5475
ReadTimeout: config.ReadTimeout,
@@ -80,7 +101,8 @@ func NewEVMHTTPServer(
80101
seiTxAPI := NewSeiTransactionAPI(tmClient, k, ctxProvider, txConfigProvider, earliestVersion, homeDir, ConnectionTypeHTTP, isPanicOrSyntheticTxFunc, globalBlockCache, cacheCreationMutex)
81102
seiDebugAPI := NewSeiDebugAPI(tmClient, k, ctxProvider, txConfigProvider, earliestVersion, simulateConfig, app, antehandler, ConnectionTypeHTTP, config, globalBlockCache, cacheCreationMutex)
82103

83-
dbReadSemaphore := make(chan struct{}, MaxDBReadConcurrency)
104+
// DB semaphore aligned with worker count
105+
dbReadSemaphore := make(chan struct{}, dbSemaphoreSize)
84106
globalLogSlicePool := NewLogSlicePool()
85107
apis := []rpc.API{
86108
{
@@ -194,8 +216,21 @@ func NewEVMWebSocketServer(
194216
// Initialize global worker pool with configuration
195217
InitGlobalWorkerPool(config.WorkerPoolSize, config.WorkerQueueSize)
196218

197-
// Initialize WebSocket tracker (stats package not available in release branch)
198-
// stats.InitWSTracker(ctxProvider(LatestCtxHeight).Context(), logger, config.RPCStatsInterval)
219+
// Initialize global metrics (idempotent - only first call takes effect)
220+
workerCountWS := config.WorkerPoolSize
221+
if workerCountWS <= 0 {
222+
workerCountWS = min(MaxWorkerPoolSize, runtime.NumCPU()*2)
223+
}
224+
queueSizeWS := config.WorkerQueueSize
225+
if queueSizeWS <= 0 {
226+
queueSizeWS = DefaultWorkerQueueSize
227+
}
228+
// Align DB semaphore with worker count
229+
dbSemaphoreSizeWS := workerCountWS
230+
InitGlobalMetrics(workerCountWS, queueSizeWS, dbSemaphoreSizeWS)
231+
232+
// Start metrics printer (idempotent - only first call starts printer)
233+
StartMetricsPrinter(5 * time.Second)
199234

200235
httpServer := NewHTTPServer(logger, rpc.HTTPTimeouts{
201236
ReadTimeout: config.ReadTimeout,
@@ -211,7 +246,8 @@ func NewEVMWebSocketServer(
211246
EVMTimeout: config.SimulationEVMTimeout,
212247
MaxConcurrentSimulationCalls: config.MaxConcurrentSimulationCalls,
213248
}
214-
dbReadSemaphore := make(chan struct{}, MaxDBReadConcurrency)
249+
// DB semaphore aligned with worker count
250+
dbReadSemaphore := make(chan struct{}, dbSemaphoreSizeWS)
215251
globalBlockCache := NewBlockCache(3000)
216252
cacheCreationMutex := &sync.Mutex{}
217253
globalLogSlicePool := NewLogSlicePool()

‎evmrpc/subscribe.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,11 +168,17 @@ func (a *SubscriptionAPI) Logs(ctx context.Context, filter *filters.FilterCriter
168168

169169
rpcSub := notifier.CreateSubscription()
170170

171+
// Track subscription metrics
172+
wpMetrics := GetGlobalMetrics()
173+
wpMetrics.RecordSubscriptionStart()
174+
171175
if filter.BlockHash != nil {
172176
go func() {
173177
defer recoverAndLog()
178+
defer wpMetrics.RecordSubscriptionEnd()
174179
logs, _, err := a.logFetcher.GetLogsByFilters(ctx, *filter, 0)
175180
if err != nil {
181+
wpMetrics.RecordSubscriptionError()
176182
_ = notifier.Notify(rpcSub.ID, err)
177183
return
178184
}
@@ -187,10 +193,12 @@ func (a *SubscriptionAPI) Logs(ctx context.Context, filter *filters.FilterCriter
187193

188194
go func() {
189195
defer recoverAndLog()
196+
defer wpMetrics.RecordSubscriptionEnd()
190197
begin := int64(0)
191198
for {
192199
logs, lastToHeight, err := a.logFetcher.GetLogsByFilters(ctx, *filter, begin)
193200
if err != nil {
201+
wpMetrics.RecordSubscriptionError()
194202
_ = notifier.Notify(rpcSub.ID, err)
195203
return
196204
}

‎evmrpc/worker_pool.go‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"runtime"
66
"sync"
7+
"time"
78
)
89

910
const (
@@ -106,22 +107,58 @@ func (wp *WorkerPool) start() {
106107
}
107108
}()
108109
// The worker will exit gracefully when the taskQueue is closed and drained.
109-
for task := range wp.taskQueue {
110+
for wrappedTask := range wp.taskQueue {
110111
func() {
111112
defer func() {
112113
if r := recover(); r != nil {
113114
// Log the panic but continue processing other tasks
114115
fmt.Printf("Task recovered from panic: %v\n", r)
116+
GetGlobalMetrics().RecordTaskPanicked()
115117
}
116118
}()
117-
task()
119+
wrappedTask()
118120
}()
119121
}
120122
}()
121123
}
122124
})
123125
}
124126

127+
// SubmitWithMetrics submits a task with full metrics tracking
128+
func (wp *WorkerPool) SubmitWithMetrics(task func()) error {
129+
metrics := GetGlobalMetrics()
130+
131+
// Check if pool is closed first
132+
wp.mu.RLock()
133+
if wp.closed {
134+
wp.mu.RUnlock()
135+
return fmt.Errorf("worker pool is closing")
136+
}
137+
wp.mu.RUnlock()
138+
139+
queuedAt := time.Now()
140+
141+
// Wrap the task with metrics
142+
wrappedTask := func() {
143+
startedAt := time.Now()
144+
metrics.RecordTaskStarted(queuedAt)
145+
defer metrics.RecordTaskCompleted(startedAt)
146+
task()
147+
}
148+
149+
select {
150+
case wp.taskQueue <- wrappedTask:
151+
metrics.RecordTaskSubmitted()
152+
return nil
153+
case <-wp.done:
154+
return fmt.Errorf("worker pool is closing")
155+
default:
156+
// Queue is full - fail fast
157+
metrics.RecordTaskRejected()
158+
return fmt.Errorf("worker pool queue is full")
159+
}
160+
}
161+
125162
// Submit submits a task to the worker pool with fail-fast behavior
126163
// Returns error if queue is full or pool is closing
127164
func (wp *WorkerPool) Submit(task func()) error {
@@ -168,3 +205,17 @@ func (wp *WorkerPool) WorkerCount() int {
168205
func (wp *WorkerPool) QueueSize() int {
169206
return cap(wp.taskQueue)
170207
}
208+
209+
// QueueDepth returns the current number of tasks in the queue
210+
func (wp *WorkerPool) QueueDepth() int {
211+
return len(wp.taskQueue)
212+
}
213+
214+
// QueueUtilization returns the percentage of queue capacity in use
215+
func (wp *WorkerPool) QueueUtilization() float64 {
216+
cap := cap(wp.taskQueue)
217+
if cap == 0 {
218+
return 0
219+
}
220+
return float64(len(wp.taskQueue)) / float64(cap) * 100
221+
}

0 commit comments

Comments
 (0)