Skip to content

Commit b57bd8b

Browse files
Wire deadline enforcer into gRPC plane (#4343)
## Summary - Wire a deadline enforcer into the gRPC plane: unary and stream interceptors bound each request's context by a configurable per-method default deadline, and normalize a bare `context.DeadlineExceeded` into `codes.DeadlineExceeded`. - Add `grpc.request-timeout` config (`DefaultGRPCRequestTimeout` = 30s), applied to both native gRPC (:9090) and gRPC-Web (:9091). Deadline interceptors run after the rate-limit interceptors, so admission rejections don't eat into the request's time budget. `0` disables the default deadline. - Thread the caller's `context.Context` through query execution: `enrichABCIQueryContext` attaches it to `sdk.Context` so gRPC module queries, ABCI `/abci_query`, and custom queriers can honor deadlines in store/MVCC iteration (`ctxkv`). - Thread `context.Context` through `BaseApp.Simulate` (and `handleQueryApp` / `txServer.Simulate`) so simulate RPC inherits the bounded gRPC context; report simulate's `context.DeadlineExceeded` as `codes.DeadlineExceeded` instead of `codes.Unknown`. - During simulation, abort long-running work when the context expires: `RunMsgs` checks between messages; EVM execution uses `context.AfterFunc` + `evm.Cancel` (DeliverTx unchanged). Context cancel/deadline is not logged or counted as a state-transition error. - Stream handlers observe the bounded context on `RecvMsg` / `SendMsg`, not only on `Context()`. ## Test plan - [x] `sei-cosmos/server/grpc/deadline_test.go` — unary/stream interceptor behavior (default deadline, client deadline wins, success path, non-deadline errors pass through, stream context override, RecvMsg/SendMsg bounded by deadline) - [x] `sei-cosmos/x/auth/tx/service_test.go` — Simulate reports `DeadlineExceeded` distinctly from other errors - [x] `sei-cosmos/server/config/config_test.go` — default, override, and negative-duration clamp for `grpc.request-timeout` - [x] `config/cosmosbase` characterization suite updated for the new config key - [x] `sei-cosmos/baseapp/query_trust_test.go` — caller context preserved on enriched ABCI/gRPC-over-ABCI queries - [x] `sei-cosmos/baseapp/grpcserver_test.go` — gRPC server path threads server interceptor deadline into `sdk.Context` - [x] `sei-cosmos/baseapp/deliver_tx_test.go` — Simulate propagates caller context to message handlers - [x] `x/evm/keeper/msg_server_test.go` — simulation deadline cancels in-flight EVM execution - [x] `x/evm/keeper/msg_server_internal_test.go` — context cancellation error classification --------- Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent bd9355a commit b57bd8b

21 files changed

Lines changed: 778 additions & 21 deletions

‎app/test_helpers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -975,7 +975,7 @@ func SignCheckDeliver(
975975
require.Nil(t, err)
976976

977977
// Must simulate now as CheckTx doesn't run Msgs anymore
978-
_, res, err := app.Simulate(txBytes)
978+
_, res, err := app.Simulate(t.Context(), txBytes)
979979

980980
if expSimPass {
981981
require.NoError(t, err)

‎config/cosmosbase/agreement_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ func readerValues(t *testing.T) map[string]string {
115115
"grpc.max-in-flight-per-ip": fmt.Sprint(cfg.GRPC.MaxInFlightPerIP),
116116
"grpc.rate-limiting-enabled": fmt.Sprint(cfg.GRPC.RateLimitingEnabled),
117117
"grpc.trusted-proxy-cidrs": fmt.Sprint(cfg.GRPC.TrustedProxyCIDRs),
118+
"grpc.request-timeout": fmt.Sprint(cfg.GRPC.RequestTimeout),
118119
"telemetry.service-name": fmt.Sprint(cfg.Telemetry.ServiceName),
119120
"telemetry.enabled": fmt.Sprint(cfg.Telemetry.Enabled),
120121
"telemetry.enable-hostname": fmt.Sprint(cfg.Telemetry.EnableHostname),

‎config/cosmosbase/cosmosbase_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func TestTheGRPCKeysAreTheOnesItsReaderResolves(t *testing.T) {
7979
"grpc.keepalive-time", "grpc.keepalive-timeout", "grpc.keepalive-min-time",
8080
"grpc.keepalive-permit-without-stream",
8181
"grpc.ip-rate-limit-rps", "grpc.ip-rate-limit-burst", "grpc.max-in-flight-per-ip",
82-
"grpc.rate-limiting-enabled", "grpc.trusted-proxy-cidrs",
82+
"grpc.rate-limiting-enabled", "grpc.trusted-proxy-cidrs", "grpc.request-timeout",
8383
})
8484
}
8585

‎sei-cosmos/baseapp/abci.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ func (app *BaseApp) Query(ctx context.Context, req *abci.RequestQuery) (res *abc
457457
switch path[0] {
458458
// "/app" prefix for special application queries
459459
case "app":
460-
resp = handleQueryApp(app, path, *req)
460+
resp = handleQueryApp(app, ctx, path, *req)
461461

462462
case "store":
463463
resp = handleQueryStore(ctx, app, path, *req)
@@ -812,8 +812,8 @@ func (app *BaseApp) GetBlockRetentionHeight(commitHeight int64) (int64, error) {
812812
return retentionHeight, nil
813813
}
814814

815-
func (app *BaseApp) Simulate(txBytes []byte) (sdk.GasInfo, *sdk.Result, error) {
816-
ctx := app.checkState.ctx.WithTxBytes(txBytes).WithConsensusParams(app.GetConsensusParams(app.checkState.ctx))
815+
func (app *BaseApp) Simulate(goCtx context.Context, txBytes []byte) (sdk.GasInfo, *sdk.Result, error) {
816+
ctx := app.checkState.ctx.WithTxBytes(txBytes).WithConsensusParams(app.GetConsensusParams(app.checkState.ctx)).WithContext(goCtx)
817817
ctx, _ = ctx.CacheContext()
818818
tx, err := app.txDecoder(txBytes)
819819
if err != nil {
@@ -823,13 +823,13 @@ func (app *BaseApp) Simulate(txBytes []byte) (sdk.GasInfo, *sdk.Result, error) {
823823
return runTxRes.gasInfo, runTxRes.result, err
824824
}
825825

826-
func handleQueryApp(app *BaseApp, path []string, req abci.RequestQuery) abci.ResponseQuery {
826+
func handleQueryApp(app *BaseApp, ctx context.Context, path []string, req abci.RequestQuery) abci.ResponseQuery {
827827
if len(path) >= 2 {
828828
switch path[1] {
829829
case "simulate":
830830
txBytes := req.Data
831831

832-
gInfo, res, err := app.Simulate(txBytes)
832+
gInfo, res, err := app.Simulate(ctx, txBytes)
833833
if err != nil {
834834
return sdkerrors.QueryResultWithDebug(sdkerrors.Wrap(err, "failed to simulate tx"), app.trace)
835835
}

‎sei-cosmos/baseapp/baseapp.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1061,6 +1061,12 @@ func (app *BaseApp) RunMsgs(ctx sdk.Context, msgs []sdk.Msg) (*sdk.Result, error
10611061
err error
10621062
)
10631063

1064+
if ctx.IsSimulation() {
1065+
if err := ctx.Context().Err(); err != nil {
1066+
return nil, err
1067+
}
1068+
}
1069+
10641070
msgCtx, msgMsCache := app.CacheTxContext(ctx, [32]byte{})
10651071
msgCtx = msgCtx.WithMessageIndex(i)
10661072

‎sei-cosmos/baseapp/deliver_tx_test.go‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -474,13 +474,13 @@ func TestSimulateTx(t *testing.T) {
474474
require.Nil(t, err)
475475

476476
// simulate a message, check gas reported
477-
gInfo, result, err := app.Simulate(txBytes)
477+
gInfo, result, err := app.Simulate(context.Background(), txBytes)
478478
require.NoError(t, err)
479479
require.NotNil(t, result)
480480
require.Equal(t, gasConsumed, gInfo.GasUsed)
481481

482482
// simulate again, same result
483-
gInfo, result, err = app.Simulate(txBytes)
483+
gInfo, result, err = app.Simulate(context.Background(), txBytes)
484484
require.NoError(t, err)
485485
require.NotNil(t, result)
486486
require.Equal(t, gasConsumed, gInfo.GasUsed)
@@ -508,6 +508,40 @@ func TestSimulateTx(t *testing.T) {
508508
}
509509
}
510510

511+
// TestSimulatePropagatesCallerContext pins that the context.Context Simulate
512+
// is called with reaches message execution through sdk.Context, rather than
513+
// stopping at the interceptor boundary. A deadline on that context must bound
514+
// the run itself; this only proves the plumbing carries it as far as the
515+
// message handler sees, not that every store operation honors it.
516+
func TestSimulatePropagatesCallerContext(t *testing.T) {
517+
type ctxKey struct{}
518+
want := context.WithValue(t.Context(), ctxKey{}, "caller-context")
519+
520+
var gotCtx context.Context
521+
routerOpt := func(bapp *BaseApp) {
522+
r := sdk.NewRoute(routeMsgCounter, func(ctx sdk.Context, msg sdk.Msg) (*sdk.Result, error) {
523+
gotCtx = ctx.Context()
524+
return &sdk.Result{}, nil
525+
})
526+
bapp.Router().AddRoute(r)
527+
}
528+
529+
app := setupBaseApp(t, routerOpt)
530+
app.InitChain(&abci.RequestInitChain{})
531+
532+
cdc := codec.NewLegacyAmino()
533+
registerTestCodec(cdc)
534+
app.setDeliverState(tmproto.Header{Height: 1})
535+
536+
tx := newTxCounter(1, 0)
537+
txBytes, err := cdc.Marshal(tx)
538+
require.NoError(t, err)
539+
540+
_, _, err = app.Simulate(want, txBytes)
541+
require.NoError(t, err)
542+
require.Equal(t, "caller-context", gotCtx.Value(ctxKey{}))
543+
}
544+
511545
func TestInitChainGenesisFlagDoesNotLeakToCheckOrSimulateBeforeFirstCommit(t *testing.T) {
512546
type anteCall struct {
513547
simulate bool
@@ -560,7 +594,7 @@ func TestInitChainGenesisFlagDoesNotLeakToCheckOrSimulateBeforeFirstCommit(t *te
560594

561595
txBytes, err := aminoTxEncoder()(tx)
562596
require.NoError(t, err)
563-
_, _, err = app.Simulate(txBytes)
597+
_, _, err = app.Simulate(context.Background(), txBytes)
564598
require.NoError(t, err)
565599

566600
require.Len(t, anteCalls, 2)

‎sei-cosmos/baseapp/grpcserver_test.go‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"net"
66
"sync/atomic"
77
"testing"
8+
"time"
89

910
"github.com/stretchr/testify/require"
1011
"google.golang.org/grpc"
@@ -14,8 +15,19 @@ import (
1415

1516
"github.com/sei-protocol/sei-chain/sei-cosmos/codec/types"
1617
"github.com/sei-protocol/sei-chain/sei-cosmos/testutil/testdata"
18+
sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
1719
)
1820

21+
type contextQuery struct {
22+
testdata.QueryImpl
23+
ctx chan context.Context
24+
}
25+
26+
func (q contextQuery) Echo(ctx context.Context, req *testdata.EchoRequest) (*testdata.EchoResponse, error) {
27+
q.ctx <- sdk.UnwrapSDKContext(ctx).Context()
28+
return q.QueryImpl.Echo(ctx, req)
29+
}
30+
1931
// serveTestQuery registers the testdata Query service on app, exposes it on a real
2032
// grpc.Server built with serverOpts, and returns a client dialled to it.
2133
//
@@ -24,12 +36,16 @@ import (
2436
// only if that path is intact, which invoking the interceptor closure directly
2537
// cannot show.
2638
func serveTestQuery(t *testing.T, app *BaseApp, serverOpts ...grpc.ServerOption) testdata.QueryClient {
39+
return serveTestQueryWithHandler(t, app, testdata.QueryImpl{}, serverOpts...)
40+
}
41+
42+
func serveTestQueryWithHandler(t *testing.T, app *BaseApp, handler testdata.QueryServer, serverOpts ...grpc.ServerOption) testdata.QueryClient {
2743
t.Helper()
2844

2945
interfaceRegistry := types.NewInterfaceRegistry()
3046
testdata.RegisterInterfaces(interfaceRegistry)
3147
app.SetInterfaceRegistry(interfaceRegistry)
32-
testdata.RegisterQueryServer(app.GRPCQueryRouter(), testdata.QueryImpl{})
48+
testdata.RegisterQueryServer(app.GRPCQueryRouter(), handler)
3349

3450
srv := grpc.NewServer(serverOpts...)
3551
app.RegisterGRPCServer(srv)
@@ -86,3 +102,20 @@ func TestRegisterGRPCServerWithoutServerInterceptor(t *testing.T) {
86102
require.NoError(t, err)
87103
require.Equal(t, "hello", res.Message)
88104
}
105+
106+
func TestRegisterGRPCServerThreadsServerContextIntoSDKContext(t *testing.T) {
107+
queryCtx := make(chan context.Context, 1)
108+
bound := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
109+
ctx, cancel := context.WithTimeout(ctx, time.Minute)
110+
defer cancel()
111+
return handler(ctx, req)
112+
}
113+
114+
client := serveTestQueryWithHandler(t, setupBaseApp(t), contextQuery{ctx: queryCtx}, grpc.ChainUnaryInterceptor(bound))
115+
_, err := client.Echo(t.Context(), &testdata.EchoRequest{Message: "hello"})
116+
require.NoError(t, err)
117+
118+
deadline, ok := (<-queryCtx).Deadline()
119+
require.True(t, ok)
120+
require.WithinDuration(t, time.Now().Add(time.Minute), deadline, time.Second)
121+
}

‎sei-cosmos/baseapp/query_trust.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ func warnQueryConfig(cfg srvconfig.QueryConfig) {
9393
}
9494

9595
func (app *BaseApp) enrichABCIQueryContext(ctx context.Context, sdkCtx sdk.Context) sdk.Context {
96-
sdkCtx = sdkCtx.WithIsABCIQuery(true)
96+
sdkCtx = sdkCtx.WithContext(ctx).WithIsABCIQuery(true)
9797

9898
if app.queryConfig.DisableLimits {
9999
return sdkCtx.WithPaginationLimits(sdk.NoPaginationLimits())

‎sei-cosmos/baseapp/query_trust_test.go‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"testing"
77

88
srvconfig "github.com/sei-protocol/sei-chain/sei-cosmos/server/config"
9+
"github.com/sei-protocol/sei-chain/sei-cosmos/testutil/testdata"
910
sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
1011
abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types"
1112
"github.com/stretchr/testify/require"
@@ -31,7 +32,9 @@ func TestEnrichABCIQueryContextUntrusted(t *testing.T) {
3132
trustedOriginMatcher: newTrustedCIDRMatcher(nil),
3233
}
3334

34-
ctx := app.enrichABCIQueryContext(t.Context(), sdk.Context{})
35+
callerCtx := context.WithValue(t.Context(), queryContextTestKey{}, "untrusted")
36+
ctx := app.enrichABCIQueryContext(callerCtx, sdk.Context{})
37+
require.Same(t, callerCtx, ctx.Context())
3538
require.True(t, ctx.IsABCIQuery())
3639
require.True(t, ctx.PaginationLimits().Enforce)
3740
require.Equal(t, uint64(1000), ctx.PaginationLimits().MaxLimit)
@@ -45,6 +48,7 @@ func TestEnrichABCIQueryContextTrusted(t *testing.T) {
4548

4649
pctx := peer.NewContext(t.Context(), &peer.Peer{Addr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 9090}})
4750
ctx := app.enrichABCIQueryContext(pctx, sdk.Context{})
51+
require.Same(t, pctx, ctx.Context())
4852
require.False(t, ctx.PaginationLimits().Enforce)
4953
}
5054

@@ -56,10 +60,24 @@ func TestEnrichABCIQueryContextKillSwitch(t *testing.T) {
5660
trustedOriginMatcher: newTrustedCIDRMatcher(nil),
5761
}
5862

59-
ctx := app.enrichABCIQueryContext(t.Context(), sdk.Context{})
63+
callerCtx := context.WithValue(t.Context(), queryContextTestKey{}, "kill-switch")
64+
ctx := app.enrichABCIQueryContext(callerCtx, sdk.Context{})
65+
require.Same(t, callerCtx, ctx.Context())
6066
require.False(t, ctx.PaginationLimits().Enforce)
6167
}
6268

69+
type queryContextTestKey struct{}
70+
71+
type abciContextQuery struct {
72+
testdata.QueryImpl
73+
ctx *context.Context
74+
}
75+
76+
func (q abciContextQuery) Echo(ctx context.Context, req *testdata.EchoRequest) (*testdata.EchoResponse, error) {
77+
*q.ctx = sdk.UnwrapSDKContext(ctx).Context()
78+
return q.QueryImpl.Echo(ctx, req)
79+
}
80+
6381
func TestStripHostPort(t *testing.T) {
6482
require.Equal(t, "127.0.0.1", stripHostPort("127.0.0.1:9090"))
6583
require.Equal(t, "2001:db8::1", stripHostPort("[2001:db8::1]:9090"))
@@ -78,12 +96,35 @@ func TestCustomQueryEnrichesABCIContext(t *testing.T) {
7896
app := setupBaseApp(t, querierOpt)
7997
app.InitChain(&abci.RequestInitChain{})
8098

81-
res, err := app.Query(context.Background(), &abci.RequestQuery{
99+
callerCtx := context.WithValue(t.Context(), queryContextTestKey{}, "custom-query")
100+
res, err := app.Query(callerCtx, &abci.RequestQuery{
82101
Path: "/custom/capture/test",
83102
})
84103
require.NoError(t, err)
85104
require.Equal(t, []byte("ok"), res.Value)
105+
require.Same(t, callerCtx, captured.Context())
86106
require.True(t, captured.IsABCIQuery())
87107
require.True(t, captured.PaginationLimits().Enforce)
88108
require.Equal(t, srvconfig.DefaultQueryMaxLimit, captured.PaginationLimits().MaxLimit)
89109
}
110+
111+
func TestGRPCABCIQueryEnrichesSDKContext(t *testing.T) {
112+
var captured context.Context
113+
grpcQueryOpt := func(app *BaseApp) {
114+
testdata.RegisterQueryServer(app.GRPCQueryRouter(), abciContextQuery{ctx: &captured})
115+
}
116+
117+
app := setupBaseApp(t, grpcQueryOpt)
118+
app.InitChain(&abci.RequestInitChain{})
119+
reqBytes, err := (&testdata.EchoRequest{Message: "hello"}).Marshal()
120+
require.NoError(t, err)
121+
122+
callerCtx := context.WithValue(t.Context(), queryContextTestKey{}, "grpc-abci-query")
123+
res, err := app.Query(callerCtx, &abci.RequestQuery{
124+
Path: "/testdata.Query/Echo",
125+
Data: reqBytes,
126+
})
127+
require.NoError(t, err)
128+
require.Equal(t, abci.CodeTypeOK, res.Code)
129+
require.Same(t, callerCtx, captured)
130+
}

‎sei-cosmos/server/config/config.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ const (
106106

107107
// DefaultOccEanbled defines whether to use OCC for tx processing
108108
DefaultOccEnabled = true
109+
110+
DefaultGRPCRequestTimeout = 30 * time.Second
109111
)
110112

111113
var (
@@ -321,6 +323,12 @@ type GRPCConfig struct {
321323
// when resolving the client IP for rate limiting. Empty means trust no proxy.
322324
// It applies to gRPC-Web (:9091) as well as native gRPC (:9090).
323325
TrustedProxyCIDRs []string `mapstructure:"trusted-proxy-cidrs"`
326+
327+
// RequestTimeout is the deadline applied to a gRPC request with no shorter
328+
// client-supplied grpc-timeout. It applies to gRPC-Web (:9091) as well as
329+
// native gRPC (:9090), since both are served by the same grpc.Server. 0
330+
// disables the default deadline.
331+
RequestTimeout time.Duration `mapstructure:"request-timeout"`
324332
}
325333

326334
// RateLimiterConfig builds the ratelimiter.Config used by gRPC admission.
@@ -469,6 +477,7 @@ func DefaultConfig() *Config {
469477
MaxInFlightPerIP: DefaultGRPCMaxInFlightPerIP,
470478
RateLimitingEnabled: false,
471479
TrustedProxyCIDRs: nil,
480+
RequestTimeout: DefaultGRPCRequestTimeout,
472481
},
473482
Rosetta: RosettaConfig{
474483
Enable: false,
@@ -683,6 +692,10 @@ func GetConfig(v *viper.Viper) (Config, error) {
683692
if v.IsSet("grpc.trusted-proxy-cidrs") {
684693
grpcTrustedProxyCIDRs = v.GetStringSlice("grpc.trusted-proxy-cidrs")
685694
}
695+
grpcRequestTimeout := DefaultGRPCRequestTimeout
696+
if v.IsSet("grpc.request-timeout") {
697+
grpcRequestTimeout = clampNonNegativeDuration(v.GetDuration("grpc.request-timeout"), DefaultGRPCRequestTimeout)
698+
}
686699

687700
cfg := Config{
688701
BaseConfig: BaseConfig{
@@ -745,6 +758,7 @@ func GetConfig(v *viper.Viper) (Config, error) {
745758
MaxInFlightPerIP: grpcMaxInFlightPerIP,
746759
RateLimitingEnabled: v.GetBool("grpc.rate-limiting-enabled"),
747760
TrustedProxyCIDRs: grpcTrustedProxyCIDRs,
761+
RequestTimeout: grpcRequestTimeout,
748762
},
749763
GRPCWeb: GRPCWebConfig{
750764
Enable: v.GetBool("grpc-web.enable"),

0 commit comments

Comments
 (0)