Skip to content

Commit 4e82f90

Browse files
authored
Precompile query regexps and respect context in event scans (#4760)
Query conditions using `MATCHES` re-parsed their pattern for every value they were tested against, and an invalid pattern was silently treated as a non-match. `Compile` now compiles each pattern once when the query is built and returns an error for an invalid pattern, or when the estimated program size of all patterns in the query, counting instructions and character class ranges, exceeds `maxRegexpSize`. `Environment.Events`, `Unsubscribe` and `BlockSearch` now apply the same `maxQueryLength` limit that `Subscribe` and `TxSearch` already enforce. `Events` stops scanning the event log once the request context is done, and only treats the end of its own wait as an empty result rather than the expiry of the caller's deadline.
1 parent 50b8312 commit 4e82f90

5 files changed

Lines changed: 132 additions & 10 deletions

File tree

‎sei-tendermint/internal/pubsub/query/query.go‎

Lines changed: 50 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,10 @@
1212
package query
1313

1414
import (
15+
"errors"
1516
"fmt"
1617
"regexp"
18+
resyntax "regexp/syntax"
1719
"strconv"
1820
"strings"
1921
"time"
@@ -56,8 +58,9 @@ func MustCompile(query string) *Query {
5658
// Compile compiles the given query AST so it can be used to match events.
5759
func Compile(ast syntax.Query) (*Query, error) {
5860
conds := make([]condition, len(ast))
61+
regexpBudget := maxRegexpSize
5962
for i, q := range ast {
60-
cond, err := compileCondition(q)
63+
cond, err := compileCondition(q, &regexpBudget)
6164
if err != nil {
6265
return nil, fmt.Errorf("compile %s: %w", q, err)
6366
}
@@ -155,7 +158,7 @@ func (c condition) matchesEvent(event types.Event) bool {
155158
return false
156159
}
157160

158-
func compileCondition(cond syntax.Condition) (condition, error) {
161+
func compileCondition(cond syntax.Condition, regexpBudget *int) (condition, error) {
159162
out := condition{tag: cond.Tag}
160163

161164
// Handle existence checks separately to simplify the logic below for
@@ -177,6 +180,13 @@ func compileCondition(cond syntax.Condition) (condition, error) {
177180
switch argType {
178181
case syntax.TString:
179182
argValue = cond.Arg.Value()
183+
if cond.Op == syntax.TMatches {
184+
re, err := compileRegexp(cond.Arg.Value(), regexpBudget)
185+
if err != nil {
186+
return condition{}, fmt.Errorf("invalid regular expression: %w", err)
187+
}
188+
argValue = re
189+
}
180190
case syntax.TNumber:
181191
argValue = cond.Arg.Number()
182192
case syntax.TTime, syntax.TDate:
@@ -193,6 +203,43 @@ func compileCondition(cond syntax.Condition) (condition, error) {
193203
return out, nil
194204
}
195205

206+
// maxRegexpSize is the maximum total estimated program size of the MATCHES
207+
// patterns in a query.
208+
const maxRegexpSize = 500
209+
210+
// compileRegexp compiles pattern and deducts its estimated size from budget.
211+
func compileRegexp(pattern string, budget *int) (*regexp.Regexp, error) {
212+
re, err := resyntax.Parse(pattern, resyntax.Perl)
213+
if err != nil {
214+
return nil, err
215+
}
216+
size := regexpSize(re)
217+
if size > *budget {
218+
return nil, errors.New("patterns are too large")
219+
}
220+
*budget -= size
221+
return regexp.Compile(pattern)
222+
}
223+
224+
// regexpSize estimates the size of the program re compiles to, counting its
225+
// instructions and character class ranges.
226+
func regexpSize(re *resyntax.Regexp) int {
227+
var size int
228+
switch re.Op {
229+
case resyntax.OpLiteral:
230+
size = len(re.Rune)
231+
case resyntax.OpCharClass:
232+
size = len(re.Rune) / 2
233+
}
234+
for _, sub := range re.Sub {
235+
size += regexpSize(sub) + 1
236+
}
237+
if re.Op == resyntax.OpRepeat {
238+
size *= max(re.Min, re.Max, 1)
239+
}
240+
return max(size, 1)
241+
}
242+
196243
// TODO(creachadair): The existing implementation allows anything number shaped
197244
// to be treated as a number. This preserves the parts of that behavior we had
198245
// tests for, but we should probably get rid of that.
@@ -218,10 +265,7 @@ var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string)
218265
},
219266
syntax.TMatches: {
220267
syntax.TString: func(v interface{}) func(string) bool {
221-
return func(s string) bool {
222-
match, _ := regexp.MatchString(v.(string), s)
223-
return match
224-
}
268+
return v.(*regexp.Regexp).MatchString
225269
},
226270
},
227271
syntax.TEq: {

‎sei-tendermint/internal/pubsub/query/query_test.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,20 @@ func TestCompiledMatches(t *testing.T) {
222222
}
223223
}
224224

225+
func TestMatchesRejectsInvalidRegexp(t *testing.T) {
226+
for _, s := range []string{
227+
`tm.event MATCHES '('`,
228+
`tm.event MATCHES '[a-z]{1,1000}'`,
229+
`tm.event MATCHES '[a-z]{1,150}' AND tm.event MATCHES '[a-z]{1,150}'`,
230+
`tm.event MATCHES '(?:[a-z]{1,1000}){0,}'`,
231+
`tm.event MATCHES '\pL\pN\pS'`,
232+
} {
233+
if q, err := query.New(s); err == nil {
234+
t.Errorf("New(%q): got %v, want error", s, q)
235+
}
236+
}
237+
}
238+
225239
func TestAllMatchesAll(t *testing.T) {
226240
events := newTestEvents(
227241
``,

‎sei-tendermint/internal/rpc/core/blocks.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package core
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
67
"sort"
78

@@ -306,6 +307,8 @@ const DescendingOrder = "desc"
306307
func (env *Environment) BlockSearch(ctx context.Context, req *coretypes.RequestBlockSearch) (*coretypes.ResultBlockSearch, error) {
307308
if !indexer.KVSinkEnabled(env.EventSinks) {
308309
return nil, fmt.Errorf("block searching is disabled due to no kvEventSink")
310+
} else if len(req.Query) > maxQueryLength {
311+
return nil, errors.New("maximum query length exceeded")
309312
}
310313

311314
q, err := tmquery.New(req.Query)

‎sei-tendermint/internal/rpc/core/events.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ func (env *Environment) Subscribe(ctx context.Context, req *coretypes.RequestSub
103103
// Unsubscribe from events via WebSocket.
104104
// More: https://docs.tendermint.com/master/rpc/#/Websocket/unsubscribe
105105
func (env *Environment) Unsubscribe(ctx context.Context, req *coretypes.RequestUnsubscribe) (*coretypes.ResultUnsubscribe, error) {
106+
if len(req.Query) > maxQueryLength {
107+
return nil, errors.New("maximum query length exceeded")
108+
}
106109
args := tmpubsub.UnsubscribeArgs{Subscriber: rpctypes.GetCallInfo(ctx).RemoteAddr()}
107110
logger.Info("Unsubscribe from query", "remote", args.Subscriber, "subscription", req.Query)
108111

@@ -173,7 +176,9 @@ func (env *Environment) Events(ctx context.Context, req *coretypes.RequestEvents
173176
}
174177

175178
query := tmquery.All
176-
if req.Filter != nil && req.Filter.Query != "" {
179+
if req.Filter != nil && len(req.Filter.Query) > maxQueryLength {
180+
return nil, errors.New("maximum query length exceeded")
181+
} else if req.Filter != nil && req.Filter.Query != "" {
177182
q, err := tmquery.New(req.Filter.Query)
178183
if err != nil {
179184
return nil, fmt.Errorf("invalid filter query: %w", err)
@@ -192,6 +197,9 @@ func (env *Environment) Events(ctx context.Context, req *coretypes.RequestEvents
192197
var info eventlog.Info
193198
var items []*eventlog.Item
194199
accept := func(itm *eventlog.Item) error {
200+
if err := ctx.Err(); err != nil {
201+
return err
202+
}
195203
// N.B. We accept up to one item more than requested, so we can tell how
196204
// to set the "more" flag in the response.
197205
if len(items) > maxItems || itm.Cursor.Before(after) {
@@ -204,17 +212,17 @@ func (env *Environment) Events(ctx context.Context, req *coretypes.RequestEvents
204212
}
205213

206214
if before.IsZero() {
207-
ctx, cancel := context.WithTimeout(ctx, waitTime)
215+
waitCtx, cancel := context.WithTimeout(ctx, waitTime)
208216
defer cancel()
209217

210218
// Long poll. The loop here is because new items may not match the query,
211219
// and we want to keep waiting until we have relevant results (or time out).
212220
cur := after
213221
for len(items) == 0 {
214-
info, err = eventLog.WaitScan(ctx, cur, accept)
222+
info, err = eventLog.WaitScan(waitCtx, cur, accept)
215223
if err != nil {
216224
// Don't report a timeout as a request failure.
217-
if errors.Is(err, context.DeadlineExceeded) {
225+
if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil {
218226
err = nil
219227
}
220228
break
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package core
2+
3+
import (
4+
"context"
5+
"strings"
6+
"testing"
7+
"time"
8+
9+
"github.com/sei-protocol/sei-chain/sei-tendermint/internal/eventlog"
10+
"github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils"
11+
"github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require"
12+
"github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes"
13+
"github.com/sei-protocol/sei-chain/sei-tendermint/types"
14+
)
15+
16+
func eventLogEnv(t *testing.T, numItems int) (*Environment, *eventlog.Log) {
17+
t.Helper()
18+
lg, err := eventlog.New(eventlog.LogSettings{WindowSize: time.Minute})
19+
require.NoError(t, err)
20+
for range numItems {
21+
require.NoError(t, lg.Add(types.EventNewBlockValue, types.EventDataString("x")))
22+
}
23+
return &Environment{EventLog: utils.Some(lg)}, lg
24+
}
25+
26+
func TestEventsRejectsOversizedQuery(t *testing.T) {
27+
env, _ := eventLogEnv(t, 0)
28+
query := "tm.event MATCHES '" + strings.Repeat("a", maxQueryLength) + "'"
29+
30+
_, err := env.Events(t.Context(), &coretypes.RequestEvents{
31+
Filter: &coretypes.EventFilter{Query: query},
32+
})
33+
require.Error(t, err)
34+
}
35+
36+
func TestEventsScanStopsWhenContextEnds(t *testing.T) {
37+
env, lg := eventLogEnv(t, 3)
38+
req := &coretypes.RequestEvents{Before: lg.Info().Newest.String()}
39+
40+
got, err := env.Events(t.Context(), req)
41+
require.NoError(t, err)
42+
require.Len(t, got.Items, 2)
43+
44+
ctx, cancel := context.WithCancel(t.Context())
45+
cancel()
46+
_, err = env.Events(ctx, req)
47+
require.ErrorIs(t, err, context.Canceled)
48+
49+
ctx, cancel = context.WithDeadline(t.Context(), time.Unix(0, 0))
50+
defer cancel()
51+
_, err = env.Events(ctx, &coretypes.RequestEvents{})
52+
require.ErrorIs(t, err, context.DeadlineExceeded)
53+
}

0 commit comments

Comments
 (0)