Skip to content

Commit c07cde4

Browse files
authored
Load consensus round state before starting peer routines (#4542)
The consensus reactor caches the round state in `NewReactor`, before the consensus state has been loaded from the store, so the cached state has height 0 and a nil vote set. It is only refreshed on the first tick of `updateRoundStateRoutine`. On a node that switches to consensus straight away (a frozen-height node, or one started with `waitSync=false`), `processPeerUpdates` can replay already-connected peers before that tick. A new peer also reports height 0, so `queryMaj23Routine` sees matching heights and calls `Prevotes` on the nil vote set, which panics and kills the node at boot. Reproduced locally on main in 9 of 150 runs with `GOMAXPROCS=1`. The reactor now stores the loaded round state once readiness is signalled, just before it starts `processPeerUpdates`, so no peer routine can see the constructor's empty state. The refresh interval becomes a reactor field so the new test can stop the periodic refresh from hiding the race. The test covers both a reactor that is ready from boot and one that switches to consensus after starting; without the fix both cases panic with the same stack as CI on every run, and with it both pass. Flaked in: https://github.com/sei-protocol/sei-chain/actions/runs/38278017470/job/114909783619
1 parent b625eff commit c07cde4

3 files changed

Lines changed: 65 additions & 7 deletions

File tree

‎sei-tendermint/internal/consensus/common_test.go‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,24 @@ func newStateWithConfigAndBlockStore(
492492
pv types.PrivValidator,
493493
app *proxy.Proxy,
494494
blockStore *store.BlockStore,
495+
) *testState {
496+
t.Helper()
497+
stateHandle := newUnloadedState(t, thisConfig, state, app, blockStore)
498+
if err := stateHandle.updateStateFromStore(); err != nil {
499+
panic(err)
500+
}
501+
stateHandle.SetPrivValidator(t.Context(), utils.Some(pv))
502+
return stateHandle
503+
}
504+
505+
// newUnloadedState returns a consensus state whose store holds state but which has
506+
// not loaded it yet, as node startup constructs it.
507+
func newUnloadedState(
508+
t *testing.T,
509+
thisConfig *config.Config,
510+
state sm.State,
511+
app *proxy.Proxy,
512+
blockStore *store.BlockStore,
495513
) *testState {
496514
t.Helper()
497515
ctx := t.Context()
@@ -534,11 +552,6 @@ func newStateWithConfigAndBlockStore(
534552
eventBus,
535553
[]trace.TracerProviderOption{},
536554
)}
537-
if err := stateHandle.updateStateFromStore(); err != nil {
538-
panic(err)
539-
}
540-
541-
stateHandle.SetPrivValidator(ctx, utils.Some(pv))
542555
t.Cleanup(func() {
543556
stateHandle.waitForTestRoutines()
544557
eventBus.Wait()

‎sei-tendermint/internal/consensus/reactor.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,8 @@ type Reactor struct {
102102
peers utils.RWMutex[map[types.NodeID]*PeerState]
103103
roundState atomic.Pointer[cstypes.RoundState]
104104
readySignal utils.AtomicSend[bool]
105+
106+
roundStateRefreshInterval time.Duration
105107
}
106108

107109
// NewReactor returns a reference to a new consensus reactor, which implements
@@ -143,7 +145,8 @@ func NewReactor(
143145
vote: voteCh,
144146
votSet: voteSetCh,
145147
},
146-
cfg: cfg,
148+
cfg: cfg,
149+
roundStateRefreshInterval: 100 * time.Microsecond,
147150
}
148151
r.roundState.Store(cs.GetRoundState())
149152
cs.eventNewRoundStep = r.broadcastNewRoundStepMessage
@@ -182,6 +185,8 @@ func (r *Reactor) OnStart(ctx context.Context) error {
182185
if _, err := r.readySignal.Wait(ctx, func(ready bool) bool { return ready }); err != nil {
183186
return err
184187
}
188+
// Peer routines read the cache straight away, so it must hold the loaded state.
189+
r.roundState.Store(r.state.GetRoundState())
185190
r.SpawnCritical("processPeerUpdates", r.processPeerUpdates)
186191
r.SpawnCritical("broadcastValidBlock", r.broadcastValidBlockRoutine)
187192
return r.state.Run(ctx)
@@ -295,7 +300,7 @@ func (r *Reactor) sendNewRoundStepMessage(peerID types.NodeID) {
295300
}
296301

297302
func (r *Reactor) updateRoundStateRoutine(ctx context.Context) error {
298-
t := time.NewTicker(100 * time.Microsecond)
303+
t := time.NewTicker(r.roundStateRefreshInterval)
299304
for {
300305
if _, err := utils.Recv(ctx, t.C); err != nil {
301306
return err

‎sei-tendermint/internal/consensus/reactor_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,46 @@ func TestReactorBasic(t *testing.T) {
233233
}
234234
}
235235

236+
// TestReactorPeersStartWithLoadedRoundState checks that peer routines see the round
237+
// state loaded at startup rather than the empty one cached at construction, both when
238+
// the reactor is ready from boot and when it switches to consensus later.
239+
func TestReactorPeersStartWithLoadedRoundState(t *testing.T) {
240+
for _, waitSync := range []bool{false, true} {
241+
t.Run(fmt.Sprintf("waitSync=%v", waitSync), func(t *testing.T) {
242+
ctx := t.Context()
243+
cfg := configSetup(t)
244+
state, privVals := makeGenesisState(ctx, t, cfg, genesisStateArgs{Validators: 1, Params: factory.ConsensusParams()})
245+
thisConfig, err := config.ResetTestRoot(t.TempDir(), "consensus_reactor_test")
246+
require.NoError(t, err)
247+
cs := newUnloadedState(t, thisConfig, state, kvstore.NewProxy(), store.NewBlockStore(dbm.NewMemDB()))
248+
cs.SetPrivValidator(ctx, utils.Some(privVals[0]))
249+
250+
// The other node runs no consensus reactor, so its peer state stays at height 0.
251+
network := p2p.MakeTestNetwork(t, p2p.TestNetworkOptions{NumNodes: 2})
252+
reactor, err := NewReactor(cs.State, network.Nodes()[0].Router, cs.eventBus, waitSync, thisConfig)
253+
require.NoError(t, err)
254+
reactor.roundStateRefreshInterval = time.Hour
255+
blocksSub, err := cs.eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{
256+
ClientID: testSubscriber,
257+
Query: types.EventQueryNewBlock,
258+
Limit: 100,
259+
})
260+
require.NoError(t, err)
261+
network.Start(t)
262+
263+
require.NoError(t, reactor.Start(ctx))
264+
t.Cleanup(reactor.Wait)
265+
if waitSync {
266+
reactor.SwitchToConsensus(cs.GetState(), false)
267+
}
268+
for range 3 {
269+
_, err := blocksSub.Next(ctx)
270+
require.NoError(t, err)
271+
}
272+
})
273+
}
274+
}
275+
236276
func TestReactorWithEvidence(t *testing.T) {
237277
ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
238278
defer cancel()

0 commit comments

Comments
 (0)