Skip to content

Commit d4853e5

Browse files
authored
Initialize app from last header when a frozen node restarts (#4447)
A node in freeze mode halts one block below its freeze height and never commits again. When its process restarts, no `Commit` runs, so `BaseApp.Initialized()` never fires and the EVM HTTP and WS servers, which wait on it in `app/app.go`, never start. The node keeps its data and its Tendermint RPC, but EVM RPC stays down from the first restart on. This is PLT-1370. In `nodeImpl.OnStart`, right after the handshake, a frozen node with a non-empty chain now loads the block meta at `state.LastBlockHeight` and passes its header to `Application.InitLastHeader`. `BaseApp` sets checkState from that header and signals initialization, which is what `Commit` does after a block. Sending only the start signal was rejected because checkState would keep the empty header from `LoadLatestVersion`, and the RPC "latest" context reads it. Nodes not in freeze mode are unchanged. A frozen node that is still catching up now opens EVM RPC at startup instead of after its first block; later commits reset checkState as before. `TestFreezeModeRestartInitializesAppFromLastHeader` covers the node wiring. `TestFrozenNodeServesEVMRPCAfterRestart` restarts seid on the height-10 frozen node in the frozen RPC router cluster. Locally it timed out on main and passed with this change. Part of PLT-1370.
1 parent 137bede commit d4853e5

3 files changed

Lines changed: 128 additions & 0 deletions

File tree

‎integration_test/frozen_rpc_router/router_test.go‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"io"
1212
"net/http"
1313
"os"
14+
"os/exec"
1415
"strconv"
1516
"strings"
1617
"testing"
@@ -22,6 +23,7 @@ const (
2223
defaultFrozen10URL = "http://127.0.0.1:8547"
2324
defaultFrozen20URL = "http://127.0.0.1:8549"
2425
defaultLiveURL = "http://127.0.0.1:8545"
26+
defaultFrozen10Box = "sei-node-1"
2527
routeHeader = "Sei-RPC-Route"
2628
)
2729

@@ -94,6 +96,59 @@ func TestFrozenRPCRouterReachesEveryNode(t *testing.T) {
9496
})
9597
}
9698

99+
// A restarted frozen node commits no further blocks, so its EVM RPC has to
100+
// come up from the stored chain state alone.
101+
func TestFrozenNodeServesEVMRPCAfterRestart(t *testing.T) {
102+
client := &http.Client{Timeout: 10 * time.Second}
103+
frozen10URL := envOrDefault("FROZEN_RPC_NODE_10_URL", defaultFrozen10URL)
104+
container := envOrDefault("FROZEN_RPC_NODE_10_CONTAINER", defaultFrozen10Box)
105+
106+
waitForHead(t, client, frozen10URL, func(height uint64) bool { return height == 9 }, "frozen node at height 10")
107+
restartSeid(t, container)
108+
waitForHead(t, client, frozen10URL, func(height uint64) bool { return height == 9 }, "restarted frozen node at height 10")
109+
110+
response, _, err := callRPC(t.Context(), client, frozen10URL, "eth_getBlockByNumber", []any{"0x5", false})
111+
if err != nil {
112+
t.Fatal(err)
113+
}
114+
if response.Error != nil {
115+
t.Fatalf("RPC returned error %d: %s", response.Error.Code, response.Error.Message)
116+
}
117+
var block struct {
118+
Number string `json:"number"`
119+
}
120+
if err := json.Unmarshal(response.Result, &block); err != nil {
121+
t.Fatalf("decode block response: %v", err)
122+
}
123+
if block.Number != "0x5" {
124+
t.Fatalf("block number = %q, want %q", block.Number, "0x5")
125+
}
126+
}
127+
128+
// restartSeid stops seid in the container and starts it again with the
129+
// container's original start script. The container itself keeps running.
130+
func restartSeid(t *testing.T, container string) {
131+
t.Helper()
132+
const stop = `
133+
seid_pids() {
134+
for p in /proc/[0-9]*; do
135+
[ "$(cat "$p/comm" 2>/dev/null)" = seid ] && echo "${p#/proc/}"
136+
done
137+
}
138+
pids=$(seid_pids)
139+
[ -n "$pids" ] || { echo "seid is not running" >&2; exit 1; }
140+
kill $pids
141+
while [ -n "$(seid_pids)" ]; do sleep 1; done
142+
cp "build/generated/logs/seid-$ID.log" "build/generated/logs/seid-$ID-before-restart.log"
143+
`
144+
if output, err := exec.CommandContext(t.Context(), "docker", "exec", container, "sh", "-c", stop).CombinedOutput(); err != nil {
145+
t.Fatalf("stop seid in %s: %v: %s", container, err, output)
146+
}
147+
if output, err := exec.CommandContext(t.Context(), "docker", "exec", "-d", container, "/usr/bin/start_sei.sh").CombinedOutput(); err != nil {
148+
t.Fatalf("start seid in %s: %v: %s", container, err, output)
149+
}
150+
}
151+
97152
func waitForHead(t *testing.T, client *http.Client, endpoint string, ready func(uint64) bool, description string) {
98153
t.Helper()
99154
ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute)

‎sei-tendermint/node/freeze_test.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,14 @@ import (
66
"slices"
77
"testing"
88

9+
"github.com/sei-protocol/sei-chain/sei-tendermint/abci/example/kvstore"
910
"github.com/sei-protocol/sei-chain/sei-tendermint/config"
1011
mempoolreactor "github.com/sei-protocol/sei-chain/sei-tendermint/internal/mempool/reactor"
12+
"github.com/sei-protocol/sei-chain/sei-tendermint/internal/pubsub"
1113
rpccore "github.com/sei-protocol/sei-chain/sei-tendermint/internal/rpc/core"
14+
"github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils"
15+
"github.com/sei-protocol/sei-chain/sei-tendermint/libs/utils/require"
16+
tmproto "github.com/sei-protocol/sei-chain/sei-tendermint/proto/tendermint/types"
1217
"github.com/sei-protocol/sei-chain/sei-tendermint/rpc/coretypes"
1318
"github.com/sei-protocol/sei-chain/sei-tendermint/types"
1419
)
@@ -142,3 +147,61 @@ func TestFreezeModeDisablesMempoolTraffic(t *testing.T) {
142147
t.Fatal("mempool channel is advertised in freeze mode")
143148
}
144149
}
150+
151+
type lastHeaderRecordingApp struct {
152+
*kvstore.Application
153+
lastHeaders []*tmproto.Header
154+
}
155+
156+
func (app *lastHeaderRecordingApp) InitLastHeader(lastHeader *tmproto.Header) {
157+
app.lastHeaders = append(app.lastHeaders, lastHeader)
158+
}
159+
160+
// A frozen node commits no further blocks after a restart, so the node must
161+
// initialize the app from the last stored header instead.
162+
func TestFreezeModeRestartInitializesAppFromLastHeader(t *testing.T) {
163+
cfg, err := config.ResetTestRoot(t.TempDir(), "freeze_restart_test")
164+
require.NoError(t, err)
165+
cfg.RPC.ListenAddress = ""
166+
// The frozen node must reopen the validator's stores.
167+
cfg.DBBackend = "goleveldb"
168+
169+
validator, err := newLocalNodeService(t.Context(), cfg)
170+
require.NoError(t, err)
171+
blocks, err := validator.(*nodeImpl).EventBus().SubscribeWithArgs(t.Context(), pubsub.SubscribeArgs{
172+
ClientID: "freeze_restart_test",
173+
Query: types.EventQueryNewBlock,
174+
Limit: 10,
175+
})
176+
require.NoError(t, err)
177+
require.NoError(t, validator.Start(t.Context()))
178+
for range 2 {
179+
_, err := blocks.Next(t.Context())
180+
require.NoError(t, err)
181+
}
182+
validator.Stop()
183+
validator.Wait()
184+
185+
cfg.Mode = config.ModeFull
186+
app := &lastHeaderRecordingApp{Application: kvstore.NewApplication()}
187+
app.SetValidators(utils.OrPanic1(types.GenesisDocFromFile(cfg.GenesisFile())).ValidatorUpdates())
188+
frozenService, err := New(t.Context(), cfg, func() {}, app, nil, nil, types.DefaultConsensusPolicy(), WithFreezeHeight(1000))
189+
require.NoError(t, err)
190+
frozen := frozenService.(*nodeImpl)
191+
lastHeight := frozen.blockStore.Height()
192+
require.GreaterOrEqual(t, lastHeight, int64(2))
193+
want := frozen.blockStore.LoadBlockMeta(lastHeight).Header
194+
195+
require.NoError(t, frozen.Start(t.Context()))
196+
t.Cleanup(func() {
197+
frozen.Stop()
198+
frozen.Wait()
199+
})
200+
201+
require.Equal(t, 1, len(app.lastHeaders))
202+
got, err := types.HeaderFromProto(app.lastHeaders[0])
203+
require.NoError(t, err)
204+
require.Equal(t, want.Hash(), got.Hash())
205+
require.Equal(t, lastHeight, got.Height)
206+
require.Equal(t, frozen.genesisDoc.ChainID, got.ChainID)
207+
}

‎sei-tendermint/node/node.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -575,6 +575,16 @@ func (n *nodeImpl) OnStart(ctx context.Context) (err error) {
575575
return fmt.Errorf("cannot load state: %w", err)
576576
}
577577

578+
// A frozen node never commits another block, so after a restart nothing
579+
// would initialize the app (e.g. its query context and EVM RPC servers).
580+
if n.freezeHeight > 0 && state.LastBlockHeight > 0 {
581+
meta := n.blockStore.LoadBlockMeta(state.LastBlockHeight)
582+
if meta == nil {
583+
return fmt.Errorf("freeze mode: block %d is missing from the block store", state.LastBlockHeight)
584+
}
585+
n.rpcEnv.App.InitLastHeader(meta.Header.ToProto())
586+
}
587+
578588
logNodeStartupInfo(state, n.rpcEnv.PubKey, n.config.Mode)
579589

580590
// TODO: Fetch and provide real options and do proper p2p bootstrapping.

0 commit comments

Comments
 (0)