Skip to content

Commit 7b415b7

Browse files
fix(rootmulti): return errors instead of panics for malformed snapshot streams (#4368)
## Summary Snapshot restore now returns an error, not a panic, for three kinds of malformed stream, and for a failed state-store import. `Restore` runs on the snapshot manager's goroutine, so a panic there stopped the node. With an error, the restore fails and state sync stops cleanly with the cause in the log. A failed restore also publishes nothing: no memIAVL snapshot, no flatkv import, and no state-store versions. Retrying with another snapshot needs a separate change. - `sei-cosmos/storev2/rootmulti/store.go`: `restore` rejects a node that is not inside a named store section. Before, such a node reached memIAVL's `MultiTreeImporter` with no tree importer open. The state-store import now runs through `stateStoreImport`. Its error fails the restore, not the process, and `restore` waits for the import to return before it returns. `finishSCImport` publishes the SC import only when the restore succeeded, and aborts it otherwise. The state store's earliest and latest versions are set only on success. - `sei-db/state_db/sc/types/types.go`: `Importer` gains `Abort`, which discards an import without publishing it. - `sei-db/state_db/sc/memiavl/import.go`: the tree importer rejects a branch node that does not have two pending children. `TreeImporter.Add` stops queueing nodes after the import has stopped, and `Close` returns the import's error. `MultiTreeImporter.Abort` removes the temp directory without publishing a snapshot or moving `current`, and a failed `Close` removes it too. - `sei-db/state_db/sc/composite/importer.go`: when the cosmos import fails, `Close` aborts the flatkv import rather than publishing it. `Abort` aborts both. Honest snapshots do not change, so there is no consensus or state impact. Checks on which stores a snapshot contains are not in scope. ## Test plan - `sei-cosmos/storev2/rootmulti/restore_test.go`: under `memiavl_only` and `test_dual_write`, these streams fail the restore with an error and do not panic: a node before any store item, a node after an unnamed store item, and a branch node before its leaves. A state-store import that fails early, or after it reads the full stream, also fails the restore. A failed restore leaves memIAVL's snapshots and `current` as they were, and leaves the state-store versions unset. A successful restore publishes the snapshot and sets the versions. - `sei-db/state_db/sc/composite/importer_test.go`: `Close` publishes flatkv only after the cosmos import succeeds, and `Abort` aborts both. - `sei-db/state_db/sc/memiavl/import_test.go`: a branch node before its leaves fails the import, and `Add` returns after the import has stopped, also when the node channel is full. --------- Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent e2c333d commit 7b415b7

8 files changed

Lines changed: 452 additions & 32 deletions

File tree

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
package rootmulti
2+
3+
import (
4+
"bytes"
5+
"errors"
6+
"fmt"
7+
"os"
8+
"path/filepath"
9+
"testing"
10+
"time"
11+
12+
protoio "github.com/gogo/protobuf/io"
13+
snapshottypes "github.com/sei-protocol/sei-chain/sei-cosmos/snapshots/types"
14+
"github.com/sei-protocol/sei-chain/sei-db/common/utils"
15+
seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config"
16+
seidbtypes "github.com/sei-protocol/sei-chain/sei-db/db_engine/types"
17+
"github.com/stretchr/testify/require"
18+
)
19+
20+
func storeItem(name string) snapshottypes.SnapshotItem {
21+
return snapshottypes.SnapshotItem{Item: &snapshottypes.SnapshotItem_Store{
22+
Store: &snapshottypes.SnapshotStoreItem{Name: name},
23+
}}
24+
}
25+
26+
func nodeItem(height int32, key string) snapshottypes.SnapshotItem {
27+
item := &snapshottypes.SnapshotIAVLItem{Key: []byte(key), Version: 1, Height: height}
28+
if height == 0 {
29+
item.Value = []byte("v")
30+
}
31+
return snapshottypes.SnapshotItem{Item: &snapshottypes.SnapshotItem_IAVL{IAVL: item}}
32+
}
33+
34+
func snapshotStream(t *testing.T, items []snapshottypes.SnapshotItem) protoio.Reader {
35+
t.Helper()
36+
var buf bytes.Buffer
37+
writer := protoio.NewDelimitedWriter(&buf)
38+
for i := range items {
39+
require.NoError(t, writer.WriteMsg(&items[i]))
40+
}
41+
return protoio.NewDelimitedReader(bytes.NewReader(buf.Bytes()), 1<<30)
42+
}
43+
44+
// restoreWithin runs Restore and fails the test if it panics or does not return in time.
45+
func restoreWithin(t *testing.T, store *Store, reader protoio.Reader) error {
46+
t.Helper()
47+
result := make(chan error, 1)
48+
go func() {
49+
defer func() {
50+
if r := recover(); r != nil {
51+
result <- fmt.Errorf("restore panicked: %v", r)
52+
}
53+
}()
54+
_, err := store.Restore(1, snapshottypes.CurrentFormat, reader)
55+
result <- err
56+
}()
57+
select {
58+
case err := <-result:
59+
require.NotContains(t, fmt.Sprint(err), "restore panicked")
60+
return err
61+
case <-time.After(60 * time.Second):
62+
t.Fatal("restore did not return")
63+
return nil
64+
}
65+
}
66+
67+
// scSnapshotState lists memIAVL's snapshot directories under home and the one current points to.
68+
type scSnapshotState struct {
69+
snapshots []string
70+
current string
71+
}
72+
73+
func readSCSnapshotState(t *testing.T, home string) scSnapshotState {
74+
t.Helper()
75+
dir := utils.GetCosmosSCStorePath(home)
76+
snapshots, err := filepath.Glob(filepath.Join(dir, "snapshot-*"))
77+
require.NoError(t, err)
78+
current, _ := os.Readlink(filepath.Join(dir, "current"))
79+
return scSnapshotState{snapshots: snapshots, current: current}
80+
}
81+
82+
// TestRestoreRejectsMalformedStream feeds Restore streams that break the snapshot format. Each must fail
83+
// with an error and leave memIAVL's published snapshots as they were.
84+
func TestRestoreRejectsMalformedStream(t *testing.T) {
85+
configs := map[string]seidbconfig.StateCommitConfig{
86+
"memiavl_only": memiavlOnlyConfig(),
87+
"test_dual_write": dualWriteConfig(),
88+
}
89+
streams := map[string]struct {
90+
items []snapshottypes.SnapshotItem
91+
wantErr string
92+
}{
93+
"node before any store item": {
94+
items: []snapshottypes.SnapshotItem{nodeItem(0, "k")},
95+
wantErr: "outside a named store section",
96+
},
97+
"node after an unnamed store item": {
98+
items: []snapshottypes.SnapshotItem{storeItem(""), nodeItem(0, "k")},
99+
wantErr: "outside a named store section",
100+
},
101+
"branch node before its leaves": {
102+
items: []snapshottypes.SnapshotItem{storeItem("bank"), nodeItem(1, "k")},
103+
wantErr: "pending children",
104+
},
105+
}
106+
107+
for cfgName, cfg := range configs {
108+
for streamName, stream := range streams {
109+
t.Run(cfgName+"/"+streamName, func(t *testing.T) {
110+
home := t.TempDir()
111+
store, _ := newTestRootMulti(t, home, cfg)
112+
before := readSCSnapshotState(t, home)
113+
err := restoreWithin(t, store, snapshotStream(t, stream.items))
114+
require.ErrorContains(t, err, stream.wantErr)
115+
require.Equal(t, before, readSCSnapshotState(t, home), "a failed restore must not publish a snapshot")
116+
})
117+
}
118+
}
119+
}
120+
121+
// fakeStateStore is a state store whose Import returns importErr, either at once or after reading every
122+
// node, and which counts the version writes restore makes.
123+
type fakeStateStore struct {
124+
seidbtypes.StateStore
125+
returnEarly bool
126+
importErr error
127+
versionWrites int
128+
}
129+
130+
func (f *fakeStateStore) Import(_ int64, ch <-chan seidbtypes.SnapshotNode) error {
131+
if !f.returnEarly {
132+
for range ch {
133+
}
134+
}
135+
return f.importErr
136+
}
137+
138+
func (f *fakeStateStore) SetEarliestVersion(int64, bool) error {
139+
f.versionWrites++
140+
return nil
141+
}
142+
143+
func (f *fakeStateStore) SetLatestVersion(int64) error {
144+
f.versionWrites++
145+
return nil
146+
}
147+
148+
func leafStream(n int) []snapshottypes.SnapshotItem {
149+
items := []snapshottypes.SnapshotItem{storeItem("bank")}
150+
for i := 0; i < n; i++ {
151+
items = append(items, nodeItem(0, fmt.Sprintf("k%05d", i)))
152+
}
153+
return items
154+
}
155+
156+
// TestRestoreFailurePublishesNothing pins that a failed restore returns an error, publishes no memIAVL
157+
// snapshot, and does not record the snapshot height on the state store, whether the state store or the
158+
// SC stream failed.
159+
func TestRestoreFailurePublishesNothing(t *testing.T) {
160+
importErr := errors.New("import failed")
161+
cases := map[string]struct {
162+
ss *fakeStateStore
163+
items []snapshottypes.SnapshotItem
164+
wantErr string
165+
}{
166+
// More leaves than the import buffer holds, so sending blocks once Import has stopped reading.
167+
"state store import returns early": {
168+
ss: &fakeStateStore{returnEarly: true, importErr: importErr},
169+
items: leafStream(20000),
170+
wantErr: "state store import",
171+
},
172+
// A single leaf is a complete tree, so only the state store fails.
173+
"state store import fails after draining": {
174+
ss: &fakeStateStore{importErr: importErr},
175+
items: leafStream(1),
176+
wantErr: "state store import",
177+
},
178+
"SC stream fails": {
179+
ss: &fakeStateStore{},
180+
items: []snapshottypes.SnapshotItem{storeItem("bank"), nodeItem(1, "k")},
181+
wantErr: "pending children",
182+
},
183+
}
184+
185+
for name, tc := range cases {
186+
t.Run(name, func(t *testing.T) {
187+
home := t.TempDir()
188+
store, _ := newTestRootMulti(t, home, memiavlOnlyConfig())
189+
store.ssStore = tc.ss
190+
before := readSCSnapshotState(t, home)
191+
err := restoreWithin(t, store, snapshotStream(t, tc.items))
192+
require.ErrorContains(t, err, tc.wantErr)
193+
require.Equal(t, before, readSCSnapshotState(t, home), "a failed restore must not publish a snapshot")
194+
require.Zero(t, tc.ss.versionWrites, "a failed restore must not record the snapshot height")
195+
})
196+
}
197+
}
198+
199+
// TestRestoreSuccessPublishes pins that a successful restore publishes the memIAVL snapshot and records
200+
// the snapshot height on the state store.
201+
func TestRestoreSuccessPublishes(t *testing.T) {
202+
home := t.TempDir()
203+
store, _ := newTestRootMulti(t, home, memiavlOnlyConfig())
204+
ss := &fakeStateStore{}
205+
store.ssStore = ss
206+
require.NoError(t, store.scStore.Close())
207+
208+
_, err := store.restore(1, snapshotStream(t, leafStream(1)))
209+
require.NoError(t, err)
210+
require.Equal(t, "snapshot-00000000000000000001", readSCSnapshotState(t, home).current)
211+
require.Equal(t, 2, ss.versionWrites)
212+
}

‎sei-cosmos/storev2/rootmulti/store.go‎

Lines changed: 78 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1224,7 +1224,7 @@ func (rs *Store) Restore(
12241224

12251225
func (rs *Store) restore(height int64, protoReader protoio.Reader) (snapshottypes.SnapshotItem, error) {
12261226
var (
1227-
ssImporter chan seidbtypes.SnapshotNode
1227+
ssImport *stateStoreImport
12281228
snapshotItem snapshottypes.SnapshotItem
12291229
storeKey string
12301230
restoreErr error
@@ -1234,13 +1234,7 @@ func (rs *Store) restore(height int64, protoReader protoio.Reader) (snapshottype
12341234
return snapshottypes.SnapshotItem{}, err
12351235
}
12361236
if rs.ssStore != nil {
1237-
ssImporter = make(chan seidbtypes.SnapshotNode, 10000)
1238-
go func() {
1239-
err := rs.ssStore.Import(height, ssImporter)
1240-
if err != nil {
1241-
panic(err)
1242-
}
1243-
}()
1237+
ssImport = startStateStoreImport(rs.ssStore, height)
12441238
}
12451239
loop:
12461240
for {
@@ -1262,6 +1256,12 @@ loop:
12621256
}
12631257
logger.Info("Start restoring store", "key", storeKey)
12641258
case *snapshottypes.SnapshotItem_IAVL:
1259+
// Importers route each node to the store opened by the last store item, so a node with no
1260+
// named store before it has nowhere to go.
1261+
if storeKey == "" {
1262+
restoreErr = errors.Wrap(sdkerrors.ErrLogic, "snapshot node appears outside a named store section")
1263+
break loop
1264+
}
12651265
if item.IAVL.Height > math.MaxInt8 {
12661266
restoreErr = errors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v",
12671267
item.IAVL.Height, math.MaxInt8)
@@ -1284,11 +1284,14 @@ loop:
12841284
scImporter.AddNode(node)
12851285

12861286
// Check if we should also import to SS store
1287-
if rs.ssStore != nil && node.Height == 0 && ssImporter != nil {
1288-
ssImporter <- seidbtypes.SnapshotNode{
1287+
if ssImport != nil && node.Height == 0 {
1288+
if err = ssImport.send(seidbtypes.SnapshotNode{
12891289
StoreKey: storeKey,
12901290
Key: node.Key,
12911291
Value: node.Value,
1292+
}); err != nil {
1293+
restoreErr = err
1294+
break loop
12921295
}
12931296
}
12941297
default:
@@ -1297,18 +1300,17 @@ loop:
12971300
}
12981301
}
12991302

1300-
if err = scImporter.Close(); err != nil {
1301-
if restoreErr == nil {
1303+
if ssImport != nil {
1304+
if err = ssImport.finish(); err != nil && restoreErr == nil {
13021305
restoreErr = err
13031306
}
13041307
}
1305-
if ssImporter != nil {
1306-
close(ssImporter)
1307-
}
1308+
restoreErr = finishSCImport(scImporter, restoreErr)
13081309
// Initialize SS version metadata. Without SetLatestVersion, GetLatestVersion()
13091310
// stays 0 until the first post-sync block commits, which is misleading to any
1310-
// caller that reads it in that window.
1311-
if rs.ssStore != nil {
1311+
// caller that reads it in that window. A failed restore may have imported only
1312+
// part of the snapshot, so it must not claim the height.
1313+
if rs.ssStore != nil && restoreErr == nil {
13121314
if err := rs.ssStore.SetEarliestVersion(height, false); err != nil {
13131315
logger.Error("Failed to set earliest version during DB restore", "err", err)
13141316
}
@@ -1320,6 +1322,65 @@ loop:
13201322
return snapshotItem, restoreErr
13211323
}
13221324

1325+
// finishSCImport publishes the SC import when the restore has succeeded, and discards it otherwise, so a
1326+
// failed restore leaves no snapshot behind. It returns the restore's final error.
1327+
func finishSCImport(imp sctypes.Importer, restoreErr error) error {
1328+
if restoreErr != nil {
1329+
if err := imp.Abort(restoreErr); err != nil && !errors.IsOf(err, restoreErr) {
1330+
logger.Error("Failed to discard the SC import of a failed restore", "err", err)
1331+
}
1332+
return restoreErr
1333+
}
1334+
return imp.Close()
1335+
}
1336+
1337+
// stateStoreImport feeds restored leaves to a state store's Import running on its own goroutine.
1338+
type stateStoreImport struct {
1339+
nodes chan seidbtypes.SnapshotNode
1340+
// done is closed once Import has returned; err holds its result.
1341+
done chan struct{}
1342+
err error
1343+
}
1344+
1345+
func startStateStoreImport(ss seidbtypes.StateStore, height int64) *stateStoreImport {
1346+
imp := &stateStoreImport{
1347+
nodes: make(chan seidbtypes.SnapshotNode, 10000),
1348+
done: make(chan struct{}),
1349+
}
1350+
go func() {
1351+
defer close(imp.done)
1352+
imp.err = ss.Import(height, imp.nodes)
1353+
}()
1354+
return imp
1355+
}
1356+
1357+
// send queues node for import. It returns an error when Import has already returned.
1358+
func (imp *stateStoreImport) send(node seidbtypes.SnapshotNode) error {
1359+
select {
1360+
case imp.nodes <- node:
1361+
return nil
1362+
case <-imp.done:
1363+
return imp.stoppedErr()
1364+
}
1365+
}
1366+
1367+
// finish closes the input and waits for Import to return, returning its error.
1368+
func (imp *stateStoreImport) finish() error {
1369+
close(imp.nodes)
1370+
<-imp.done
1371+
if imp.err != nil {
1372+
return fmt.Errorf("state store import: %w", imp.err)
1373+
}
1374+
return nil
1375+
}
1376+
1377+
func (imp *stateStoreImport) stoppedErr() error {
1378+
if imp.err != nil {
1379+
return fmt.Errorf("state store import: %w", imp.err)
1380+
}
1381+
return fmt.Errorf("state store import returned before the snapshot stream ended")
1382+
}
1383+
13231384
// Snapshot Implements the interface from Snapshotter
13241385
func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
13251386
if height > math.MaxUint32 {

‎sei-db/state_db/sc/composite/importer.go‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,31 @@ func (si *SnapshotImporter) AddNode(node *types.SnapshotNode) {
7373
}
7474
}
7575

76+
// Close publishes both backends' imports. When the cosmos import fails, the flatkv import is discarded
77+
// rather than published, so a failed import does not leave flatkv ahead of memiavl.
7678
func (si *SnapshotImporter) Close() error {
79+
if si.cosmosImporter != nil {
80+
if err := si.cosmosImporter.Close(); err != nil {
81+
if si.flatkvImporter != nil {
82+
_ = si.flatkvImporter.Abort(err)
83+
}
84+
return err
85+
}
86+
}
87+
if si.flatkvImporter != nil {
88+
return si.flatkvImporter.Close()
89+
}
90+
return nil
91+
}
92+
93+
// Abort discards both backends' imports, publishing neither.
94+
func (si *SnapshotImporter) Abort(reason error) error {
7795
var errCosmos, errFlatKV error
7896
if si.cosmosImporter != nil {
79-
errCosmos = si.cosmosImporter.Close()
97+
errCosmos = si.cosmosImporter.Abort(reason)
8098
}
8199
if si.flatkvImporter != nil {
82-
errFlatKV = si.flatkvImporter.Close()
100+
errFlatKV = si.flatkvImporter.Abort(reason)
83101
}
84102
return errors.Join(errCosmos, errFlatKV)
85103
}

0 commit comments

Comments
 (0)