Skip to content

Commit 5fe90c0

Browse files
authored
fix(rootmulti): stop snapshot restore when node is rejected by SC importer (#4412)
Snapshot restore now exit early when the SC (State Commitment) importer cannot import a node, instead of continuing without it. Previously the FlatKV importer skipped such nodes silently, while the restore still passed them on to the SS (State Store). - `Importer.AddNode` now returns an error. KVImporter rejects non-leaf nodes, nodes without a key, and nodes at a version other than the import height, instead of skipping them. - `rootmulti.Store.restore` stops on that error, and forwards a leaf to the state store only after SC accepts it. - The memiavl importer always returns nil, the composite importer returns its backend's error, and seidb import-flatkv-from-memiavl stops at the first rejected pair.
1 parent 82c0cda commit 5fe90c0

13 files changed

Lines changed: 281 additions & 72 deletions

File tree

‎sei-cosmos/storev2/rootmulti/flatkv_snapshot_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,12 @@ import (
99
"testing"
1010

1111
protoio "github.com/gogo/protobuf/io"
12+
snapshottypes "github.com/sei-protocol/sei-chain/sei-cosmos/snapshots/types"
1213
"github.com/sei-protocol/sei-chain/sei-cosmos/store/types"
1314
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
1415
seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config"
16+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype"
17+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
1518
abci "github.com/sei-protocol/sei-chain/sei-tendermint/abci/types"
1619
"github.com/stretchr/testify/require"
1720
)
@@ -355,6 +358,52 @@ func TestFlatKVOnlySnapshotRestorePopulatesSS(t *testing.T) {
355358
queryEqual("/evm/key", evmData.nonKey, makeNonce(uint64(snapHeight)))
356359
}
357360

361+
// TestFlatKVOnlySnapshotRestoreRejectsNodeAtOtherVersion restores a flatkv_only snapshot whose flatkv
362+
// section ends with a node stamped with a version other than the snapshot height. The restore must fail,
363+
// and the state store must not hold that node's value.
364+
func TestFlatKVOnlySnapshotRestoreRejectsNodeAtOtherVersion(t *testing.T) {
365+
cfg := flatKVOnlyConfig()
366+
ssCfg := seidbconfig.DefaultStateStoreConfig()
367+
ssCfg.Enable = true
368+
ssCfg.AsyncWriteBuffer = 0
369+
evmData := newEVMTestData(0x42)
370+
371+
const snapHeight = 8
372+
373+
srcStore, srcKeys := newTestRootMultiWithSS(t, t.TempDir(), cfg, ssCfg)
374+
for block := 1; block <= snapHeight; block++ {
375+
simulateFlatKVOnlyBlock(t, srcStore, srcKeys, block, evmData)
376+
}
377+
waitUntilSSVersion(t, srcStore, snapHeight)
378+
379+
var buf bytes.Buffer
380+
writer := protoio.NewDelimitedWriter(&buf)
381+
require.NoError(t, srcStore.Snapshot(snapHeight, writer))
382+
require.NoError(t, srcStore.Close())
383+
384+
// The flatkv section is the only one in a flatkv_only snapshot, so an item written after the snapshot
385+
// belongs to it.
386+
value := []byte("v9")
387+
require.NoError(t, writer.WriteMsg(&snapshottypes.SnapshotItem{
388+
Item: &snapshottypes.SnapshotItem_IAVL{IAVL: &snapshottypes.SnapshotIAVLItem{
389+
Key: ktype.ModulePhysicalKey("bank", []byte("supply")),
390+
Value: vtype.SerializeMisc(snapHeight, value),
391+
Version: snapHeight + 1,
392+
}},
393+
}))
394+
395+
dstStore, _ := newTestRootMultiWithSS(t, t.TempDir(), cfg, ssCfg)
396+
// The import reopened the commitment store, so closing releases it along with the state store.
397+
defer func() { _ = dstStore.Close() }()
398+
reader := protoio.NewDelimitedReader(bytes.NewReader(buf.Bytes()), 1<<30)
399+
_, err := dstStore.Restore(snapHeight, 1, reader)
400+
require.ErrorContains(t, err, "the import is at version")
401+
402+
got, err := dstStore.GetStateStore().Get("bank", snapHeight, []byte("supply"))
403+
require.NoError(t, err)
404+
require.NotEqual(t, value, got, "a node the commitment store rejected must not reach the state store")
405+
}
406+
358407
func simulateFlatKVOnlyBlock(
359408
t *testing.T,
360409
store *Store,

‎sei-cosmos/storev2/rootmulti/restore_test.go‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ import (
1111

1212
protoio "github.com/gogo/protobuf/io"
1313
snapshottypes "github.com/sei-protocol/sei-chain/sei-cosmos/snapshots/types"
14+
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
1415
"github.com/sei-protocol/sei-chain/sei-db/common/utils"
1516
seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config"
1617
seidbtypes "github.com/sei-protocol/sei-chain/sei-db/db_engine/types"
18+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
1719
"github.com/stretchr/testify/require"
1820
)
1921

@@ -118,18 +120,20 @@ func TestRestoreRejectsMalformedStream(t *testing.T) {
118120
}
119121
}
120122

121-
// fakeStateStore is a state store whose Import returns importErr, either at once or after reading every
122-
// node, and which counts the version writes restore makes.
123+
// fakeStateStore is a state store whose Import returns importErr, either at once or after reading and
124+
// recording every node, and which counts the version writes restore makes.
123125
type fakeStateStore struct {
124126
seidbtypes.StateStore
125127
returnEarly bool
126128
importErr error
129+
imported []seidbtypes.SnapshotNode
127130
versionWrites int
128131
}
129132

130133
func (f *fakeStateStore) Import(_ int64, ch <-chan seidbtypes.SnapshotNode) error {
131134
if !f.returnEarly {
132-
for range ch {
135+
for node := range ch {
136+
f.imported = append(f.imported, node)
133137
}
134138
}
135139
return f.importErr
@@ -196,6 +200,29 @@ func TestRestoreFailurePublishesNothing(t *testing.T) {
196200
}
197201
}
198202

203+
// TestRestoreStopsWhenCommitStoreRejectsNode pins that a node the SC importer rejects fails the restore, and
204+
// that the state store receives only the nodes accepted before it.
205+
func TestRestoreStopsWhenCommitStoreRejectsNode(t *testing.T) {
206+
store, _ := newTestRootMulti(t, t.TempDir(), flatKVOnlyConfig())
207+
ss := &fakeStateStore{}
208+
store.ssStore = ss
209+
210+
leaf := func(key string, version int64) snapshottypes.SnapshotItem {
211+
return snapshottypes.SnapshotItem{Item: &snapshottypes.SnapshotItem_IAVL{IAVL: &snapshottypes.SnapshotIAVLItem{
212+
Key: []byte(key),
213+
Value: vtype.SerializeMisc(1, []byte("v")),
214+
Version: version,
215+
}}}
216+
}
217+
items := []snapshottypes.SnapshotItem{storeItem(keys.FlatKVStoreKey), leaf("bank/a", 1), leaf("bank/b", 2)}
218+
219+
err := restoreWithin(t, store, snapshotStream(t, items))
220+
require.ErrorContains(t, err, "the import is at version")
221+
require.Len(t, ss.imported, 1)
222+
require.Equal(t, []byte("bank/a"), ss.imported[0].Key)
223+
require.Zero(t, ss.versionWrites, "a failed restore must not record the snapshot height")
224+
}
225+
199226
// TestRestoreSuccessPublishes pins that a successful restore publishes the memIAVL snapshot and records
200227
// the snapshot height on the state store.
201228
func TestRestoreSuccessPublishes(t *testing.T) {

‎sei-cosmos/storev2/rootmulti/store.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1302,9 +1302,12 @@ loop:
13021302
if node.Height == 0 && node.Value == nil {
13031303
node.Value = []byte{}
13041304
}
1305-
scImporter.AddNode(node)
1305+
if err = scImporter.AddNode(node); err != nil {
1306+
restoreErr = err
1307+
break loop
1308+
}
13061309

1307-
// Check if we should also import to SS store
1310+
// Only leaves the SC importer accepted reach the state store.
13081311
if ssImport != nil && node.Height == 0 {
13091312
if err = ssImport.send(seidbtypes.SnapshotNode{
13101313
StoreKey: storeKey,

‎sei-db/state_db/sc/composite/importer.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,16 +61,17 @@ func (si *SnapshotImporter) AddModule(name string) error {
6161
return nil
6262
}
6363

64-
func (si *SnapshotImporter) AddNode(node *types.SnapshotNode) {
64+
func (si *SnapshotImporter) AddNode(node *types.SnapshotNode) error {
6565
if si.currentModule == keys.FlatKVStoreKey {
6666
if si.flatkvImporter != nil {
67-
si.flatkvImporter.AddNode(node)
67+
return si.flatkvImporter.AddNode(node)
6868
}
69-
return
69+
return nil
7070
}
7171
if si.cosmosImporter != nil {
72-
si.cosmosImporter.AddNode(node)
72+
return si.cosmosImporter.AddNode(node)
7373
}
74+
return nil
7475
}
7576

7677
// Close publishes both backends' imports. When the cosmos import fails, the flatkv import is discarded

‎sei-db/state_db/sc/composite/importer_test.go‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,22 @@ import (
44
"errors"
55
"testing"
66

7+
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
78
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types"
89
"github.com/stretchr/testify/require"
910
)
1011

11-
// endRecordingImporter is an importer whose Close returns closeErr, and which records whether Close or
12-
// Abort ended it.
12+
// endRecordingImporter is an importer whose AddNode returns addNodeErr and whose Close returns closeErr, and
13+
// which records whether Close or Abort ended it.
1314
type endRecordingImporter struct {
14-
closeErr error
15-
endedBy string
15+
addNodeErr error
16+
closeErr error
17+
endedBy string
1618
}
1719

1820
func (e *endRecordingImporter) AddModule(string) error { return nil }
1921

20-
func (e *endRecordingImporter) AddNode(*types.SnapshotNode) {}
22+
func (e *endRecordingImporter) AddNode(*types.SnapshotNode) error { return e.addNodeErr }
2123

2224
func (e *endRecordingImporter) Close() error {
2325
e.endedBy = "close"
@@ -51,3 +53,17 @@ func TestSnapshotImporterCloseDiscardsFlatKVWhenCosmosFails(t *testing.T) {
5153
require.Equal(t, "abort", flatkv.endedBy)
5254
})
5355
}
56+
57+
// TestSnapshotImporterAddNodeReturnsBackendError pins that AddNode returns the error of the backend that the
58+
// current section routes to.
59+
func TestSnapshotImporterAddNodeReturnsBackendError(t *testing.T) {
60+
cosmos := &endRecordingImporter{addNodeErr: errors.New("cosmos rejected node")}
61+
flatkv := &endRecordingImporter{addNodeErr: errors.New("flatkv rejected node")}
62+
imp := NewImporter(cosmos, flatkv, nil)
63+
node := &types.SnapshotNode{Key: []byte("k"), Value: []byte("v")}
64+
65+
require.NoError(t, imp.AddModule("bank"))
66+
require.ErrorContains(t, imp.AddNode(node), "cosmos rejected node")
67+
require.NoError(t, imp.AddModule(keys.FlatKVStoreKey))
68+
require.ErrorContains(t, imp.AddNode(node), "flatkv rejected node")
69+
}

‎sei-db/state_db/sc/composite/store_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1281,8 +1281,9 @@ func (ti *trackingImporter) AddModule(name string) error {
12811281
return nil
12821282
}
12831283

1284-
func (ti *trackingImporter) AddNode(node *types.SnapshotNode) {
1284+
func (ti *trackingImporter) AddNode(node *types.SnapshotNode) error {
12851285
*ti.nodes = append(*ti.nodes, node)
1286+
return nil
12861287
}
12871288

12881289
func (ti *trackingImporter) Close() error { return nil }

‎sei-db/state_db/sc/flatkv/import_export_test.go‎

Lines changed: 43 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -585,7 +585,7 @@ func TestImporterOnReadOnlyStore(t *testing.T) {
585585
require.NoError(t, s.Close())
586586
}
587587

588-
func TestImporterHeightNonZeroSkipped(t *testing.T) {
588+
func TestImporterHeightNonZeroRejected(t *testing.T) {
589589
dir := t.TempDir()
590590
cfg := config.DefaultTestConfig(t)
591591
cfg.DataDir = filepath.Join(dir, flatkvRootDir)
@@ -598,45 +598,53 @@ func TestImporterHeightNonZeroSkipped(t *testing.T) {
598598
imp, err := s.Importer(1)
599599
require.NoError(t, err)
600600

601-
// Non-leaf nodes (Height != 0) are silently skipped.
602-
imp.AddNode(&types.SnapshotNode{
603-
Key: keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addrN(0x01), slotN(0x01))),
604-
Value: padLeft32(0x11),
605-
Height: 1, // non-leaf
606-
})
601+
key := storagePhysKey(addrN(0x01), slotN(0x01))
607602

608-
require.NoError(t, imp.Close())
603+
// Non-leaf nodes (Height != 0) fail the import.
604+
addErr := imp.AddNode(&types.SnapshotNode{
605+
Key: key,
606+
Value: padLeft32(0x11),
607+
Version: 1,
608+
Height: 1, // non-leaf
609+
})
610+
require.ErrorContains(t, addErr, "only leaves can be imported")
611+
require.ErrorIs(t, imp.Close(), addErr)
609612

610-
// Data should NOT have been imported.
611-
key := keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addrN(0x01), slotN(0x01)))
612-
_, found := s.Get(keys.EVMStoreKey, key)
613-
require.False(t, found, "height != 0 node should be skipped")
613+
_, found := s.Get(keys.EVMStoreKey, keys.BuildEVMKey(keys.EVMKeyStorage, ktype.StorageKey(addrN(0x01), slotN(0x01))))
614+
require.False(t, found, "height != 0 node should be rejected")
615+
require.Zero(t, s.Version(), "a rejected import must not finalize")
614616
require.NoError(t, s.Close())
615617
}
616618

617-
func TestImporterNilKeySkipped(t *testing.T) {
618-
dir := t.TempDir()
619-
cfg := config.DefaultTestConfig(t)
620-
cfg.DataDir = filepath.Join(dir, flatkvRootDir)
621-
622-
s, err := newCommitStoreWithWAL(t.Context(), cfg)
623-
require.NoError(t, err)
624-
err = s.LoadLatest()
625-
require.NoError(t, err)
626-
627-
imp, err := s.Importer(1)
628-
require.NoError(t, err)
629-
630-
// Nodes with nil key are silently skipped.
631-
imp.AddNode(&types.SnapshotNode{
632-
Key: nil,
633-
Value: []byte{0xAA},
634-
Height: 0,
635-
})
636-
637-
require.NoError(t, imp.Close())
638-
require.Equal(t, int64(1), s.Version())
639-
require.NoError(t, s.Close())
619+
func TestImporterEmptyKeyRejected(t *testing.T) {
620+
// A restore hands the importer an empty key where the snapshot carried none, so both forms must fail.
621+
for name, key := range map[string][]byte{"nil": nil, "zero-length": {}} {
622+
t.Run(name, func(t *testing.T) {
623+
dir := t.TempDir()
624+
cfg := config.DefaultTestConfig(t)
625+
cfg.DataDir = filepath.Join(dir, flatkvRootDir)
626+
627+
s, err := newCommitStoreWithWAL(t.Context(), cfg)
628+
require.NoError(t, err)
629+
err = s.LoadLatest()
630+
require.NoError(t, err)
631+
632+
imp, err := s.Importer(1)
633+
require.NoError(t, err)
634+
635+
addErr := imp.AddNode(&types.SnapshotNode{
636+
Key: key,
637+
Value: []byte{0xAA},
638+
Version: 1,
639+
Height: 0,
640+
})
641+
require.ErrorContains(t, addErr, "node has an empty key")
642+
643+
require.ErrorIs(t, imp.Close(), addErr)
644+
require.Zero(t, s.Version(), "a rejected import must not finalize")
645+
require.NoError(t, s.Close())
646+
})
647+
}
640648
}
641649

642650
func TestImporterEmptyStore(t *testing.T) {

‎sei-db/state_db/sc/flatkv/importer.go‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -337,22 +337,45 @@ func (imp *KVImporter) AddModule(_ string) error {
337337
return nil
338338
}
339339

340-
func (imp *KVImporter) AddNode(node *types.SnapshotNode) {
341-
if node.Height != 0 || node.Key == nil || node.Version != imp.version {
342-
return
340+
// AddNode queues node for import. It fails the import and returns an error unless node is a leaf at the
341+
// import's version with a non-empty key and value. Once the import has failed, it returns that failure.
342+
func (imp *KVImporter) AddNode(node *types.SnapshotNode) error {
343+
if err := imp.getErr(); err != nil {
344+
return err
345+
}
346+
if err := imp.checkNode(node); err != nil {
347+
imp.setErr(err)
348+
return err
349+
}
350+
select {
351+
case imp.ingestCh <- rawKVPair{Key: node.Key, Value: node.Value}:
352+
case <-imp.done:
353+
}
354+
// A worker can fail while the send is pending, and select may still pick the send, so the failure is
355+
// read again here rather than reporting the node as accepted.
356+
return imp.getErr()
357+
}
358+
359+
// checkNode returns an error unless node is a row this import can store.
360+
func (imp *KVImporter) checkNode(node *types.SnapshotNode) error {
361+
if node.Height != 0 {
362+
return fmt.Errorf("flatkv import: node %x has height %d; only leaves can be imported", node.Key, node.Height)
363+
}
364+
if len(node.Key) == 0 {
365+
return errors.New("flatkv import: node has an empty key")
366+
}
367+
if node.Version != imp.version {
368+
return fmt.Errorf("flatkv import: node %x has version %d; the import is at version %d",
369+
node.Key, node.Version, imp.version)
343370
}
344371
// FlatKV import nodes carry already-serialized physical values. Even an
345372
// empty logical misc value has a non-empty serialized header, so a
346373
// zero-length physical value is malformed. Reject it instead of writing a
347374
// Pebble row that LtHash and verification would both skip.
348375
if len(node.Value) == 0 {
349-
imp.setErr(fmt.Errorf("flatkv import: empty physical value for key %x", node.Key))
350-
return
351-
}
352-
select {
353-
case imp.ingestCh <- rawKVPair{Key: node.Key, Value: node.Value}:
354-
case <-imp.done:
376+
return fmt.Errorf("flatkv import: empty physical value for key %x", node.Key)
355377
}
378+
return nil
356379
}
357380

358381
// Abort tears down the worker pipeline without finalizing the import.

0 commit comments

Comments
 (0)