Skip to content

Commit 8a4dff3

Browse files
seidroid[bot]cody-littleyCody Littley
authored
Backport release/v6.7: Fix FlatKV state sync bad-hash scenario. (#4377)
Backport of #4370 to `release/v6.7`. --------- Co-authored-by: Cody Littley <56973212+cody-littley@users.noreply.github.com> Co-authored-by: Cody Littley <cody.littley@seinetwork.io>
1 parent 3558509 commit 8a4dff3

8 files changed

Lines changed: 208 additions & 23 deletions

File tree

‎sei-db/state_db/sc/flatkv/exporter.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ import (
1414
var _ types.Exporter = (*KVExporter)(nil)
1515

1616
// KVExporter exports all committed data from a read-only FlatKV store as raw
17-
// physical key/value pairs. It uses RawGlobalIterator to walk every data DB
18-
// in global lexicographic order and emits each row as a single
19-
// SnapshotNode without any parsing or conversion.
17+
// physical key/value pairs, one SnapshotNode per row, in strictly ascending
18+
// physical-key order across every data DB. CommitStore.Importer rejects any
19+
// other order, so that order is part of the snapshot format.
2020
//
2121
// The stream is self-describing: the first item is the keys.FlatKVStoreKey
2222
// module header (a string), mirroring the memiavl MultiTreeExporter which

‎sei-db/state_db/sc/flatkv/importer.go‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package flatkv
22

33
import (
4+
"bytes"
45
"context"
56
"errors"
67
"fmt"
@@ -168,6 +169,8 @@ func (w *dbWorker) flush() (err error) {
168169
type KVImporter struct {
169170
store *CommitStore
170171
version int64
172+
// requireAscendingKeys rejects the import unless physical keys arrive in strictly ascending order.
173+
requireAscendingKeys bool
171174

172175
ingestCh chan rawKVPair
173176
workers map[seidbtypes.KeyValueDB]*dbWorker
@@ -182,13 +185,16 @@ type KVImporter struct {
182185
finishErr error
183186
}
184187

185-
func NewKVImporter(store *CommitStore, version int64) types.Importer {
188+
// NewKVImporter builds the import pipeline that writes into store at version. When requireAscendingKeys is
189+
// set, the import fails unless every physical key is strictly greater than the one before it.
190+
func NewKVImporter(store *CommitStore, version int64, requireAscendingKeys bool) types.Importer {
186191
imp := &KVImporter{
187-
store: store,
188-
version: version,
189-
ingestCh: make(chan rawKVPair, ingestChanSize),
190-
workers: make(map[seidbtypes.KeyValueDB]*dbWorker, 4),
191-
done: make(chan struct{}),
192+
store: store,
193+
version: version,
194+
requireAscendingKeys: requireAscendingKeys,
195+
ingestCh: make(chan rawKVPair, ingestChanSize),
196+
workers: make(map[seidbtypes.KeyValueDB]*dbWorker, 4),
197+
done: make(chan struct{}),
192198
}
193199

194200
for _, ndb := range store.namedDataDBs() {
@@ -233,12 +239,20 @@ func (imp *KVImporter) dispatch() {
233239
}
234240
}()
235241

242+
var prevKey []byte
236243
for {
237244
select {
238245
case kv, ok := <-imp.ingestCh:
239246
if !ok {
240247
return
241248
}
249+
if imp.requireAscendingKeys {
250+
if err := checkAscending(prevKey, kv.Key); err != nil {
251+
imp.setErr(err)
252+
return
253+
}
254+
prevKey = kv.Key
255+
}
242256
db, err := imp.store.routePhysicalKey(kv.Key)
243257
if err != nil {
244258
imp.setErr(fmt.Errorf("route key: %w", err))
@@ -255,6 +269,23 @@ func (imp *KVImporter) dispatch() {
255269
}
256270
}
257271

272+
// checkAscending returns an error unless key is strictly greater than prevKey. A nil prevKey means key is
273+
// the first of the import.
274+
func checkAscending(prevKey []byte, key []byte) error {
275+
if prevKey == nil {
276+
return nil
277+
}
278+
switch cmp := bytes.Compare(prevKey, key); {
279+
case cmp == 0:
280+
return fmt.Errorf("flatkv import: duplicate physical key %x", key)
281+
case cmp > 0:
282+
return fmt.Errorf(
283+
"flatkv import: physical key %x is below preceding key %x; keys must be in ascending order",
284+
key, prevKey)
285+
}
286+
return nil
287+
}
288+
258289
func (imp *KVImporter) setErr(err error) {
259290
if imp.firstErr.CompareAndSwap(nil, &err) {
260291
imp.closeOnce.Do(func() { close(imp.done) })

‎sei-db/state_db/sc/flatkv/importer_test.go‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
package flatkv
22

33
import (
4+
"bytes"
45
"errors"
6+
"slices"
57
"sync/atomic"
68
"testing"
79
"time"
810

11+
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
12+
"github.com/sei-protocol/sei-chain/sei-db/proto"
913
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype"
14+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
1015
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types"
1116
"github.com/stretchr/testify/require"
1217
)
@@ -446,3 +451,98 @@ func TestKVImporter_BackpressureBlocksProducerUntilWorkersDrain(t *testing.T) {
446451
require.GreaterOrEqual(t, flushes, int64(2),
447452
"expected multiple flushes for %d storage pairs (got %d)", totalPairs, flushes)
448453
}
454+
455+
// storageNode returns an import node carrying a serialized storage row at version 1.
456+
func storageNode(t *testing.T, addr ktype.Address, slot ktype.Slot, value []byte) *types.SnapshotNode {
457+
t.Helper()
458+
row := vtype.NewStorageData().SetBlockHeight(1).SetValue((*[32]byte)(value)).Serialize()
459+
return &types.SnapshotNode{Key: storagePhysKey(addr, slot), Value: row, Version: 1}
460+
}
461+
462+
// addNodesInKeyOrder feeds nodes to imp in ascending physical-key order, the order Importer requires.
463+
func addNodesInKeyOrder(imp types.Importer, nodes []*types.SnapshotNode) {
464+
sorted := slices.Clone(nodes)
465+
slices.SortFunc(sorted, func(a *types.SnapshotNode, b *types.SnapshotNode) int {
466+
return bytes.Compare(a.Key, b.Key)
467+
})
468+
for _, n := range sorted {
469+
imp.AddNode(n)
470+
}
471+
}
472+
473+
// TestKVImporter_RepeatedPairRejected feeds an honest export followed by one forged pair 2^16 times. Accepted,
474+
// the forged row would be live while its uint16 LtHash limbs wrapped to zero, leaving the root hash equal to
475+
// the honest source's.
476+
func TestKVImporter_RepeatedPairRejected(t *testing.T) {
477+
src := setupTestStore(t)
478+
defer func() { require.NoError(t, src.Close()) }()
479+
480+
require.NoError(t, src.ApplyChangeSets(src.Version()+1, []*proto.NamedChangeSet{
481+
{Name: "evm", Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{
482+
{Key: evmStorageKey(addrN(0x01), slotN(0x01)), Value: padLeft32(0x11)},
483+
{Key: evmStorageKey(addrN(0x02), slotN(0x02)), Value: padLeft32(0x22)},
484+
}}},
485+
}))
486+
commitAndCheck(t, src)
487+
488+
exp, err := src.Exporter(1)
489+
require.NoError(t, err)
490+
nodes := drainExporter(t, exp)
491+
require.NoError(t, exp.Close())
492+
require.NotEmpty(t, nodes)
493+
494+
dst, imp := newKVImporterForTest(t, 1)
495+
defer func() { require.NoError(t, dst.Close()) }()
496+
497+
require.NoError(t, imp.AddModule(keys.FlatKVStoreKey))
498+
for _, n := range nodes {
499+
imp.AddNode(n)
500+
}
501+
forged := storageNode(t, addrN(0xBA), slotN(0xD0), padLeft32(0x66))
502+
for i := 0; i < 1<<16; i++ {
503+
imp.AddNode(forged)
504+
}
505+
506+
require.ErrorContains(t, imp.Close(), "duplicate physical key")
507+
require.Equal(t, int64(0), dst.Version(), "a rejected import must not finalize")
508+
}
509+
510+
// TestKVImporter_OutOfOrderKeyRejected verifies that Importer fails an import whose physical keys descend.
511+
func TestKVImporter_OutOfOrderKeyRejected(t *testing.T) {
512+
s, imp := newKVImporterForTest(t, 1)
513+
defer func() { require.NoError(t, s.Close()) }()
514+
515+
imp.AddNode(storageNode(t, addrN(0x02), slotN(0x02), padLeft32(0x22)))
516+
imp.AddNode(storageNode(t, addrN(0x01), slotN(0x01), padLeft32(0x11)))
517+
518+
require.ErrorContains(t, imp.Close(), "keys must be in ascending order")
519+
require.Equal(t, int64(0), s.Version(), "a rejected import must not finalize")
520+
}
521+
522+
// TestTrustedImporter_AcceptsAnyOrder verifies that TrustedImporter imports physical keys given in
523+
// descending order.
524+
func TestTrustedImporter_AcceptsAnyOrder(t *testing.T) {
525+
s := setupTestStore(t)
526+
defer func() { require.NoError(t, s.Close()) }()
527+
528+
imp, err := s.TrustedImporter(1)
529+
require.NoError(t, err)
530+
imp.AddNode(storageNode(t, addrN(0x02), slotN(0x02), padLeft32(0x22)))
531+
imp.AddNode(storageNode(t, addrN(0x01), slotN(0x01), padLeft32(0x11)))
532+
require.NoError(t, imp.Close())
533+
534+
require.Equal(t, int64(1), s.Version())
535+
for _, want := range []struct {
536+
addr ktype.Address
537+
slot ktype.Slot
538+
value []byte
539+
}{
540+
{addrN(0x01), slotN(0x01), padLeft32(0x11)},
541+
{addrN(0x02), slotN(0x02), padLeft32(0x22)},
542+
} {
543+
got, found := s.Get(keys.EVMStoreKey, evmStorageKey(want.addr, want.slot))
544+
require.True(t, found)
545+
require.Equal(t, want.value, got)
546+
}
547+
require.NoError(t, VerifyLtHash(s))
548+
}

‎sei-db/state_db/sc/flatkv/perdb_lthash_test.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -338,14 +338,20 @@ func TestPerDBLtHashAfterImport(t *testing.T) {
338338
imp, err := s.Importer(1)
339339
require.NoError(t, err)
340340

341+
var nodes []*scTypes.SnapshotNode
341342
for i := byte(1); i <= 5; i++ {
342343
addr := addrN(i)
343344
slot := slotN(i)
344345
storVal := vtype.NewStorageData().SetBlockHeight(1).SetValue(&[32]byte{i, 0xAA}).Serialize()
345346
acctVal := vtype.NewAccountData().SetBlockHeight(1).SetNonce(uint64(i)).Serialize()
346-
imp.AddNode(&scTypes.SnapshotNode{Key: storagePhysKey(addr, slot), Value: storVal, Version: 1})
347-
imp.AddNode(&scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 1})
347+
nodes = append(nodes, &scTypes.SnapshotNode{
348+
Key: storagePhysKey(addr, slot),
349+
Value: storVal,
350+
Version: 1,
351+
})
352+
nodes = append(nodes, &scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 1})
348353
}
354+
addNodesInKeyOrder(imp, nodes)
349355
require.NoError(t, imp.Close())
350356

351357
verifyPerDBLtHash(t, s)

‎sei-db/state_db/sc/flatkv/permodule_lthash_test.go‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -254,18 +254,28 @@ func TestPerModuleLtHashAfterImport(t *testing.T) {
254254
imp, err := s.Importer(1)
255255
require.NoError(t, err)
256256

257+
var nodes []*scTypes.SnapshotNode
257258
for i := byte(1); i <= 5; i++ {
258259
addr := addrN(i)
259260
slot := slotN(i)
260261
storVal := vtype.NewStorageData().SetBlockHeight(1).SetValue(&[32]byte{i, 0xAA}).Serialize()
261262
acctVal := vtype.NewAccountData().SetBlockHeight(1).SetNonce(uint64(i)).Serialize()
262-
imp.AddNode(&scTypes.SnapshotNode{Key: storagePhysKey(addr, slot), Value: storVal, Version: 1})
263-
imp.AddNode(&scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 1})
263+
nodes = append(nodes, &scTypes.SnapshotNode{
264+
Key: storagePhysKey(addr, slot),
265+
Value: storVal,
266+
Version: 1,
267+
})
268+
nodes = append(nodes, &scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 1})
264269

265270
// A non-EVM module row goes to miscDB under "gov/".
266271
govVal := vtype.NewMiscData().SetBlockHeight(1).SetValue([]byte{i, 0xC0}).Serialize()
267-
imp.AddNode(&scTypes.SnapshotNode{Key: ktype.ModulePhysicalKey("gov", []byte{i}), Value: govVal, Version: 1})
272+
nodes = append(nodes, &scTypes.SnapshotNode{
273+
Key: ktype.ModulePhysicalKey("gov", []byte{i}),
274+
Value: govVal,
275+
Version: 1,
276+
})
268277
}
278+
addNodesInKeyOrder(imp, nodes)
269279
require.NoError(t, imp.Close())
270280

271281
verifyModuleLtHash(t, s)
@@ -296,19 +306,33 @@ func TestPerModuleLtHashStateSyncImportSurvivesRestart(t *testing.T) {
296306
imp, err := s1.Importer(7)
297307
require.NoError(t, err)
298308

309+
var nodes []*scTypes.SnapshotNode
299310
for i := byte(1); i <= 5; i++ {
300311
addr := addrN(i)
301312
slot := slotN(i)
302313
storVal := vtype.NewStorageData().SetBlockHeight(7).SetValue(&[32]byte{i, 0xAA}).Serialize()
303314
acctVal := vtype.NewAccountData().SetBlockHeight(7).SetNonce(uint64(i)).Serialize()
304-
imp.AddNode(&scTypes.SnapshotNode{Key: storagePhysKey(addr, slot), Value: storVal, Version: 7})
305-
imp.AddNode(&scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 7})
315+
nodes = append(nodes, &scTypes.SnapshotNode{
316+
Key: storagePhysKey(addr, slot),
317+
Value: storVal,
318+
Version: 7,
319+
})
320+
nodes = append(nodes, &scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 7})
306321

307322
govVal := vtype.NewMiscData().SetBlockHeight(7).SetValue([]byte{i, 0xC0}).Serialize()
308323
bankVal := vtype.NewMiscData().SetBlockHeight(7).SetValue([]byte{i, 0xD0}).Serialize()
309-
imp.AddNode(&scTypes.SnapshotNode{Key: ktype.ModulePhysicalKey("gov", []byte{i}), Value: govVal, Version: 7})
310-
imp.AddNode(&scTypes.SnapshotNode{Key: ktype.ModulePhysicalKey("bank", []byte{i}), Value: bankVal, Version: 7})
324+
nodes = append(nodes, &scTypes.SnapshotNode{
325+
Key: ktype.ModulePhysicalKey("gov", []byte{i}),
326+
Value: govVal,
327+
Version: 7,
328+
})
329+
nodes = append(nodes, &scTypes.SnapshotNode{
330+
Key: ktype.ModulePhysicalKey("bank", []byte{i}),
331+
Value: bankVal,
332+
Version: 7,
333+
})
311334
}
335+
addNodesInKeyOrder(imp, nodes)
312336
require.NoError(t, imp.Close())
313337
verifyModuleLtHash(t, s1)
314338

‎sei-db/state_db/sc/flatkv/permodule_stats_test.go‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -209,17 +209,27 @@ func TestPerModuleStatsAfterImportSurvivesRestart(t *testing.T) {
209209

210210
imp, err := s1.Importer(7)
211211
require.NoError(t, err)
212+
var nodes []*scTypes.SnapshotNode
212213
for i := byte(1); i <= 5; i++ {
213214
addr := addrN(i)
214215
slot := slotN(i)
215216
storVal := vtype.NewStorageData().SetBlockHeight(7).SetValue(&[32]byte{i, 0xAA}).Serialize()
216217
acctVal := vtype.NewAccountData().SetBlockHeight(7).SetNonce(uint64(i)).Serialize()
217-
imp.AddNode(&scTypes.SnapshotNode{Key: storagePhysKey(addr, slot), Value: storVal, Version: 7})
218-
imp.AddNode(&scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 7})
218+
nodes = append(nodes, &scTypes.SnapshotNode{
219+
Key: storagePhysKey(addr, slot),
220+
Value: storVal,
221+
Version: 7,
222+
})
223+
nodes = append(nodes, &scTypes.SnapshotNode{Key: accountPhysKey(addr), Value: acctVal, Version: 7})
219224

220225
govVal := vtype.NewMiscData().SetBlockHeight(7).SetValue([]byte{i, 0xC0}).Serialize()
221-
imp.AddNode(&scTypes.SnapshotNode{Key: ktype.ModulePhysicalKey("gov", []byte{i}), Value: govVal, Version: 7})
226+
nodes = append(nodes, &scTypes.SnapshotNode{
227+
Key: ktype.ModulePhysicalKey("gov", []byte{i}),
228+
Value: govVal,
229+
Version: 7,
230+
})
222231
}
232+
addNodesInKeyOrder(imp, nodes)
223233
require.NoError(t, imp.Close())
224234
verifyModuleStats(t, s1)
225235

‎sei-db/state_db/sc/flatkv/store.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -816,7 +816,21 @@ func (s *CommitStore) Flush() error {
816816
return nil
817817
}
818818

819+
// Importer returns an importer for data from an untrusted source, such as a state sync peer. The import
820+
// fails unless physical keys arrive in strictly ascending order, which is the order KVExporter emits.
819821
func (s *CommitStore) Importer(version int64) (types.Importer, error) {
822+
return s.newImporter(version, true)
823+
}
824+
825+
// TrustedImporter returns an importer for data from a trusted source, such as an offline migration of this
826+
// node's own state. It accepts physical keys in any order, but each key must appear at most once: a repeat
827+
// is not detected and leaves the imported hash wrong.
828+
func (s *CommitStore) TrustedImporter(version int64) (types.Importer, error) {
829+
return s.newImporter(version, false)
830+
}
831+
832+
// newImporter prepares the store for an import at version and returns the importer.
833+
func (s *CommitStore) newImporter(version int64, requireAscendingKeys bool) (types.Importer, error) {
820834
if s.readOnly {
821835
return nil, errReadOnly
822836
}
@@ -839,7 +853,7 @@ func (s *CommitStore) Importer(version int64) (types.Importer, error) {
839853
if err := s.resetForImport(); err != nil {
840854
return nil, fmt.Errorf("reset store for import: %w", err)
841855
}
842-
return NewKVImporter(s, version), nil
856+
return NewKVImporter(s, version, requireAscendingKeys), nil
843857
}
844858

845859
// resetForImport purges all existing data so that a subsequent import

‎sei-db/tools/cmd/seidb/operations/import_flatkv_from_memiavl.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ func importMemiavlModulesToFlatKV(ctx context.Context, homeDir string, modules [
227227
}
228228
defer func() { _ = exporter.Close() }()
229229

230-
importer, err := store.Importer(height)
230+
importer, err := store.TrustedImporter(height)
231231
if err != nil {
232232
return fmt.Errorf("failed to create FlatKV importer at height %d: %w", height, err)
233233
}

0 commit comments

Comments
 (0)