Skip to content

Commit cd793ec

Browse files
seidroid[bot]masih
andauthored
Backport release/v6.7: Make the composite store router an atomic pointer (#4473)
Backport of #4470 to `release/v6.7`. --------- Co-authored-by: Masih H. Derkani <m@derkani.org>
1 parent 7de88f1 commit cd793ec

3 files changed

Lines changed: 100 additions & 23 deletions

File tree

‎sei-db/state_db/sc/composite/store.go‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,10 @@ type CompositeCommitStore struct {
4646
// Manages routing of traffic between the memiavl and flatkv backends.
4747
// Built (and rebuilt) inside LoadVersion against the just-opened
4848
// backends so that lazily-eager constructors like
49-
// NewMemiavlMigrationIterator see a non-nil memiavl DB.
50-
router migration.Router
49+
// NewMemiavlMigrationIterator see a non-nil memiavl DB. SetWriteMode
50+
// replaces it while views from GetChildStoreByName may be reading it, so
51+
// access it only through loadRouter and storeRouter.
52+
router atomic.Pointer[migration.Router]
5153

5254
// ctx is the constructor's context. Each invocation of buildRouter
5355
// derives a per-router child context from it and stores the
@@ -412,7 +414,7 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type
412414
}
413415

414416
// Build a per-handle composite with its own router. Without this the read-only handle has
415-
// cs.router == nil and every read-side method nil-dereferences on first call. The new composite
417+
// a nil router and every read-side method nil-dereferences on first call. The new composite
416418
// inherits cs.ctx so cancellation of the parent context cascades, but buildRouter installs its own
417419
// child cancel so closing this handle does not affect the parent.
418420
ro := &CompositeCommitStore{
@@ -475,8 +477,25 @@ func (cs *CompositeCommitStore) resolveCurrentWriteMode(closeIdleFlatKV bool) er
475477
return nil
476478
}
477479

480+
// loadRouter returns the installed router, or nil before LoadVersion and
481+
// after Close.
482+
func (cs *CompositeCommitStore) loadRouter() migration.Router {
483+
if router := cs.router.Load(); router != nil {
484+
return *router
485+
}
486+
return nil
487+
}
488+
489+
func (cs *CompositeCommitStore) storeRouter(router migration.Router) {
490+
if router == nil {
491+
cs.router.Store(nil)
492+
return
493+
}
494+
cs.router.Store(&router)
495+
}
496+
478497
// buildRouter constructs the migration router against the currently-opened
479-
// backends and assigns it to cs.router. Must be called after memIAVL and
498+
// backends and installs it with storeRouter. Must be called after memIAVL and
480499
// flatKV (if any) have been opened via LoadVersion and after the effective
481500
// mode has been resolved.
482501
func (cs *CompositeCommitStore) buildRouter() error {
@@ -496,7 +515,7 @@ func (cs *CompositeCommitStore) buildRouter() error {
496515
if cs.routerCancel != nil {
497516
cs.routerCancel()
498517
}
499-
cs.router = router
518+
cs.storeRouter(router)
500519
cs.routerCancel = cancel
501520
return nil
502521
}
@@ -519,8 +538,8 @@ func (cs *CompositeCommitStore) SetMigrationBatchSize(batchSize int) error {
519538
batchSize = 0
520539
}
521540
cs.migrationBatchSize.Store(int64(batchSize))
522-
if cs.router != nil {
523-
cs.router.SetMigrationBatchSize(batchSize)
541+
if router := cs.loadRouter(); router != nil {
542+
router.SetMigrationBatchSize(batchSize)
524543
}
525544
return nil
526545
}
@@ -596,7 +615,7 @@ func (cs *CompositeCommitStore) SetWriteMode(targetWriteMode types.WriteMode) er
596615
"write mode is fixed at %q by configuration; runtime switching requires write mode %q",
597616
cs.config.WriteMode, types.Auto)
598617
}
599-
if cs.router == nil {
618+
if cs.loadRouter() == nil {
600619
return errors.New("SetWriteMode called before LoadVersion")
601620
}
602621
if targetWriteMode == cs.currentWriteMode {
@@ -718,7 +737,7 @@ func (cs *CompositeCommitStore) materializeFlatKV() error {
718737
func (cs *CompositeCommitStore) ApplyChangeSets(changesets []*proto.NamedChangeSet) error {
719738
if cs.currentWriteMode.IsMigrationMode() {
720739
firstBatchInBlock := !cs.migrationAdvancedThisCommit
721-
if err := cs.router.ApplyChangeSets(changesets, firstBatchInBlock); err != nil {
740+
if err := cs.loadRouter().ApplyChangeSets(changesets, firstBatchInBlock); err != nil {
722741
return fmt.Errorf("failed to apply changesets: %w", err)
723742
}
724743
cs.migrationAdvancedThisCommit = true
@@ -727,7 +746,7 @@ func (cs *CompositeCommitStore) ApplyChangeSets(changesets []*proto.NamedChangeS
727746
return nil
728747
}
729748

730-
err := cs.router.ApplyChangeSets(changesets, false)
749+
err := cs.loadRouter().ApplyChangeSets(changesets, false)
731750
if err != nil {
732751
return fmt.Errorf("failed to apply changesets: %w", err)
733752
}
@@ -1114,11 +1133,11 @@ func (cs *CompositeCommitStore) GetChildStoreByName(name string) types.CommitKVS
11141133
}
11151134
}
11161135

1117-
// The provider resolves cs.router at call time: SetWriteMode replaces
1136+
// The provider resolves the router at call time: SetWriteMode replaces
11181137
// the router while views vended here stay cached by rootmulti, and a
11191138
// captured router value would keep serving the pre-transition mode.
11201139
return migration.NewRouterCommitKVStore(
1121-
func() migration.Router { return cs.router },
1140+
cs.loadRouter,
11221141
name,
11231142
cs.Version,
11241143
func(start, end []byte, ascending bool) (db.Iterator, error) {
@@ -1166,7 +1185,7 @@ func (cs *CompositeCommitStore) ReleaseSnapshotRefs() error {
11661185
cs.routerCancel()
11671186
cs.routerCancel = nil
11681187
}
1169-
cs.router = nil
1188+
cs.storeRouter(nil)
11701189
if cs.memIAVL == nil {
11711190
return nil
11721191
}
@@ -1412,7 +1431,7 @@ func (cs *CompositeCommitStore) Close() error {
14121431
cs.routerCancel()
14131432
cs.routerCancel = nil
14141433
}
1415-
cs.router = nil
1434+
cs.storeRouter(nil)
14161435

14171436
if cs.memIAVL != nil {
14181437
if err := cs.memIAVL.Close(); err != nil {
@@ -1437,7 +1456,7 @@ func (cs *CompositeCommitStore) Get(store string, key []byte) (value []byte, ok
14371456
return nil, false, fmt.Errorf("key cannot be nil")
14381457
}
14391458

1440-
value, ok, err = cs.router.Read(store, key)
1459+
value, ok, err = cs.loadRouter().Read(store, key)
14411460
if err != nil {
14421461
return nil, false, fmt.Errorf("failed to read value: %w", err)
14431462
}
@@ -1452,7 +1471,7 @@ func (cs *CompositeCommitStore) GetProof(store string, key []byte) (*ics23.Commi
14521471
return nil, fmt.Errorf("key cannot be nil")
14531472
}
14541473

1455-
proof, err := cs.router.GetProof(store, key)
1474+
proof, err := cs.loadRouter().GetProof(store, key)
14561475
if err != nil {
14571476
return nil, fmt.Errorf("failed to get proof: %w", err)
14581477
}

‎sei-db/state_db/sc/composite/store_auto_test.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package composite
22

33
import (
4+
"bytes"
5+
"fmt"
6+
"sync"
47
"testing"
58

69
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
@@ -643,3 +646,58 @@ func TestComposite_Auto_CopyAvailability(t *testing.T) {
643646
require.Nil(t, cs.Copy(),
644647
"Copy is unavailable once flatkv is open")
645648
}
649+
650+
// TestComposite_Auto_ChildStoreReadsDuringWriteModeSwitch reads through a
651+
// cached child-store view while SetWriteMode replaces the router, as happens
652+
// when queries run during the migration kickoff block. Run with -race.
653+
func TestComposite_Auto_ChildStoreReadsDuringWriteModeSwitch(t *testing.T) {
654+
cs := openAutoStore(t, t.TempDir(), 25)
655+
defer func() { _ = cs.Close() }()
656+
657+
key, value := []byte("k"), []byte("v")
658+
require.NoError(t, cs.ApplyChangeSets([]*proto.NamedChangeSet{
659+
{Name: keys.BankStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{
660+
{Key: key, Value: value},
661+
}}},
662+
}))
663+
_, err := cs.Commit()
664+
require.NoError(t, err)
665+
666+
view := cs.GetChildStoreByName(keys.BankStoreKey)
667+
stop := make(chan struct{})
668+
reading := make(chan struct{})
669+
done := make(chan error, 1)
670+
go func() {
671+
defer close(done)
672+
for i := 0; ; i++ {
673+
if got := view.Get(key); !bytes.Equal(got, value) {
674+
done <- fmt.Errorf("Get(%q) = %x during the write mode switch, want %x", key, got, value)
675+
return
676+
}
677+
if i == 0 {
678+
close(reading)
679+
}
680+
select {
681+
case <-stop:
682+
return
683+
default:
684+
}
685+
}
686+
}()
687+
var stopOnce sync.Once
688+
stopReader := func() error {
689+
stopOnce.Do(func() { close(stop) })
690+
return <-done
691+
}
692+
defer func() { _ = stopReader() }()
693+
694+
// Switch only once the reader is running so the reads overlap the switch.
695+
select {
696+
case <-reading:
697+
case err := <-done:
698+
require.NoError(t, err)
699+
}
700+
require.NoError(t, cs.SetWriteMode(types.MigrateEVM))
701+
require.Equal(t, value, view.Get(key))
702+
require.NoError(t, stopReader())
703+
}

‎sei-db/state_db/sc/composite/store_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -858,7 +858,7 @@ func TestLoadVersionFlatKVOnlyReadWrite(t *testing.T) {
858858
err = cs.LoadLatest()
859859
require.NoError(t, err, "LoadLatest must not nil-deref memIAVL in FlatKVOnly")
860860
defer func() { _ = cs.Close() }()
861-
require.NotNil(t, cs.router, "router must be built after LoadLatest")
861+
require.NotNil(t, cs.loadRouter(), "router must be built after LoadLatest")
862862

863863
require.NoError(t, cs.ApplyChangeSets([]*proto.NamedChangeSet{
864864
{Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{
@@ -902,7 +902,7 @@ func TestLoadVersionFlatKVOnlyReadOnly(t *testing.T) {
902902
roComposite, ok := ro.(*CompositeCommitStore)
903903
require.True(t, ok)
904904
require.Nil(t, roComposite.memIAVL, "FlatKVOnly read-only must not have memIAVL")
905-
require.NotNil(t, roComposite.router, "read-only handle must have its own router")
905+
require.NotNil(t, roComposite.loadRouter(), "read-only handle must have its own router")
906906

907907
got, ok, err := roComposite.Get(keys.EVMStoreKey, []byte("k1"))
908908
require.NoError(t, err, "read-only handle must serve reads without nil-dereferencing router")
@@ -927,20 +927,20 @@ func TestLoadVersionRebuildsRouterOnReload(t *testing.T) {
927927

928928
err = cs.LoadLatest()
929929
require.NoError(t, err)
930-
firstRouter := cs.router
930+
firstRouter := cs.loadRouter()
931931
firstCancel := cs.routerCancel
932932
require.NotNil(t, firstRouter)
933933
require.NotNil(t, firstCancel)
934934

935935
err = cs.LoadLatest()
936936
require.NoError(t, err)
937-
require.NotNil(t, cs.router)
938-
require.NotSame(t, firstRouter, cs.router, "LoadVersion must rebuild the router")
937+
require.NotNil(t, cs.loadRouter())
938+
require.NotSame(t, firstRouter, cs.loadRouter(), "LoadVersion must rebuild the router")
939939
require.NotNil(t, cs.routerCancel)
940940

941941
require.NoError(t, cs.Close())
942942
require.Nil(t, cs.routerCancel, "Close must clear routerCancel")
943-
require.Nil(t, cs.router, "Close must clear router")
943+
require.Nil(t, cs.loadRouter(), "Close must clear router")
944944
}
945945

946946
// TestLoadVersionDoesNotMountMigrationStoreInMigrationMode pins the
@@ -2557,7 +2557,7 @@ func TestLoadVersionReadOnlyDuringMigrateEVMTransition(t *testing.T) {
25572557
roComposite, ok := ro.(*CompositeCommitStore)
25582558
require.True(t, ok)
25592559
require.NotSame(t, cs2, roComposite, "read-only LoadVersion returns an isolated handle")
2560-
require.NotNil(t, roComposite.router, "read-only handle must have its own router")
2560+
require.NotNil(t, roComposite.loadRouter(), "read-only handle must have its own router")
25612561

25622562
// The read-only memiavl also must not have a migration tree.
25632563
require.Nil(t, roComposite.memIAVL.GetChildStoreByName(migration.MigrationStore),

0 commit comments

Comments
 (0)