Skip to content

Commit 82c0cda

Browse files
blindchaserdevin-ai-integration[bot]cursoragent
authored
fix(seidb): fix stale FlatKV migration gauges on snapshotting nodes (#4436)
On atlantic-2, archive-0-0-0, snapshotter-0 and state-sync-node-0 finished the EVM migration (all 123,876,555 keys moved, `seidb_migration_version` went to 1), but the FlatKV migration dashboard still shows them as migrating, at 99.3% with ~875k EVM keys left in memIAVL. The migration is complete; two gauges report stale values, and only on nodes that export state-sync snapshots. A snapshot export opens a read-only composite store at the snapshot height, and when that height is before completion, the `MigrationManager` built for that handle records version 0 on the process-wide `seidb_migration_version` gauge. Nothing records 1 again until restart. Separately, `rootmulti.Store.Snapshot` records `iavl_total_num_keys` only for stores that exported at least one node, so once the memIAVL `evm` tree is empty its last pre-migration value is exported forever. This is the same retention problem #4327 fixed for `seidb_migration_boundary_snapshot`. `migration.BuildRouter` now takes `RouterOption`s, and `WithoutTelemetry()` gives the router's `MigrationManager` `newLocalMigrationMetrics()` instead of the OTel-backed instance. `CompositeCommitStore.buildRouter` passes it for derived stores (the `LoadVersionReadOnly` view and `Copy`), so only the live store publishes migration metrics. `Snapshot` sets the per-store totals to zero on each store header, so a store with no nodes records 0. Converting the version gauge to an observable gauge would also work, but it leaves read-only handles publishing the other migration counters, so they are cut off at the router instead. No consensus, state, or wire-format impact: only metric emission changes, and the option is variadic, so existing `BuildRouter` callers are unchanged. A store absent from an export entirely (rather than exported empty) still keeps its last value; that does not occur for `evm` after the migration. Affected nodes show correct values after deploy, once they restart and export their next snapshot. `TestLoadVersionReadOnlyDoesNotReportMigrationVersion` and `TestSnapshotReportsZeroKeysForEmptyStore` each fail without their half of the fix; the migration, composite and rootmulti suites pass. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 53168c2 commit 82c0cda

5 files changed

Lines changed: 235 additions & 7 deletions

File tree

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package rootmulti
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"testing"
7+
8+
protoio "github.com/gogo/protobuf/io"
9+
"github.com/stretchr/testify/require"
10+
"go.opentelemetry.io/otel"
11+
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
12+
"go.opentelemetry.io/otel/sdk/metric/metricdata"
13+
)
14+
15+
// collectTotalNumKeys collects once from reader and returns the
16+
// iavl_total_num_keys value of every store_name data point.
17+
func collectTotalNumKeys(t *testing.T, reader *sdkmetric.ManualReader) map[string]int64 {
18+
t.Helper()
19+
var collected metricdata.ResourceMetrics
20+
require.NoError(t, reader.Collect(context.Background(), &collected))
21+
22+
values := map[string]int64{}
23+
for _, scope := range collected.ScopeMetrics {
24+
for _, m := range scope.Metrics {
25+
if m.Name != "iavl_total_num_keys" {
26+
continue
27+
}
28+
gauge, ok := m.Data.(metricdata.Gauge[int64])
29+
require.True(t, ok, "unexpected data type %T", m.Data)
30+
for _, point := range gauge.DataPoints {
31+
name, ok := point.Attributes.Value("store_name")
32+
require.True(t, ok, "data point has no store_name attribute")
33+
values[name.AsString()] = point.Value
34+
}
35+
}
36+
}
37+
return values
38+
}
39+
40+
// TestSnapshotReportsZeroKeysForEmptyStore pins that a store exported with no
41+
// nodes is reported with zero keys rather than left unreported. After the EVM
42+
// migration the memiavl evm store is exported empty; a gauge that is never
43+
// re-recorded keeps exporting its pre-migration value.
44+
func TestSnapshotReportsZeroKeysForEmptyStore(t *testing.T) {
45+
reader := sdkmetric.NewManualReader()
46+
provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
47+
prev := otel.GetMeterProvider()
48+
otel.SetMeterProvider(provider)
49+
t.Cleanup(func() {
50+
otel.SetMeterProvider(prev)
51+
require.NoError(t, provider.Shutdown(context.Background()))
52+
})
53+
54+
cfg := evmMigratedConfig()
55+
evmData := newEVMTestData(0x56)
56+
store, storeKeys := newTestRootMulti(t, t.TempDir(), cfg)
57+
for block := 1; block <= 3; block++ {
58+
simulateBlock(t, store, storeKeys, block, evmData)
59+
}
60+
61+
var buf bytes.Buffer
62+
require.NoError(t, store.Snapshot(3, protoio.NewDelimitedWriter(&buf)))
63+
require.NoError(t, store.Close())
64+
65+
totals := collectTotalNumKeys(t, reader)
66+
require.Contains(t, totals, "evm", "empty memiavl evm store must still be reported")
67+
require.Zero(t, totals["evm"])
68+
require.Positive(t, totals["flatkv"], "migrated evm keys are reported under flatkv")
69+
}

‎sei-cosmos/storev2/rootmulti/store.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1463,6 +1463,9 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
14631463
return err
14641464
}
14651465
currentStoreName = item
1466+
keySizePerStore[item] = 0
1467+
valueSizePerStore[item] = 0
1468+
numKeysPerStore[item] = 0
14661469
default:
14671470
return fmt.Errorf("unknown item type %T", item)
14681471
}

‎sei-db/state_db/sc/composite/store.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -536,8 +536,14 @@ func (cs *CompositeCommitStore) resolveCurrentWriteMode(closeIdleFlatKV bool) er
536536
// mode has been resolved.
537537
func (cs *CompositeCommitStore) buildRouter() error {
538538
routerCtx, cancel := context.WithCancel(cs.ctx)
539+
var options []migration.RouterOption
540+
if cs.derived {
541+
// A derived store sees the migration state at its own height; publishing it would overwrite the
542+
// live store's gauges.
543+
options = append(options, migration.WithoutTelemetry())
544+
}
539545
router, err := migration.BuildRouter(
540-
routerCtx, cs.currentWriteMode, cs.memIAVL, cs.flatKV, int(cs.migrationBatchSize.Load()))
546+
routerCtx, cs.currentWriteMode, cs.memIAVL, cs.flatKV, int(cs.migrationBatchSize.Load()), options...)
541547
if err != nil {
542548
cancel()
543549
return fmt.Errorf("failed to build router: %w", err)
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package composite
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/stretchr/testify/require"
9+
"go.opentelemetry.io/otel"
10+
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
11+
"go.opentelemetry.io/otel/sdk/metric/metricdata"
12+
13+
"github.com/sei-protocol/sei-chain/sei-db/common/keys"
14+
"github.com/sei-protocol/sei-chain/sei-db/config"
15+
"github.com/sei-protocol/sei-chain/sei-db/proto"
16+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration"
17+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types"
18+
)
19+
20+
const migrationVersionInstrument = "seidb_migration_version"
21+
22+
// installManualMeterProvider routes the global OTel meter to a ManualReader
23+
// for the duration of the test.
24+
func installManualMeterProvider(t *testing.T) *sdkmetric.ManualReader {
25+
t.Helper()
26+
reader := sdkmetric.NewManualReader()
27+
provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
28+
prev := otel.GetMeterProvider()
29+
otel.SetMeterProvider(provider)
30+
t.Cleanup(func() {
31+
otel.SetMeterProvider(prev)
32+
require.NoError(t, provider.Shutdown(context.Background()))
33+
})
34+
return reader
35+
}
36+
37+
// collectMigrationVersions collects once from reader and returns every
38+
// seidb_migration_version data point value.
39+
func collectMigrationVersions(t *testing.T, reader *sdkmetric.ManualReader) []int64 {
40+
t.Helper()
41+
var collected metricdata.ResourceMetrics
42+
require.NoError(t, reader.Collect(context.Background(), &collected))
43+
44+
var values []int64
45+
for _, scope := range collected.ScopeMetrics {
46+
for _, m := range scope.Metrics {
47+
if m.Name != migrationVersionInstrument {
48+
continue
49+
}
50+
gauge, ok := m.Data.(metricdata.Gauge[int64])
51+
require.True(t, ok, "unexpected data type %T", m.Data)
52+
for _, point := range gauge.DataPoints {
53+
values = append(values, point.Value)
54+
}
55+
}
56+
}
57+
return values
58+
}
59+
60+
// TestLoadVersionReadOnlyDoesNotReportMigrationVersion pins that a read-only
61+
// handle opened at a mid-migration height leaves the process-wide migration
62+
// version gauge at the live manager's value. State-sync snapshot exports open
63+
// such handles after the migration completes; they must not report the
64+
// pre-completion version they observe.
65+
func TestLoadVersionReadOnlyDoesNotReportMigrationVersion(t *testing.T) {
66+
dir := t.TempDir()
67+
68+
// Phase 1: seed evm/ keys in MemiavlOnly so the migration spans several blocks.
69+
v0Cfg := config.DefaultStateCommitConfig()
70+
v0Cfg.WriteMode = types.MemiavlOnly
71+
cs1, err := NewCompositeCommitStore(t.Context(), dir, v0Cfg)
72+
require.NoError(t, err)
73+
require.NoError(t, cs1.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey}))
74+
require.NoError(t, cs1.LoadLatest())
75+
pairs := make([]*proto.KVPair, 0, 8)
76+
for i := 0; i < 8; i++ {
77+
pairs = append(pairs, &proto.KVPair{Key: []byte(fmt.Sprintf("evm_key_%d", i)), Value: []byte("v")})
78+
}
79+
require.NoError(t, cs1.ApplyChangeSets([]*proto.NamedChangeSet{
80+
{Name: keys.EVMStoreKey, Changeset: proto.ChangeSet{Pairs: pairs}},
81+
}))
82+
_, err = cs1.Commit(cs1.Version() + 1)
83+
require.NoError(t, err)
84+
require.NoError(t, cs1.Close())
85+
86+
reader := installManualMeterProvider(t)
87+
88+
// Phase 2: migrate one key per block until the completion block lands.
89+
migrateCfg := config.DefaultStateCommitConfig()
90+
migrateCfg.WriteMode = types.MigrateEVM
91+
cs2, err := NewCompositeCommitStore(t.Context(), dir, migrateCfg)
92+
require.NoError(t, err)
93+
require.NoError(t, cs2.SetMigrationBatchSize(1))
94+
require.NoError(t, cs2.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey}))
95+
require.NoError(t, cs2.LoadLatest())
96+
defer cs2.Close()
97+
98+
var midMigrationVersion int64
99+
for block := 0; block < 32; block++ {
100+
require.NoError(t, cs2.ApplyChangeSets([]*proto.NamedChangeSet{
101+
{Name: keys.BankStoreKey, Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{
102+
{Key: []byte(fmt.Sprintf("bank_%d", block)), Value: []byte("v")},
103+
}}},
104+
}))
105+
_, err = cs2.Commit(cs2.Version() + 1)
106+
require.NoError(t, err)
107+
if _, done := cs2.flatKV.Get(migration.MigrationStore, []byte(migration.MigrationVersionKey)); done {
108+
break
109+
}
110+
midMigrationVersion = cs2.Version()
111+
}
112+
require.NotZero(t, midMigrationVersion, "migration must span more than one block")
113+
require.Equal(t, []int64{int64(migration.Version1_MigrateEVM)}, collectMigrationVersions(t, reader),
114+
"precondition: live manager reports the completed migration version")
115+
116+
ro, err := cs2.LoadVersionReadOnly(midMigrationVersion)
117+
require.NoError(t, err)
118+
defer func() { _ = ro.Close() }()
119+
120+
require.Equal(t, []int64{int64(migration.Version1_MigrateEVM)}, collectMigrationVersions(t, reader),
121+
"read-only handle at a mid-migration height must not overwrite the live migration version")
122+
}

‎sei-db/state_db/sc/migration/router_builder.go‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,26 @@ import (
1212
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/types"
1313
)
1414

15+
// RouterOption adjusts how BuildRouter assembles a router.
16+
type RouterOption func(*routerOptions)
17+
18+
type routerOptions struct {
19+
telemetry bool
20+
}
21+
22+
// WithoutTelemetry makes the router's MigrationManager keep its migration
23+
// metrics in process, without publishing them on the process-wide OTel instruments.
24+
func WithoutTelemetry() RouterOption {
25+
return func(o *routerOptions) { o.telemetry = false }
26+
}
27+
28+
func (o routerOptions) migrationMetrics(ctx context.Context, targetVersion uint64) *MigrationMetrics {
29+
if !o.telemetry {
30+
return newLocalMigrationMetrics()
31+
}
32+
return NewMigrationMetrics(ctx, targetVersion)
33+
}
34+
1535
// Builds a router for the given migration write mode. A router is responsible for splitting
1636
// reads/writes between the memiavl and flatkv backends.
1737
func BuildRouter(
@@ -21,7 +41,12 @@ func BuildRouter(
2141
flatKV gigatypes.LiveStateStore,
2242
// If this router will be doing data migration, this is the number of keys to migrate in each batch.
2343
migrationBatchSize int,
44+
options ...RouterOption,
2445
) (Router, error) {
46+
opts := routerOptions{telemetry: true}
47+
for _, apply := range options {
48+
apply(&opts)
49+
}
2550

2651
switch writeMode {
2752
case types.MemiavlOnly:
@@ -31,7 +56,7 @@ func BuildRouter(
3156
}
3257
return router, nil
3358
case types.MigrateEVM:
34-
router, err := buildMigrateEVMRouter(ctx, memIAVL, flatKV, migrationBatchSize)
59+
router, err := buildMigrateEVMRouter(ctx, memIAVL, flatKV, migrationBatchSize, opts)
3560
if err != nil {
3661
return nil, fmt.Errorf("buildMigrateEVMRouter: %w", err)
3762
}
@@ -51,7 +76,7 @@ func BuildRouter(
5176
}
5277
return threadSafe, nil
5378
case types.MigrateAllButBank:
54-
router, err := buildMigrateAllButBankRouter(ctx, memIAVL, flatKV, migrationBatchSize)
79+
router, err := buildMigrateAllButBankRouter(ctx, memIAVL, flatKV, migrationBatchSize, opts)
5580
if err != nil {
5681
return nil, fmt.Errorf("buildMigrateAllButBankRouter: %w", err)
5782
}
@@ -71,7 +96,7 @@ func BuildRouter(
7196
}
7297
return threadSafe, nil
7398
case types.MigrateBank:
74-
router, err := buildMigrateBankRouter(ctx, memIAVL, flatKV, migrationBatchSize)
99+
router, err := buildMigrateBankRouter(ctx, memIAVL, flatKV, migrationBatchSize, opts)
75100
if err != nil {
76101
return nil, fmt.Errorf("buildMigrateBankRouter: %w", err)
77102
}
@@ -149,6 +174,7 @@ func buildMigrateEVMRouter(
149174
memIAVL *memiavl.CommitStore,
150175
flatKV gigatypes.LiveStateStore,
151176
migrationBatchSize int,
177+
opts routerOptions,
152178
) (Router, error) {
153179

154180
if memIAVL == nil {
@@ -167,7 +193,7 @@ func buildMigrateEVMRouter(
167193
buildFlatKVReader(flatKV),
168194
buildFlatKVWriter(flatKV),
169195
NewMemiavlMigrationIterator(memIAVL.GetDB(), []string{keys.EVMStoreKey}),
170-
NewMigrationMetrics(ctx, Version1_MigrateEVM),
196+
opts.migrationMetrics(ctx, Version1_MigrateEVM),
171197
)
172198
if err != nil {
173199
return nil, fmt.Errorf("NewMigrationManager: %w", err)
@@ -276,6 +302,7 @@ func buildMigrateAllButBankRouter(
276302
memIAVL *memiavl.CommitStore,
277303
flatKV gigatypes.LiveStateStore,
278304
migrationBatchSize int,
305+
opts routerOptions,
279306
) (Router, error) {
280307

281308
if memIAVL == nil {
@@ -299,7 +326,7 @@ func buildMigrateAllButBankRouter(
299326
buildFlatKVReader(flatKV),
300327
buildFlatKVWriter(flatKV),
301328
NewMemiavlMigrationIterator(memIAVL.GetDB(), allModulesButEvmAndBank),
302-
NewMigrationMetrics(ctx, Version2_MigrateAllButBank),
329+
opts.migrationMetrics(ctx, Version2_MigrateAllButBank),
303330
)
304331
if err != nil {
305332
return nil, fmt.Errorf("NewMigrationManager: %w", err)
@@ -401,6 +428,7 @@ func buildMigrateBankRouter(
401428
memIAVL *memiavl.CommitStore,
402429
flatKV gigatypes.LiveStateStore,
403430
migrationBatchSize int,
431+
opts routerOptions,
404432
) (Router, error) {
405433

406434
if memIAVL == nil {
@@ -426,7 +454,7 @@ func buildMigrateBankRouter(
426454
buildFlatKVReader(flatKV),
427455
buildFlatKVWriter(flatKV),
428456
NewMemiavlMigrationIterator(memIAVL.GetDB(), []string{keys.BankStoreKey}),
429-
NewMigrationMetrics(ctx, Version3_FlatKVOnly),
457+
opts.migrationMetrics(ctx, Version3_FlatKVOnly),
430458
)
431459
if err != nil {
432460
return nil, fmt.Errorf("NewMigrationManager: %w", err)

0 commit comments

Comments
 (0)