Skip to content

Commit 6e2c783

Browse files
authored
Log a pinned node's skipped migration kick-off once per batch size (#4408)
`app/abci.go` calls `rootmulti.Store.SetMigrationBatchSize` with `NumKeysToMigratePerBlock` on every block. On a node pinned to `memiavl_only` (a memIAVL reserve), each call with a positive batch size logs the "skipping migration kick-off" ERROR, so once governance raises the rate a reserve logs it every block for the rest of its life: about 8.2k lines an hour per arctic-1 reserve pod. That buries anything else at ERROR and makes the line useless as a signal. The skip is now logged once per batch size. `Store` remembers the batch size it last logged a skip for, logs again only when governance changes the value, and forgets it when the rate goes back to 0, so raising it again logs again. A restart logs once on the first block. The same applies to the INFO line for an SC store that does not expose a configured write mode. Nothing else changes: the kick-off decision and the write mode are untouched, so this is not app-hash-breaking. Covered by `TestRootMultiAutoKickoff_FixedMemiavlOnlySkipLoggedOncePerBatchSize`, which counts log records across repeated, changed and reset batch sizes; the existing rootmulti tests pass.
1 parent 792ff53 commit 6e2c783

2 files changed

Lines changed: 61 additions & 0 deletions

File tree

‎sei-cosmos/storev2/rootmulti/set_write_mode_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package rootmulti
22

33
import (
4+
"context"
5+
"log/slog"
6+
"sync/atomic"
47
"testing"
58

69
seidbconfig "github.com/sei-protocol/sei-chain/sei-db/config"
@@ -139,6 +142,43 @@ func TestRootMultiAutoKickoff_FixedMemiavlOnlySkips(t *testing.T) {
139142
require.Equal(t, sctypes.MemiavlOnly, mode)
140143
}
141144

145+
type countingHandler struct{ records *atomic.Int64 }
146+
147+
func (h countingHandler) Enabled(context.Context, slog.Level) bool { return true }
148+
func (h countingHandler) Handle(context.Context, slog.Record) error {
149+
h.records.Add(1)
150+
return nil
151+
}
152+
func (h countingHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
153+
func (h countingHandler) WithGroup(string) slog.Handler { return h }
154+
155+
// TestRootMultiAutoKickoff_FixedMemiavlOnlySkipLoggedOncePerBatchSize proves a
156+
// pinned node logs the skipped kick-off once per batch size rather than on
157+
// every block, and logs again when governance changes the batch size.
158+
func TestRootMultiAutoKickoff_FixedMemiavlOnlySkipLoggedOncePerBatchSize(t *testing.T) {
159+
dir := t.TempDir()
160+
store, _ := newTestRootMulti(t, dir, memiavlOnlyConfig())
161+
defer func() { require.NoError(t, store.Close()) }()
162+
163+
var records atomic.Int64
164+
prev := logger
165+
logger = slog.New(countingHandler{records: &records})
166+
defer func() { logger = prev }()
167+
168+
for range 3 {
169+
require.NoError(t, store.SetMigrationBatchSize(100))
170+
}
171+
require.Equal(t, int64(1), records.Load(), "repeated batch size must log once")
172+
173+
require.NoError(t, store.SetMigrationBatchSize(200))
174+
require.NoError(t, store.SetMigrationBatchSize(200))
175+
require.Equal(t, int64(2), records.Load(), "a new batch size must log again")
176+
177+
require.NoError(t, store.SetMigrationBatchSize(0))
178+
require.NoError(t, store.SetMigrationBatchSize(200))
179+
require.Equal(t, int64(3), records.Load(), "re-raising after 0 must log again")
180+
}
181+
142182
// TestRootMultiAutoKickoff_RestartResumesMigrateEVM proves the crash/restart
143183
// safety of the batch-size migration kick-off. Once a positive batch size has
144184
// advanced an auto store memiavl_only -> migrate_evm and the first migrating

‎sei-cosmos/storev2/rootmulti/store.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ type Store struct {
6868

6969
snapshotSCStoreWarnOnce sync.Once
7070

71+
// migrationSkipLoggedBatchSize is the batch size whose skipped migration kick-off was last
72+
// logged. SetMigrationBatchSize runs every block, so the skip is logged once per batch size.
73+
migrationSkipLoggedBatchSize int
74+
7175
// Hash logger state (per-block hash logging; a debugging/forensics tool). See hashlog.go.
7276
hashLoggerConfig config.HashLoggerConfig
7377
hashLoggerDisabled bool
@@ -811,6 +815,7 @@ func (rs *Store) SetMigrationBatchSize(batchSize int) error {
811815
return fmt.Errorf("failed to set SC store migration batch size: %w", err)
812816
}
813817
if batchSize <= 0 {
818+
rs.migrationSkipLoggedBatchSize = 0
814819
return nil
815820
}
816821
mode, ok := rs.GetWriteMode()
@@ -825,13 +830,19 @@ func (rs *Store) SetMigrationBatchSize(batchSize int) error {
825830
// cannot tell whether this is an auto store. Don't assume a deliberate
826831
// opt-out — skip with a distinct message so it isn't mistaken for the
827832
// pinned-memiavl_only case below during debugging.
833+
if !rs.shouldLogMigrationSkip(batchSize) {
834+
return nil
835+
}
828836
logger.Info(
829837
"migration requested (batch size > 0) but the SC store does not expose a "+
830838
"configured write mode; skipping migration kick-off",
831839
"effectiveWriteMode", mode, "batchSize", batchSize)
832840
return nil
833841
}
834842
if configured != sctypes.Auto {
843+
if !rs.shouldLogMigrationSkip(batchSize) {
844+
return nil
845+
}
835846
logger.Error(
836847
"migration requested (batch size > 0) but the SC write mode is pinned to fixed "+
837848
"memiavl_only by configuration; skipping migration kick-off. This node opts out of "+
@@ -847,6 +858,16 @@ func (rs *Store) SetMigrationBatchSize(batchSize int) error {
847858
return nil
848859
}
849860

861+
// shouldLogMigrationSkip reports whether a skipped kick-off at batchSize has
862+
// not been logged yet, and records it as logged.
863+
func (rs *Store) shouldLogMigrationSkip(batchSize int) bool {
864+
if rs.migrationSkipLoggedBatchSize == batchSize {
865+
return false
866+
}
867+
rs.migrationSkipLoggedBatchSize = batchSize
868+
return true
869+
}
870+
850871
// GetWriteMode returns the SC store's effective write mode. The bool is
851872
// false when the underlying SC store does not expose one. Intended for the
852873
// migration kick-off in SetMigrationBatchSize, observability, and tests.

0 commit comments

Comments
 (0)