Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions sei-cosmos/server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -557,9 +557,10 @@ func GetConfig(v *viper.Viper) (Config, error) {
// FlatKV knobs are not rendered in the default app.toml template. GetConfig
// is a faithful parse of app.toml/flags: it only reads the explicit
// state-commit.flatkv.* keys (if an operator adds them by hand) on top of the
// in-code defaults. The FlatKV-follows-memIAVL mirror (and snapshot cadence
// normalization) is applied later by composite.alignFlatKVSnapshotWithMemIAVL
// at store construction, so we deliberately do not mirror the sc-* keys here.
// in-code defaults. The FlatKV-follows-memIAVL interval mirror (and its
// cadence normalization) is applied later by
// composite.alignFlatKVSnapshotIntervalWithMemIAVL at store construction, so
// we deliberately do not mirror the sc-* keys here.
flatKVConfig := config.DefaultStateCommitConfig().FlatKVConfig
if v.IsSet("state-commit.flatkv.fsync") {
flatKVConfig.Fsync = v.GetBool("state-commit.flatkv.fsync")
Expand Down
12 changes: 7 additions & 5 deletions sei-cosmos/server/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -566,8 +566,9 @@ func TestGetConfigParsesRawSnapshotKeepRecent(t *testing.T) {
require.NoError(t, err)
// GetConfig is a faithful parse of app.toml/flags: the raw 0 is preserved for
// memIAVL here and only floored later at store construction. FlatKV does not
// mirror the sc-* keys in GetConfig (that is composite.alignFlatKVSnapshotWithMemIAVL's
// job), so it keeps its in-code default.
// mirror the sc-* keys in GetConfig (the interval mirror is
// composite.alignFlatKVSnapshotIntervalWithMemIAVL's job, and the retention
// count is never mirrored), so it keeps its in-code default.
require.Equal(t, uint32(0), cfg.StateCommit.MemIAVLConfig.SnapshotKeepRecent)
require.Equal(t, seidbconfig.DefaultStateCommitConfig().FlatKVConfig.SnapshotKeepRecent, cfg.StateCommit.FlatKVConfig.SnapshotKeepRecent)
}
Expand All @@ -594,9 +595,10 @@ func TestGetConfigHonorsExplicitFlatKVOverrides(t *testing.T) {
}

// TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent locks in the regression fix:
// GetConfig does not mirror the sc-* keys onto FlatKV (that is
// composite.alignFlatKVSnapshotWithMemIAVL's job at store construction), and an
// absent sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV
// GetConfig does not mirror the sc-* keys onto FlatKV (the interval mirror is
// composite.alignFlatKVSnapshotIntervalWithMemIAVL's job at store construction,
// and the retention count is never mirrored), and an absent
// sc-snapshot-interval / sc-keep-recent must preserve the in-code FlatKV
// defaults rather than reading back 0 (which disables FlatKV snapshots and drops
// all old snapshots).
func TestGetConfigFlatKVDefaultsWhenSCSnapshotAbsent(t *testing.T) {
Expand Down
5 changes: 3 additions & 2 deletions sei-db/config/ss_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,9 @@ func TestAlignSSSnapshotWithSCZeroesCadenceWhenDisabled(t *testing.T) {
require.Zero(t, ssConfig.SnapshotMinTimeInterval)
}

// FlatKV and SS both mirror memIAVL's cadence, and they must resolve it
// identically or the two backends drift onto different snapshot heights.
// SS mirrors memIAVL's whole cadence and FlatKV mirrors its interval, and every
// mirror must resolve that interval identically or the backends drift onto
// different snapshot heights.
func TestAlignSSSnapshotMatchesEffectiveMemIAVLCadence(t *testing.T) {
for _, tc := range []struct {
name string
Expand Down
60 changes: 30 additions & 30 deletions sei-db/state_db/sc/composite/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func NewCompositeCommitStore(
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("invalid state commit config: %w", err)
}
alignFlatKVSnapshotWithMemIAVL(&cfg)
alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)

var memIAVL *memiavl.CommitStore
if cfg.WriteMode != types.FlatKVOnly {
Expand Down Expand Up @@ -233,38 +233,38 @@ func (cs *CompositeCommitStore) recordFlatKVHash(_ context.Context, _ int64, has
return nil
}

// alignFlatKVSnapshotWithMemIAVL keeps the two backends' snapshot cadence in
// sync. FlatKV has no independently-exposed snapshot knobs in app.toml, so it
// derives its snapshot-interval / keep-recent from memIAVL's sc-* keys. This is
// the single place both backends are constructed from the same config, so it is
// where the alignment is enforced.
// alignFlatKVSnapshotIntervalWithMemIAVL makes FlatKV take its snapshot interval
// from memIAVL's sc-snapshot-interval. This is the single place both backends are
// constructed from the same config, so it is where the alignment is enforced.
//
// This derivation is intentionally unconditional across write modes, including
// FlatKVOnly — where NewCompositeCommitStore never constructs a memIAVL store.
// The sc-* keys are the only operator-visible snapshot-cadence knobs now that
// the flatkv.* keys are hidden from the app.toml template, so they must govern
// FlatKV's cadence in every mode; otherwise FlatKVOnly would have no
// template-visible way to tune it. It is harmless when memIAVL is absent: the
// sc-* defaults match FlatKV's own in-code defaults, and only cfg.FlatKVConfig
// is read when building the FlatKVOnly store.
// The interval must match because a composite operation needs a version *both*
// backends hold a snapshot for: a rollback rewinds memIAVL and then FlatKV, and a
// cross-backend digest has to open each at the same height. Two backends
// checkpointing on different heights have no such version in common.
//
// FlatKV mirrors memIAVL's *effective* cadence: a zero memIAVL value is first
// resolved to the same default Options.FillDefaults would apply at OpenDB
// (interval 0 -> DefaultSnapshotInterval, keep-recent 0 -> DefaultSnapshotKeepRecent),
// then assigned to FlatKV unconditionally. Resolving-then-assigning (rather than
// skipping on a zero and letting FlatKV keep its own in-code default) keeps the
// two backends in true lockstep without relying on FlatKV's default happening to
// equal memIAVL's healed default. That reliance is fragile — the defaults are
// only kept equal by hand — and it breaks for an upgrading node whose old
// app.toml still carries an explicit state-commit.flatkv.snapshot-keep-recent
// (rendered by the old template) alongside sc-keep-recent = 0: skipping would
// leave FlatKV pinned to the stale explicit value while memIAVL healed to a
// different default. Note that mirroring a raw 0 is never correct here (0 means
// "disable auto-snapshots" for FlatKV), which is why the zero is resolved first.
func alignFlatKVSnapshotWithMemIAVL(cfg *config.StateCommitConfig) {
interval, keepRecent := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig)
// Retention count is deliberately not mirrored. It is a per-backend disk decision
// rather than a cadence, and the two backends' costs differ by a factor of
// roughly 200: measured at mainnet state size, one further retained snapshot
// costs 56,782 MiB on memIAVL, whose snapshots are independent full copies, and
// about 286 MiB on FlatKV, whose checkpoints hardlink their SSTs. A single shared
// count cannot serve both — the depth FlatKV wants for forensic reach into the
// migration window would ask memIAVL for more than the volume holds. FlatKV
// therefore keeps config.DefaultSnapshotKeepRecent, which is sized for that reach.
//
// The mirror is unconditional across write modes, including FlatKVOnly, where
// NewCompositeCommitStore never constructs a memIAVL store. sc-snapshot-interval
// is the only operator-visible cadence knob, since the flatkv.* keys are hidden
// from the app.toml template and the production reader does not consult them, so
// it has to govern FlatKV's interval in every mode. It is harmless when memIAVL is
// absent, because only cfg.FlatKVConfig is read when building that store.
//
// A zero is resolved before it is assigned, to the same default
// Options.FillDefaults would apply at OpenDB. Mirroring a raw 0 is never correct:
// 0 disables auto-snapshots for FlatKV, which lets the WAL grow without bound and
// makes every restart replay from snapshot-0.
func alignFlatKVSnapshotIntervalWithMemIAVL(cfg *config.StateCommitConfig) {
interval, _ := config.EffectiveMemIAVLSnapshotCadence(cfg.MemIAVLConfig)
cfg.FlatKVConfig.SnapshotInterval = interval
cfg.FlatKVConfig.SnapshotKeepRecent = keepRecent
}

// Initialize records the set of child store names that should exist on
Expand Down
82 changes: 46 additions & 36 deletions sei-db/state_db/sc/composite/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (

gigatypes "github.com/sei-protocol/sei-chain/sei-db/state_db/giga/types"
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv"
flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config"
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/ktype"
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash"
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl"
Expand Down Expand Up @@ -1058,12 +1059,13 @@ func evmMigratedConfig() config.StateCommitConfig {
cfg.MemIAVLConfig.SnapshotInterval = 1
cfg.MemIAVLConfig.SnapshotMinTimeInterval = 0
cfg.MemIAVLConfig.AsyncCommitBuffer = 0
// With SnapshotInterval=1 every commit produces a snapshot, and FlatKV
// mirrors this cadence via alignFlatKVSnapshotWithMemIAVL. The default
// keep-recent of 1 would prune all but the two newest snapshots, so a
// With SnapshotInterval=1 every commit produces a snapshot, and memIAVL's
// default keep-recent of 1 would prune all but the two newest, so a
// rollback/reconcile to an older version (e.g. v3 after committing v5)
// could no longer find a base snapshot at-or-below the target. Retain all
// snapshots for the short duration of a test so those paths stay valid.
// FlatKV needs no equivalent here: it mirrors the interval but keeps its
// own retention count, which is deep enough already.
cfg.MemIAVLConfig.SnapshotKeepRecent = 100
return cfg
}
Expand Down Expand Up @@ -2595,60 +2597,68 @@ func TestLoadVersionReadOnlyDuringMigrateEVMTransition(t *testing.T) {
require.Equal(t, []byte(evmVal), got)
}

func TestAlignFlatKVSnapshotWithMemIAVL(t *testing.T) {
t.Run("FlatKV derives interval and keep-recent from a non-zero memIAVL", func(t *testing.T) {
func TestAlignFlatKVSnapshotIntervalWithMemIAVL(t *testing.T) {
t.Run("FlatKV derives its interval from a non-zero memIAVL", func(t *testing.T) {
cfg := config.DefaultStateCommitConfig()
cfg.MemIAVLConfig.SnapshotInterval = 5000
cfg.MemIAVLConfig.SnapshotKeepRecent = 3
// Start FlatKV from divergent values to prove they get overwritten.
// Start FlatKV from a divergent value to prove it gets overwritten.
cfg.FlatKVConfig.SnapshotInterval = 111
cfg.FlatKVConfig.SnapshotKeepRecent = 222

alignFlatKVSnapshotWithMemIAVL(&cfg)
alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)

require.Equal(t, uint32(5000), cfg.FlatKVConfig.SnapshotInterval)
require.Equal(t, uint32(3), cfg.FlatKVConfig.SnapshotKeepRecent)
})

t.Run("a zero memIAVL keep-recent resolves to the healed default", func(t *testing.T) {
cfg := config.DefaultStateCommitConfig()
cfg.MemIAVLConfig.SnapshotKeepRecent = 0
// FlatKV must not mirror the raw 0 (which would prune everything but the
// latest). Instead it mirrors the value FillDefaults will heal memIAVL to,
// keeping the two in lockstep. memIAVL's own 0 is left for FillDefaults.
alignFlatKVSnapshotWithMemIAVL(&cfg)

require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent)
require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent)
})

t.Run("a zero memIAVL interval resolves to the healed default", func(t *testing.T) {
cfg := config.DefaultStateCommitConfig()
cfg.MemIAVLConfig.SnapshotInterval = 0
// A raw 0 would disable FlatKV auto-snapshots; instead FlatKV mirrors the
// value FillDefaults will heal memIAVL's interval to.
alignFlatKVSnapshotWithMemIAVL(&cfg)
alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] This subtest no longer proves anything after the deleted "an explicit FlatKV override loses to memIAVL's healed default" case. It starts from DefaultStateCommitConfig(), so cfg.FlatKVConfig.SnapshotInterval is already 10000, and memiavl.DefaultSnapshotInterval is also 10000 (sei-db/state_db/sc/memiavl/config.go:4) — the assertion passes even if alignFlatKVSnapshotIntervalWithMemIAVL did nothing at all on a zero memIAVL interval.

The deleted subtest was the only one covering resolve-then-assign versus skip-on-zero, i.e. the upgrade case where an old app.toml still pins an explicit state-commit.flatkv.snapshot-interval while sc-snapshot-interval is 0. That scenario is still live for the interval even though the retention half of it is gone. One line restores it:

cfg.MemIAVLConfig.SnapshotInterval = 0
cfg.FlatKVConfig.SnapshotInterval = 7777 // must lose to memIAVL's healed default


require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval)
require.NotZero(t, cfg.FlatKVConfig.SnapshotInterval)
})

t.Run("an explicit FlatKV override loses to memIAVL's healed default", func(t *testing.T) {
// Upgrade scenario: an old app.toml still pins an explicit FlatKV
// keep-recent/interval (the previous template rendered flatkv.* keys)
// while sc-* is 0. FlatKV must follow memIAVL's effective (healed) cadence
// rather than staying pinned to the stale explicit value, otherwise the
// two backends diverge (memIAVL heals 0 -> default, FlatKV keeps the old
// explicit value).
t.Run("retention count is not mirrored", func(t *testing.T) {
// The two backends share a cadence and not a disk budget. A retained
// memIAVL snapshot is an independent full copy where a FlatKV checkpoint
// hardlinks its SSTs, so one shared count cannot serve both.
cfg := config.DefaultStateCommitConfig()
cfg.MemIAVLConfig.SnapshotKeepRecent = 3
cfg.FlatKVConfig.SnapshotKeepRecent = 222

alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)

require.Equal(t, uint32(222), cfg.FlatKVConfig.SnapshotKeepRecent)
require.Equal(t, uint32(3), cfg.MemIAVLConfig.SnapshotKeepRecent)
})

t.Run("a zero memIAVL keep-recent does not reach FlatKV", func(t *testing.T) {
// A zero memIAVL keep-recent is healed to memiavl.DefaultSnapshotKeepRecent
// for memIAVL's own use. That healed value must not reach FlatKV, whose
// own default stands.
cfg := config.DefaultStateCommitConfig()
cfg.MemIAVLConfig.SnapshotKeepRecent = 0
cfg.MemIAVLConfig.SnapshotInterval = 0
cfg.FlatKVConfig.SnapshotKeepRecent = 2
cfg.FlatKVConfig.SnapshotInterval = 7777

alignFlatKVSnapshotWithMemIAVL(&cfg)
alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)

require.Equal(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent)
require.Equal(t, uint32(memiavl.DefaultSnapshotInterval), cfg.FlatKVConfig.SnapshotInterval)
require.Equal(t, uint32(0), cfg.MemIAVLConfig.SnapshotKeepRecent)
require.Equal(t, flatkvconfig.DefaultSnapshotKeepRecent, cfg.FlatKVConfig.SnapshotKeepRecent)
require.NotEqual(t, uint32(memiavl.DefaultSnapshotKeepRecent), cfg.FlatKVConfig.SnapshotKeepRecent,
"FlatKV must not inherit memIAVL's retention count")
})
}

// The default exists to bound what can be asked about a past height, so a reach
// shorter than the migration window it was sized for is the regression to catch.
// 72 checkpoints at a 10000-block interval is 720,000 blocks, about 89 hours at
// mainnet's measured 2.247 blocks/s, against an 85-hour drain at K=1024.
func TestFlatKVDefaultRetentionSpansTheMigrationWindow(t *testing.T) {
cfg := config.DefaultStateCommitConfig()
alignFlatKVSnapshotIntervalWithMemIAVL(&cfg)

reach := uint64(cfg.FlatKVConfig.SnapshotKeepRecent) * uint64(cfg.FlatKVConfig.SnapshotInterval)
require.GreaterOrEqual(t, reach, uint64(690_000),
"default FlatKV retention must reach back across an 85-hour drain at 2.247 blocks/s")
}
24 changes: 23 additions & 1 deletion sei-db/state_db/sc/flatkv/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,23 @@ import (
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/lthash"
)

// DefaultSnapshotKeepRecent is how many old checkpoints (besides the latest) to
// keep, which at the default snapshot interval of 10000 is a guaranteed reach of
// 720,000 blocks — about 89 hours at mainnet's block rate, so it spans the EVM
// migration window at the rate that window is planned for.
//
// It is this deep because a FlatKV checkpoint is nearly free. Checkpoints
// hardlink their SSTs, so one only costs the bytes compaction has since made
// obsolete: measured at mainnet state size, 261 MiB of pinned SSTs plus about
// 25 MiB of retained state WAL. 72 of them is roughly 20 GiB. The cost is linear

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[suggestion] The 261 MiB / 25 MiB per-checkpoint figures are measured at the tip post-H_done, but the depth is chosen for the drain window — and both components scale with write volume, which is much higher during the drain than at the tip.

The WAL half is the clearer one: tryTruncateWAL (sei-db/state_db/sc/flatkv/snapshot.go:905) prunes below the earliest snapshot, so raising retention from 1 to 72 widens the retained WAL span from ~20k to 720k blocks. During the drain every block's changeset additionally carries NumKeysToMigratePerBlock (1024) migrated keys through s.wal.Write (store_write.go:86), so WAL bytes per interval are far above the steady-state 25 MiB. Near the end of an 85-hour drain the retained WAL approximates the whole migrated EVM dataset in changeset form. The pinned-SST half moves the same way: a bulk ingest obsoletes far more per interval than tip traffic does, so each checkpoint pins more than 261 MiB.

So peak disk during exactly the window this depth exists for is plausibly several times the ~20 GiB the doc and CHANGELOG quote. It likely still fits the 2 TiB volume, but the number operators will size from is the steady-state one. Worth either measuring one interval mid-drain and quoting the peak, or stating that 20 GiB is the post-migration steady state and the drain peak is higher.

This compounds with there being no escape hatch: parseSCConfigs (app/seidb.go:104) reads no FlatKV snapshot key, and this PR removes sc-keep-recent as the indirect lever, so an operator who runs short of disk mid-drain cannot lower the retention without a new binary.

// in depth, because each older checkpoint pins exactly the files obsoleted during
Comment on lines +18 to +22

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Rewrite the constant Godoc to describe only the constant

This new Godoc devotes multiple paragraphs to migration rationale, mainnet measurements, hardlink mechanics, and affected commands rather than documenting what DefaultSnapshotKeepRecent represents. The repository explicitly requires Godocs to describe their subject in one or two sentences and keep rationale, mechanisms, and surrounding architecture out, so this should be reduced to the retention semantics and the rationale moved inline or omitted.

AGENTS.md reference: AGENTS.md:L79-L90

Useful? React with 👍 / 👎.

// its own interval and those sets are disjoint.
//
// Reach matters because it bounds what can be answered about a past height at
// all. Below it, migrate-evm-status, dump-flatkv and a cross-backend digest
// cannot open a version, and a rollback has no base snapshot to rewind to.
const DefaultSnapshotKeepRecent uint32 = 72

// Config defines configuration for the FlatKV (EVM) commit store.
type Config struct {
// DataDir is the root directory for the FlatKV data files.
Expand All @@ -34,6 +51,11 @@ type Config struct {
// SnapshotKeepRecent defines how many old snapshots to keep besides the
// latest one. 0 means keep only the current snapshot (no old snapshots).
// Ignored entirely when ExternalPruning is set.
//
// It is not mirrored from memIAVL's sc-keep-recent, and the production store
// reads no app.toml key for it, so a node runs the DefaultConfig value. See
// composite.alignFlatKVSnapshotIntervalWithMemIAVL for why the two backends
// share an interval but not a retention count.
Comment on lines +55 to +58

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Remove configuration wiring from the field Godoc

The added paragraph documents which production reader and composite helper configure this field, rather than what SnapshotKeepRecent means to callers. That system-level wiring is expressly excluded from field Godocs by the repository guidance and will also become stale when configuration paths change; keep the existing retention and pruning semantics here and place wiring rationale at the relevant implementation site instead.

AGENTS.md reference: AGENTS.md:L79-L90

Useful? React with 👍 / 👎.

SnapshotKeepRecent uint32 `mapstructure:"snapshot-keep-recent"`

// MaxSnapshotLagBlocks is how many committed blocks may queue up behind a snapshot that is still
Expand Down Expand Up @@ -136,7 +158,7 @@ func DefaultConfig() *Config {
Fsync: false,
AsyncWriteBuffer: 0,
SnapshotInterval: 10000,
SnapshotKeepRecent: 1,
SnapshotKeepRecent: DefaultSnapshotKeepRecent,
MaxSnapshotLagBlocks: 64,
EnablePebbleMetrics: true,
AccountDBConfig: pebbledb.DefaultConfig(),
Expand Down
Loading