Skip to content

Commit 8f17625

Browse files
seidroid[bot]alexander-seicursoragentmasih
authored
Backport release/v6.7: Hard-code the state sync trust period per chain (#4690)
Backport of #4679 to `release/v6.7`. --------- Co-authored-by: alexander-sei <alexanderh@seinetwork.io> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Masih H. Derkani <m@derkani.org>
1 parent 02df238 commit 8f17625

15 files changed

Lines changed: 186 additions & 81 deletions

File tree

‎.github/workflows/rocksdb-unit_tests.yml‎

Lines changed: 9 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -11,54 +11,30 @@ concurrency:
1111
cancel-in-progress: true
1212
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
1313

14-
# TODO: we should combine these tests into go-test at some point. For now this is complicated enough to approach separately.
14+
# TODO: we should combine these tests into go-test at some point.
1515

1616
jobs:
1717
tests-rocksdb:
1818
name: RocksDB Tests
19-
runs-on: ubuntu-latest
19+
# Pinned for the RocksDB version: ubuntu-24.04's librocksdb-dev is 8.9.1.
20+
runs-on: ubuntu-24.04
2021
steps:
2122
- uses: actions/setup-go@v6
2223
with:
2324
go-version: '1.25.6'
2425
# See: https://github.com/actions/checkout/releases/tag/v7.0.0
2526
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
2627

27-
- name: Install RocksDB dependencies
28+
- name: Install RocksDB
2829
run: |
2930
sudo apt-get update
30-
sudo apt-get install -y build-essential pkg-config cmake git zlib1g-dev libbz2-dev libsnappy-dev liblz4-dev libzstd-dev libjemalloc-dev libgflags-dev liburing-dev
31-
32-
- name: Cache RocksDB
33-
id: cache-rocksdb
34-
uses: actions/cache@v3
35-
with:
36-
path: |
37-
/usr/local/lib/librocksdb.*
38-
/usr/local/include/rocksdb
39-
key: rocksdb-v8.9.1-${{ runner.os }}
40-
41-
- name: Build and Install RocksDB from source
42-
if: steps.cache-rocksdb.outputs.cache-hit != 'true'
43-
run: |
44-
git clone https://github.com/facebook/rocksdb.git
45-
cd rocksdb
46-
git checkout v8.9.1
47-
make clean
48-
CXXFLAGS='-march=native -DNDEBUG' make -j"$(nproc)" shared_lib
49-
sudo make install-shared
50-
cd ..
51-
52-
- name: Configure RocksDB library path
53-
run: |
54-
echo '/usr/local/lib' | sudo tee /etc/ld.so.conf.d/rocksdb.conf
55-
sudo ldconfig
31+
# 8.9.x is the RocksDB the grocksdb v1.8.x binding in go.mod targets;
32+
# the glob takes Ubuntu's patch revisions but fails on a version change.
33+
sudo apt-get install -y 'librocksdb-dev=8.9.1-*'
5634
5735
- name: Run RocksDB Tests
58-
run: |
59-
export CGO_CFLAGS="-I/usr/local/include"
60-
export CGO_LDFLAGS="-L/usr/local/lib -lrocksdb -lz -lbz2 -lsnappy -llz4 -lzstd -ljemalloc"
61-
go test -v -mod=readonly -tags=rocksdbBackend ./sei-db/db_engine/rocksdb/... -covermode=atomic -coverprofile=./rdb-profile.out
36+
run: go test -v -mod=readonly -tags=rocksdbBackend ./sei-db/db_engine/rocksdb/... -covermode=atomic -coverprofile=./rdb-profile.out
37+
6238
- name: Upload coverage to Codecov
6339
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
6440
with:
@@ -68,4 +44,3 @@ jobs:
6844
flags: sei-db
6945
disable_search: true
7046
name: sei-db-coverage
71-

‎docker/localnode/config/config.toml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -314,14 +314,10 @@ enable = false
314314

315315
# RPC servers (comma-separated) for light client verification of the synced state machine and
316316
# retrieval of state data for node bootstrapping. Also needs a trusted height and corresponding
317-
# header hash obtained from a trusted source, and a period during which validators can be trusted.
318-
#
319-
# For Cosmos SDK-based chains, trust_period should usually be about 2/3 of the unbonding time (~2
320-
# weeks) during which they can be financially punished (slashed) for misbehavior.
317+
# header hash obtained from a trusted source.
321318
rpc-servers = ""
322319
trust-height = 0
323320
trust-hash = ""
324-
trust-period = "168h0m0s"
325321

326322
# Time to spend discovering snapshots before initiating a restore.
327323
discovery-time = "15s"

‎docker/rpcnode/config/config.toml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -314,14 +314,10 @@ enable = false
314314

315315
# RPC servers (comma-separated) for light client verification of the synced state machine and
316316
# retrieval of state data for node bootstrapping. Also needs a trusted height and corresponding
317-
# header hash obtained from a trusted source, and a period during which validators can be trusted.
318-
#
319-
# For Cosmos SDK-based chains, trust_period should usually be about 2/3 of the unbonding time (~2
320-
# weeks) during which they can be financially punished (slashed) for misbehavior.
317+
# header hash obtained from a trusted source.
321318
rpc-servers = ""
322319
trust-height = 0
323320
trust-hash = ""
324-
trust-period = "168h0m0s"
325321

326322
# Time to spend discovering snapshots before initiating a restore.
327323
discovery-time = "15s"

‎sei-cosmos/server/start.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,12 @@ import (
2929
genesistypes "github.com/sei-protocol/sei-chain/sei-cosmos/types/genesis"
3030
"github.com/sei-protocol/sei-chain/sei-cosmos/utils/tracing"
3131
tcmd "github.com/sei-protocol/sei-chain/sei-tendermint/cmd/tendermint/commands"
32+
tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config"
3233
"github.com/sei-protocol/sei-chain/sei-tendermint/node"
3334
"github.com/sei-protocol/sei-chain/sei-tendermint/rpc/client/local"
3435
tmtypes "github.com/sei-protocol/sei-chain/sei-tendermint/types"
3536
"github.com/spf13/cobra"
37+
"github.com/spf13/viper"
3638
"go.opentelemetry.io/otel/sdk/trace"
3739
)
3840

@@ -322,6 +324,7 @@ func startInProcess(
322324
"This defaults to 0 in the current version, but will error in the next version " +
323325
"(SDK v0.45). Please explicitly put the desired minimum-gas-prices in your app.toml.")
324326
}
327+
warnIgnoredStateSyncTrustPeriod(ctx.Viper, clientCtx.ChainID)
325328
app := appCreator(nil, traceWriter, ctx.Config, ctx.Viper)
326329
defer func() {
327330
logger.Info("close any other open resource...")
@@ -486,3 +489,15 @@ func startInProcess(
486489
// wait for signal capture and gracefully return
487490
return WaitForQuitSignals(goCtx, restartCh)
488491
}
492+
493+
// warnIgnoredStateSyncTrustPeriod logs a warning when v sets [statesync] trust-period to a
494+
// value other than the trust period state sync uses on chainID.
495+
func warnIgnoredStateSyncTrustPeriod(v *viper.Viper, chainID string) {
496+
const key = "statesync.trust-period"
497+
trustPeriod := tmcfg.StateSyncTrustPeriod(chainID)
498+
if !v.IsSet(key) || v.GetDuration(key) == trustPeriod {
499+
return
500+
}
501+
logger.Warn("statesync.trust-period is set in config.toml, but state sync uses the trust period built into seid for this chain; the setting is ignored and can be removed",
502+
"chainID", chainID, "configured", v.GetString(key), "trustPeriod", trustPeriod.String())
503+
}

‎sei-cosmos/server/start_test.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,18 @@
11
package server
22

33
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"log/slog"
8+
"strings"
49
"testing"
510

11+
"github.com/spf13/viper"
612
"github.com/stretchr/testify/require"
713

814
"github.com/sei-protocol/sei-chain/sei-cosmos/telemetry"
15+
tmcfg "github.com/sei-protocol/sei-chain/sei-tendermint/config"
916
)
1017

1118
func TestInjectTelemetryChainID(t *testing.T) {
@@ -45,3 +52,46 @@ func TestInjectTelemetryChainID(t *testing.T) {
4552
require.Equal(t, cfg.GlobalLabels, got.GlobalLabels)
4653
})
4754
}
55+
56+
func TestWarnIgnoredStateSyncTrustPeriod(t *testing.T) {
57+
for name, tc := range map[string]struct {
58+
chainID string
59+
trustPeriod string // config.toml value; empty leaves the key out
60+
wantWarning bool
61+
}{
62+
"unset": {chainID: "pacific-1"},
63+
"matches the chain's period": {chainID: "pacific-1", trustPeriod: "112h"},
64+
"old default on pacific-1": {chainID: "pacific-1", trustPeriod: "168h0m0s", wantWarning: true},
65+
"old default on an unlisted chain": {chainID: "sei", trustPeriod: "168h0m0s"},
66+
"override on an unlisted chain": {chainID: "sei", trustPeriod: "72h", wantWarning: true},
67+
"not a duration": {chainID: "sei", trustPeriod: "one week", wantWarning: true},
68+
} {
69+
t.Run(name, func(t *testing.T) {
70+
configToml := "[statesync]\n"
71+
if tc.trustPeriod != "" {
72+
configToml += fmt.Sprintf("trust-period = %q\n", tc.trustPeriod)
73+
}
74+
v := viper.New()
75+
v.SetConfigType("toml")
76+
require.NoError(t, v.ReadConfig(strings.NewReader(configToml)))
77+
78+
logs := &bytes.Buffer{}
79+
prev := logger
80+
logger = slog.New(slog.NewJSONHandler(logs, nil))
81+
t.Cleanup(func() { logger = prev })
82+
83+
warnIgnoredStateSyncTrustPeriod(v, tc.chainID)
84+
85+
if !tc.wantWarning {
86+
require.Empty(t, logs.String())
87+
return
88+
}
89+
var record map[string]any
90+
require.NoError(t, json.Unmarshal(logs.Bytes(), &record))
91+
require.Equal(t, slog.LevelWarn.String(), record[slog.LevelKey])
92+
require.Equal(t, tc.chainID, record["chainID"])
93+
require.Equal(t, tc.trustPeriod, record["configured"])
94+
require.Equal(t, tmcfg.StateSyncTrustPeriod(tc.chainID).String(), record["trustPeriod"])
95+
})
96+
}
97+
}

‎sei-db/db_engine/rocksdb/mvcc/db.go‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,13 @@ type VersionedChangesets struct {
5252
}
5353

5454
type Database struct {
55-
storage *grocksdb.DB
56-
config config.StateStoreConfig
55+
storage *grocksdb.DB
56+
config config.StateStoreConfig
57+
// cfHandle is the state_storage column family used for all reads and writes.
5758
cfHandle *grocksdb.ColumnFamilyHandle
59+
// defaultCFHandle is the unused default column family, held only so Close
60+
// can destroy it.
61+
defaultCFHandle *grocksdb.ColumnFamilyHandle
5862

5963
// tsLow reflects the full_history_ts_low CF value. Since pruning is done in
6064
// a lazy manner, we use this value to prevent reads for versions that will
@@ -78,12 +82,12 @@ type Database struct {
7882
func OpenDB(dataDir string, config config.StateStoreConfig) (*Database, error) {
7983
//TODO: add a new config and check if readonly = true to support readonly mode
8084

81-
storage, cfHandle, err := OpenRocksDB(dataDir)
85+
storage, cf, err := OpenRocksDB(dataDir)
8286
if err != nil {
8387
return nil, fmt.Errorf("failed to open RocksDB: %w", err)
8488
}
8589

86-
slice, err := storage.GetFullHistoryTsLow(cfHandle)
90+
slice, err := storage.GetFullHistoryTsLow(cf.StateStorage)
8791
if err != nil {
8892
return nil, fmt.Errorf("failed to get full_history_ts_low: %w", err)
8993
}
@@ -108,7 +112,8 @@ func OpenDB(dataDir string, config config.StateStoreConfig) (*Database, error) {
108112
database := &Database{
109113
storage: storage,
110114
config: config,
111-
cfHandle: cfHandle,
115+
cfHandle: cf.StateStorage,
116+
defaultCFHandle: cf.Default,
112117
tsLow: tsLow,
113118
earliestVersion: earliestVersion,
114119
latestVersion: atomic.Int64{},
@@ -505,7 +510,14 @@ func (db *Database) Close() error {
505510
// Only set to nil after background goroutine has finished
506511
db.streamHandler = nil
507512
}
508-
db.cfHandle = nil
513+
if db.cfHandle != nil {
514+
db.cfHandle.Destroy()
515+
db.cfHandle = nil
516+
}
517+
if db.defaultCFHandle != nil {
518+
db.defaultCFHandle.Destroy()
519+
db.defaultCFHandle = nil
520+
}
509521
if db.storage != nil {
510522
db.storage.Close()
511523
db.storage = nil

‎sei-db/db_engine/rocksdb/mvcc/opts.go‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,19 @@ func NewRocksDBOpts(sstFileWriter bool) *grocksdb.Options {
6464
return opts
6565
}
6666

67-
// OpenRocksDB opens a RocksDB database connection for versioned reading and writing.
68-
// It also returns a column family handle for versioning using user-defined timestamps.
69-
// The default column family is used for metadata, specifically key/value pairs
70-
// that are stored on another column family named with "state_storage", which has
71-
// user-defined timestamp enabled.
72-
func OpenRocksDB(dataDir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
67+
// ColumnFamilies is the pair of column family handles opened alongside a
68+
// RocksDB database: the default column family, which RocksDB requires to be
69+
// opened but is never used, and state_storage, which holds versioned state
70+
// with user-defined timestamps. Every handle must be destroyed before the DB
71+
// closes or the ColumnFamilySet destructor asserts on the leftover reference.
72+
type ColumnFamilies struct {
73+
Default *grocksdb.ColumnFamilyHandle
74+
StateStorage *grocksdb.ColumnFamilyHandle
75+
}
76+
77+
// OpenRocksDB opens a RocksDB database connection for versioned reading and
78+
// writing along with handles to both column families.
79+
func OpenRocksDB(dataDir string) (*grocksdb.DB, ColumnFamilies, error) {
7380
opts := grocksdb.NewDefaultOptions()
7481
opts.SetCreateIfMissing(true)
7582
opts.SetCreateIfMissingColumnFamilies(true)
@@ -87,16 +94,16 @@ func OpenRocksDB(dataDir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, er
8794
},
8895
)
8996
if err != nil {
90-
return nil, nil, err
97+
return nil, ColumnFamilies{}, err
9198
}
9299

93-
return db, cfHandles[1], nil
100+
return db, ColumnFamilies{Default: cfHandles[0], StateStorage: cfHandles[1]}, nil
94101
}
95102

96103
// OpenRocksDBAndTrimHistory opens a RocksDB handle similar to `OpenRocksDB`,
97104
// but it also trims the versions newer than target one, such that it can be used
98105
// for rollback.
99-
func OpenRocksDBAndTrimHistory(dataDir string, version int64) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
106+
func OpenRocksDBAndTrimHistory(dataDir string, version int64) (*grocksdb.DB, ColumnFamilies, error) {
100107
var ts [TimestampSize]byte
101108
binary.LittleEndian.PutUint64(ts[:], uint64(version))
102109

@@ -118,8 +125,8 @@ func OpenRocksDBAndTrimHistory(dataDir string, version int64) (*grocksdb.DB, *gr
118125
ts[:],
119126
)
120127
if err != nil {
121-
return nil, nil, err
128+
return nil, ColumnFamilies{}, err
122129
}
123130

124-
return db, cfHandles[1], nil
131+
return db, ColumnFamilies{Default: cfHandles[0], StateStorage: cfHandles[1]}, nil
125132
}

‎sei-db/db_engine/test/storage_test_suite.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,7 @@ func (s *StorageTestSuite) TestDatabasePrune() {
711711
itr, err := db.Iterator(storeKey1, 25, []byte("key000"), nil)
712712
s.Require().NoError(err)
713713
s.Require().False(itr.Valid())
714+
s.Require().NoError(itr.Close())
714715

715716
// prune the latest version which should prune the entire dataset
716717
s.Require().NoError(db.Prune(50))

‎sei-tendermint/config/config.go‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1067,6 +1067,28 @@ func (cfg *MempoolConfig) ValidateBasic() error {
10671067
//-----------------------------------------------------------------------------
10681068
// StateSyncConfig
10691069

1070+
// stateSyncTrustPeriods maps a well-known chain ID to its state sync trust period.
1071+
var stateSyncTrustPeriods = map[string]time.Duration{
1072+
// A trust period plus the time needed to check headers and to report and
1073+
// punish misbehavior must stay below the network's unbonding time.
1074+
"pacific-1": 112 * time.Hour,
1075+
"atlantic-2": 112 * time.Hour,
1076+
"arctic-1": 112 * time.Hour,
1077+
}
1078+
1079+
// defaultStateSyncTrustPeriod is the state sync trust period of a chain that
1080+
// stateSyncTrustPeriods does not list.
1081+
const defaultStateSyncTrustPeriod = 168 * time.Hour
1082+
1083+
// StateSyncTrustPeriod returns how long after its time a header of chainID can
1084+
// still serve as the trusted root state sync verifies light blocks from.
1085+
func StateSyncTrustPeriod(chainID string) time.Duration {
1086+
if trustPeriod, ok := stateSyncTrustPeriods[chainID]; ok {
1087+
return trustPeriod
1088+
}
1089+
return defaultStateSyncTrustPeriod
1090+
}
1091+
10701092
// StateSyncConfig defines the configuration for the Tendermint state sync service
10711093
type StateSyncConfig struct {
10721094
// State sync rapidly bootstraps a new node by discovering, fetching, and restoring a
@@ -1086,15 +1108,11 @@ type StateSyncConfig struct {
10861108
// with net.Dial, for example: "host.example.com:2125".
10871109
RPCServers []string `mapstructure:"rpc-servers"`
10881110

1089-
// The hash and height of a trusted block. Must be within the trust-period.
1111+
// The hash and height of a trusted block. Must be within the trust period
1112+
// StateSyncTrustPeriod returns for the chain.
10901113
TrustHeight int64 `mapstructure:"trust-height"`
10911114
TrustHash string `mapstructure:"trust-hash"`
10921115

1093-
// The trust period should be set so that Tendermint can detect and gossip
1094-
// misbehavior before it is considered expired. For chains based on the Cosmos SDK,
1095-
// one day less than the unbonding period should suffice.
1096-
TrustPeriod time.Duration `mapstructure:"trust-period"`
1097-
10981116
// Backfill sequentially fetches, verifies and stores light blocks in reverse order.
10991117
// It does not stop verifying blocks until reaching a height
11001118
// that is less or equal to the latestBlock - backfill-blocks and latest block time - backfill-duration.
@@ -1145,7 +1163,6 @@ func (cfg *StateSyncConfig) TrustHashBytes() []byte {
11451163
// DefaultStateSyncConfig returns a default configuration for the state sync service
11461164
func DefaultStateSyncConfig() *StateSyncConfig {
11471165
return &StateSyncConfig{
1148-
TrustPeriod: 168 * time.Hour,
11491166
DiscoveryTime: 15 * time.Second,
11501167
ChunkRequestTimeout: 15 * time.Second,
11511168
Fetchers: 2,
@@ -1186,10 +1203,6 @@ func (cfg *StateSyncConfig) ValidateBasic() error {
11861203
return errors.New("discovery time must be 0s or greater than five seconds")
11871204
}
11881205

1189-
if cfg.TrustPeriod <= 0 {
1190-
return errors.New("trusted-period is required")
1191-
}
1192-
11931206
if cfg.TrustHeight <= 0 {
11941207
return errors.New("trusted-height is required")
11951208
}

0 commit comments

Comments
 (0)