Skip to content

Commit 95fb2a3

Browse files
authored
Run RocksDB CI tests against the Ubuntu librocksdb package (#4446)
The sei-db workflow rebuilt RocksDB 8.9.1 from source on every cache miss, threaded the result through a custom cache key, `ldconfig`, and hand-rolled `CGO_CFLAGS`/`CGO_LDFLAGS`, all to run a sub-second test suite. Ubuntu 24.04's universe repo ships `librocksdb-dev` 8.9.1-2 — the same upstream version the job pinned — so the workflow now installs it with `apt-get` and drops the clone, `make shared_lib`, cache, `ldconfig`, and CGO plumbing. The runner is pinned to `ubuntu-24.04` so the distro's RocksDB version stays deterministic when `ubuntu-latest` rolls forward. One wrinkle: the distro build keeps assertions enabled where the CI source build set `-DNDEBUG`, and it catches a real handle leak. `grocksdb.OpenDbColumnFamilies` returns handles for both the `default` column family (which RocksDB requires to be opened) and `state_storage`, but only `state_storage` was kept; the default handle's reference survived until `rocksdb_close`, which trips the `ColumnFamilySet` `last_ref` assertion. `OpenRocksDB`/`OpenRocksDBAndTrimHistory` now return a `ColumnFamilies` struct naming both handles, and `Database.Close` destroys them before closing the DB. Validated by running the full `db_engine/rocksdb` suite in an `ubuntu:24.04` container against the apt package: it passes, and each run is faster — ~10s of apt instead of a ~20-minute source build on cache miss, with no cache or library-path setup on hit.
1 parent 1d8e5f7 commit 95fb2a3

3 files changed

Lines changed: 45 additions & 51 deletions

File tree

‎.github/workflows/rocksdb-unit_tests.yml‎

Lines changed: 9 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -11,55 +11,31 @@ concurrency:
1111
cancel-in-progress: true
1212
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
1313

14-
# TODO: we should combine these tests into go-test at some point. For now this is complicated enough to approach separately.
14+
# TODO: we should combine these tests into go-test at some point.
1515

1616
jobs:
1717
tests-rocksdb:
1818
name: RocksDB Tests
19-
runs-on: ubuntu-latest
19+
# Pinned for the RocksDB version: ubuntu-24.04's librocksdb-dev is 8.9.1.
20+
runs-on: ubuntu-24.04
2021
steps:
2122
- uses: actions/setup-go@v7
2223
with:
2324
go-version: '1.27.1'
2425
# See: https://github.com/actions/checkout/releases/tag/v7.0.0
2526
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
2627

27-
- name: Install RocksDB dependencies
28+
- name: Install RocksDB
2829
run: |
2930
bash .github/scripts/ci-drop-third-party-apt-sources.sh
3031
sudo apt-get update
31-
sudo apt-get install -y build-essential pkg-config cmake git zlib1g-dev libbz2-dev libsnappy-dev liblz4-dev libzstd-dev libjemalloc-dev libgflags-dev liburing-dev
32-
33-
- name: Cache RocksDB
34-
id: cache-rocksdb
35-
uses: actions/cache@v3
36-
with:
37-
path: |
38-
/usr/local/lib/librocksdb.*
39-
/usr/local/include/rocksdb
40-
key: rocksdb-v8.9.1-${{ runner.os }}
41-
42-
- name: Build and Install RocksDB from source
43-
if: steps.cache-rocksdb.outputs.cache-hit != 'true'
44-
run: |
45-
git clone https://github.com/facebook/rocksdb.git
46-
cd rocksdb
47-
git checkout v8.9.1
48-
make clean
49-
CXXFLAGS='-march=native -DNDEBUG' make -j"$(nproc)" shared_lib
50-
sudo make install-shared
51-
cd ..
52-
53-
- name: Configure RocksDB library path
54-
run: |
55-
echo '/usr/local/lib' | sudo tee /etc/ld.so.conf.d/rocksdb.conf
56-
sudo ldconfig
32+
# 8.9.x is the RocksDB the grocksdb v1.8.x binding in go.mod targets;
33+
# the glob takes Ubuntu's patch revisions but fails on a version change.
34+
sudo apt-get install -y 'librocksdb-dev=8.9.1-*'
5735
5836
- name: Run RocksDB Tests
59-
run: |
60-
export CGO_CFLAGS="-I/usr/local/include"
61-
export CGO_LDFLAGS="-L/usr/local/lib -lrocksdb -lz -lbz2 -lsnappy -llz4 -lzstd -ljemalloc"
62-
go test -v -mod=readonly -tags=rocksdbBackend ./sei-db/db_engine/rocksdb/... -covermode=atomic -coverprofile=./rdb-profile.out
37+
run: go test -v -mod=readonly -tags=rocksdbBackend ./sei-db/db_engine/rocksdb/... -covermode=atomic -coverprofile=./rdb-profile.out
38+
6339
- name: Upload coverage to Codecov
6440
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
6541
with:
@@ -69,4 +45,3 @@ jobs:
6945
flags: sei-db
7046
disable_search: true
7147
name: sei-db-coverage
72-

‎sei-db/db_engine/rocksdb/mvcc/db.go‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,13 @@ type VersionedChangesets struct {
5252
}
5353

5454
type Database struct {
55-
storage *grocksdb.DB
56-
config config.StateStoreConfig
55+
storage *grocksdb.DB
56+
config config.StateStoreConfig
57+
// cfHandle is the state_storage column family used for all reads and writes.
5758
cfHandle *grocksdb.ColumnFamilyHandle
59+
// defaultCFHandle is the unused default column family, held only so Close
60+
// can destroy it.
61+
defaultCFHandle *grocksdb.ColumnFamilyHandle
5862

5963
// tsLow reflects the full_history_ts_low CF value. Since pruning is done in
6064
// a lazy manner, we use this value to prevent reads for versions that will
@@ -78,12 +82,12 @@ type Database struct {
7882
func OpenDB(dataDir string, config config.StateStoreConfig) (*Database, error) {
7983
//TODO: add a new config and check if readonly = true to support readonly mode
8084

81-
storage, cfHandle, err := OpenRocksDB(dataDir)
85+
storage, cf, err := OpenRocksDB(dataDir)
8286
if err != nil {
8387
return nil, fmt.Errorf("failed to open RocksDB: %w", err)
8488
}
8589

86-
slice, err := storage.GetFullHistoryTsLow(cfHandle)
90+
slice, err := storage.GetFullHistoryTsLow(cf.StateStorage)
8791
if err != nil {
8892
return nil, fmt.Errorf("failed to get full_history_ts_low: %w", err)
8993
}
@@ -108,7 +112,8 @@ func OpenDB(dataDir string, config config.StateStoreConfig) (*Database, error) {
108112
database := &Database{
109113
storage: storage,
110114
config: config,
111-
cfHandle: cfHandle,
115+
cfHandle: cf.StateStorage,
116+
defaultCFHandle: cf.Default,
112117
tsLow: tsLow,
113118
earliestVersion: earliestVersion,
114119
latestVersion: atomic.Int64{},
@@ -505,7 +510,14 @@ func (db *Database) Close() error {
505510
// Only set to nil after background goroutine has finished
506511
db.streamHandler = nil
507512
}
508-
db.cfHandle = nil
513+
if db.cfHandle != nil {
514+
db.cfHandle.Destroy()
515+
db.cfHandle = nil
516+
}
517+
if db.defaultCFHandle != nil {
518+
db.defaultCFHandle.Destroy()
519+
db.defaultCFHandle = nil
520+
}
509521
if db.storage != nil {
510522
db.storage.Close()
511523
db.storage = nil

‎sei-db/db_engine/rocksdb/mvcc/opts.go‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,19 @@ func NewRocksDBOpts(sstFileWriter bool) *grocksdb.Options {
6464
return opts
6565
}
6666

67-
// OpenRocksDB opens a RocksDB database connection for versioned reading and writing.
68-
// It also returns a column family handle for versioning using user-defined timestamps.
69-
// The default column family is used for metadata, specifically key/value pairs
70-
// that are stored on another column family named with "state_storage", which has
71-
// user-defined timestamp enabled.
72-
func OpenRocksDB(dataDir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
67+
// ColumnFamilies is the pair of column family handles opened alongside a
68+
// RocksDB database: the default column family, which RocksDB requires to be
69+
// opened but is never used, and state_storage, which holds versioned state
70+
// with user-defined timestamps. Every handle must be destroyed before the DB
71+
// closes or the ColumnFamilySet destructor asserts on the leftover reference.
72+
type ColumnFamilies struct {
73+
Default *grocksdb.ColumnFamilyHandle
74+
StateStorage *grocksdb.ColumnFamilyHandle
75+
}
76+
77+
// OpenRocksDB opens a RocksDB database connection for versioned reading and
78+
// writing along with handles to both column families.
79+
func OpenRocksDB(dataDir string) (*grocksdb.DB, ColumnFamilies, error) {
7380
opts := grocksdb.NewDefaultOptions()
7481
opts.SetCreateIfMissing(true)
7582
opts.SetCreateIfMissingColumnFamilies(true)
@@ -87,16 +94,16 @@ func OpenRocksDB(dataDir string) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, er
8794
},
8895
)
8996
if err != nil {
90-
return nil, nil, err
97+
return nil, ColumnFamilies{}, err
9198
}
9299

93-
return db, cfHandles[1], nil
100+
return db, ColumnFamilies{Default: cfHandles[0], StateStorage: cfHandles[1]}, nil
94101
}
95102

96103
// OpenRocksDBAndTrimHistory opens a RocksDB handle similar to `OpenRocksDB`,
97104
// but it also trims the versions newer than target one, such that it can be used
98105
// for rollback.
99-
func OpenRocksDBAndTrimHistory(dataDir string, version int64) (*grocksdb.DB, *grocksdb.ColumnFamilyHandle, error) {
106+
func OpenRocksDBAndTrimHistory(dataDir string, version int64) (*grocksdb.DB, ColumnFamilies, error) {
100107
var ts [TimestampSize]byte
101108
binary.LittleEndian.PutUint64(ts[:], uint64(version))
102109

@@ -118,8 +125,8 @@ func OpenRocksDBAndTrimHistory(dataDir string, version int64) (*grocksdb.DB, *gr
118125
ts[:],
119126
)
120127
if err != nil {
121-
return nil, nil, err
128+
return nil, ColumnFamilies{}, err
122129
}
123130

124-
return db, cfHandles[1], nil
131+
return db, ColumnFamilies{Default: cfHandles[0], StateStorage: cfHandles[1]}, nil
125132
}

0 commit comments

Comments
 (0)