Skip to content

Commit 3118674

Browse files
authored
Fix misc undo log issues (#4571)
* This PR fixes misc ss undo log issues. * An empty block (no changes) crashed the node because an empty StateChangeSet was nil instead of empty. Fixed. * InitChain was hanging when the initial height was > 1. Fixed. * If an RPC still had a history view open at shutdown, the undo log refused to close and was never retried, so it lost its queued blocks and with them all served history. Fixed. * Remove redundant lock earliestMu, already serialized by another lock.
1 parent 884bfe9 commit 3118674

7 files changed

Lines changed: 295 additions & 147 deletions

File tree

‎sei-db/db_engine/pebbledb/undo/db.go‎

Lines changed: 41 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -109,16 +109,15 @@ type Database struct {
109109
earliestHeight atomic.Uint64
110110
openViews map[uint64]int
111111

112-
// earliestMu serializes the check, durable write, and publication of earliestHeight.
113-
earliestMu sync.Mutex
114-
115-
// lifecycleMu excludes Resume and Close from queue submissions, pruning and opening views,
116-
// and guards closed.
112+
// lifecycleMu excludes Resume and Close from queue submissions, pruning, opening views and
113+
// reads, and guards closed.
117114
lifecycleMu sync.RWMutex
118115
closed bool
119116

120-
// pruneMu serializes prune passes and guards prunedBucket, the lowest bucket not yet excised.
121-
pruneMu sync.Mutex
117+
// historyMu serializes every change to the served history: a resume, a prune pass and a
118+
// rollback, each of which moves earliestHeight. It guards prunedBucket, the lowest bucket not
119+
// yet excised.
120+
historyMu sync.Mutex
122121
prunedBucket uint64
123122

124123
pending chan pendingBlock
@@ -306,11 +305,18 @@ func (db *Database) Resume(version int64, current LiveStateView) error {
306305
return nil
307306
}
308307

308+
// Resumed reports whether a Resume has succeeded and the store is not closed.
309+
func (db *Database) Resumed() bool {
310+
return db.head.Load() != nil
311+
}
312+
309313
func (db *Database) resume(version int64) (uint64, error) {
310314
height, err := heightOf(version)
311315
if err != nil {
312316
return 0, err
313317
}
318+
db.historyMu.Lock()
319+
defer db.historyMu.Unlock()
314320
latest := db.latestHeight.Load()
315321
if latest > height {
316322
return 0, fmt.Errorf("undo: log at %d is ahead of current state at %d", latest, height)
@@ -542,7 +548,7 @@ func (w *recordWriter) finish() error {
542548
}
543549

544550
// head is a current-state view and the height it describes. The store holds one reference and
545-
// every view pinning it holds another; the last one released closes the view.
551+
// every read in flight holds another; the last one released closes the view.
546552
type head struct {
547553
height uint64
548554
view LiveStateView
@@ -583,12 +589,10 @@ func (db *Database) acquireHead() *head {
583589
}
584590
}
585591

586-
// View reads the store at one height. It pins the current-state view it falls back to and the
587-
// buckets it reads, holding back both the state commit store and pruning, so it must be closed
588-
// promptly.
592+
// View reads the store at one height. It pins the buckets it reads, holding back pruning, so it must
593+
// be closed promptly. Its reads fail once the database is closed.
589594
type View struct {
590595
db *Database
591-
head *head
592596
height uint64
593597
closeOnce sync.Once
594598

@@ -597,7 +601,7 @@ type View struct {
597601
}
598602

599603
// OpenView returns a view of the state after block version, and false when the store is closed or
600-
// does not serve version. The caller must close the view before closing the database.
604+
// does not serve version.
601605
func (db *Database) OpenView(version int64) (*View, bool) {
602606
db.lifecycleMu.RLock()
603607
defer db.lifecycleMu.RUnlock()
@@ -608,15 +612,10 @@ func (db *Database) OpenView(version int64) (*View, bool) {
608612
if err != nil {
609613
return nil, false
610614
}
611-
h := db.acquireHead()
612-
if h == nil {
615+
if h := db.head.Load(); h == nil || height > h.height || !db.pinHeight(height) {
613616
return nil, false
614617
}
615-
if height > h.height || !db.pinHeight(height) {
616-
h.release()
617-
return nil, false
618-
}
619-
v := &View{db: db, head: h, height: height}
618+
v := &View{db: db, height: height}
620619
v.closed = utils.MustClose(v, "undo view")
621620
return v, true
622621
}
@@ -641,30 +640,36 @@ func (db *Database) unpinHeight(height uint64) {
641640
}
642641
}
643642

644-
// Close releases the view's pins. Idempotent.
643+
// Close releases the view's pin. Idempotent.
645644
func (v *View) Close() {
646645
v.closeOnce.Do(func() {
647646
v.closed.Close(v)
648-
v.head.release()
649-
// Keep the view registered until its live-state reference has been released.
650647
v.db.unpinHeight(v.height)
651648
})
652649
}
653650

654651
// Get returns a supported EVM key's value at the view's height, nil when it did not exist.
655-
// Unsupported or malformed keys return an error.
652+
// Unsupported or malformed keys, and reads after the database is closed, return an error.
656653
func (v *View) Get(key []byte) (_ []byte, err error) {
657654
keyLen, _ := keyLayout(key)
658655
if keyLen == 0 || len(key) != keyLen {
659656
return nil, fmt.Errorf("undo: unsupported or malformed EVM key %x", key)
660657
}
658+
v.db.lifecycleMu.RLock()
659+
defer v.db.lifecycleMu.RUnlock()
660+
if v.db.closed {
661+
return nil, errors.New("undo: read on a closed store")
662+
}
661663
start := time.Now()
662664
probes := 0
663665
defer func() {
664666
otelMetrics.getLatency.Record(context.Background(), time.Since(start).Seconds(), successAttr(err))
665667
otelMetrics.getProbes.Record(context.Background(), int64(probes))
666668
}()
667-
h := v.head
669+
// The newest head answers a read at any height below it the same way, since every record up to
670+
// it is written before it is published.
671+
h := v.db.acquireHead()
672+
defer h.release()
668673
if v.height == h.height {
669674
return foundValue(h.view.Get(module, key)), nil
670675
}
@@ -714,29 +719,27 @@ func (db *Database) GetEarliestVersion() int64 {
714719
return int64(db.earliestHeight.Load()) //nolint:gosec // heights enter the store as int64
715720
}
716721

717-
// raiseEarliest makes height the lowest served height when it is above the current one.
722+
// raiseEarliest makes height the lowest served height when it is above the current one. The caller
723+
// must hold historyMu.
718724
func (db *Database) raiseEarliest(height uint64) error {
719-
db.earliestMu.Lock()
720-
defer db.earliestMu.Unlock()
721725
if height <= db.earliestHeight.Load() {
722726
return nil
723727
}
724728
return db.setEarliestLocked(height)
725729
}
726730

727731
// lowerEarliest makes height the lowest served height when it is below the current one. A rollback
728-
// below the earliest height leaves no record above height, and a read at height needs none.
732+
// below the earliest height leaves no record above height, and a read at height needs none. The
733+
// caller must hold historyMu.
729734
func (db *Database) lowerEarliest(height uint64) error {
730-
db.earliestMu.Lock()
731-
defer db.earliestMu.Unlock()
732735
if height >= db.earliestHeight.Load() {
733736
return nil
734737
}
735738
return db.setEarliestLocked(height)
736739
}
737740

738741
// setEarliestLocked durably records height as the lowest served height, then publishes it.
739-
// The caller must hold earliestMu.
742+
// The caller must hold historyMu.
740743
func (db *Database) setEarliestLocked(height uint64) error {
741744
if err := writeMarker(db.storage, earliestVersionKey, height, pebble.Sync); err != nil {
742745
return fmt.Errorf("undo: record earliest height %d: %w", height, err)
@@ -777,8 +780,8 @@ func (db *Database) PruneHistory(blockNumber uint64) error {
777780
if blockNumber == 0 || head == 0 {
778781
return nil
779782
}
780-
db.pruneMu.Lock()
781-
defer db.pruneMu.Unlock()
783+
db.historyMu.Lock()
784+
defer db.historyMu.Unlock()
782785
if err := db.raiseEarliest(min(blockNumber, head)); err != nil {
783786
return err
784787
}
@@ -849,6 +852,8 @@ func DiscardStateAbove(dataDir string, cfg config.StateStoreConfig, target int64
849852
// lowers the earliest served height to it. Buckets above target's are excised and target's own
850853
// bucket is scanned.
851854
func (db *Database) discardAbove(target uint64) error {
855+
db.historyMu.Lock()
856+
defer db.historyMu.Unlock()
852857
latest := db.latestHeight.Load()
853858
if latest < target {
854859
return nil
@@ -924,20 +929,14 @@ func (db *Database) deleteRecordsAbove(bucket uint64, height uint64) error {
924929
return b.Commit(pebble.Sync)
925930
}
926931

927-
// Close drains the write queue, releases the current-state view and closes Pebble. If views are
928-
// still open, it leaves the database open and returns an error; close the views and retry. Idempotent.
932+
// Close drains the write queue, releases the current-state view and closes Pebble. Views still open
933+
// fail their reads from then on. Idempotent.
929934
func (db *Database) Close() error {
930935
db.lifecycleMu.Lock()
931936
defer db.lifecycleMu.Unlock()
932937
if db.closed {
933938
return nil
934939
}
935-
db.mu.Lock()
936-
hasOpenViews := len(db.openViews) != 0
937-
db.mu.Unlock()
938-
if hasOpenViews {
939-
return errors.New("undo: cannot close with open views")
940-
}
941940
db.closed = true
942941
close(db.pending)
943942
<-db.writerDone

0 commit comments

Comments
 (0)