Skip to content

Commit 22abcdd

Browse files
cody-littleyCody Littley
andauthored
giga hash logging (#4534)
Wires the hash logging code into the giga app hash builder. The pattern needed for giga is a lot simpler than what cosmos needed, so we can use a simplified hashlogger API (i.e. we don't need the code for asynchronously collecting hashes, since the app hash builder already performs this task for us). --------- Co-authored-by: Cody Littley <cody.littley@seinetwork.io>
1 parent 2e11071 commit 22abcdd

16 files changed

Lines changed: 951 additions & 406 deletions

‎giga/apphash/builder/apphash_builder.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,22 @@ type AppHashBuilder interface {
1919
// methods.
2020
ReportBlockHash(ctx context.Context, blockHeight uint64, blockHash [32]byte) error
2121

22-
// Reports the state hash after executing the block.
22+
// Reports the state hash after executing the block, along with debugging hashes of the state's parts.
2323
//
2424
// blockHeight must be one above the previous call's, except on the first call. Illegal input to this method
2525
// may not cause it to return an error immediately, since hash data may be ingested asynchronously. Illegal
2626
// input will, however, eventually cause the app hash builder to shut down and start returning errors from all
2727
// methods.
28-
ReportStateHash(ctx context.Context, blockHeight uint64, stateHash [32]byte) error
28+
ReportStateHash(
29+
ctx context.Context,
30+
blockHeight uint64,
31+
stateHash [32]byte,
32+
// Named hashes of the parts of the state, such as each flatKV database and module (see
33+
// hashlog.FlatKVComponentHashes()). They exist for debugging only: they are written to the hash
34+
// log beside the block's app hash, are never part of the app hash, and have no effect on it. May
35+
// be nil. Must not be mutated after the call.
36+
componentHashes map[string][]byte,
37+
) error
2938

3039
// Reports the Block Update Digest of the block.
3140
//

‎giga/apphash/builder/apphash_builder_config.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55

66
"github.com/sei-protocol/sei-chain/sei-db/seiwal"
7+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog"
78
)
89

910
// AppHashBuilderConfig configures a StandardAppHashBuilder.
@@ -20,6 +21,14 @@ type AppHashBuilderConfig struct {
2021
// Configures the WAL holding the app hashes. Its Path is the hash vault's directory, and its PermitGaps is
2122
// ignored. App hashes survive a power loss only if its FsyncOnFlush is true.
2223
HashVaultConfig seiwal.Config
24+
25+
// Whether the builder writes each app hash it builds to a hash log, as a debugging aid.
26+
HashLogEnabled bool
27+
28+
// Configures the hash log. Read only if HashLogEnabled. Its Path and Version have no default and must be set,
29+
// and its Path must not be shared with another hash logger. Only the fields a hashlog.HashLogWriter reads
30+
// apply.
31+
HashLogConfig hashlog.HashLoggerConfig
2332
}
2433

2534
// Returns an AppHashBuilderConfig holding every default. HashVaultConfig.Path has no default and must be set.
@@ -30,6 +39,8 @@ func DefaultAppHashBuilderConfig() AppHashBuilderConfig {
3039
InputBufferSize: 1024,
3140
MaxBatchSize: 1024,
3241
HashVaultConfig: *hashVault,
42+
HashLogEnabled: false,
43+
HashLogConfig: *hashlog.DefaultHashLoggerConfig("", ""),
3344
}
3445
}
3546

‎giga/apphash/builder/builder_requests.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ type reportRequest struct {
77
input hashInput
88
blockHeight uint64
99
value [32]byte
10+
11+
// The debugging hashes passed to ReportStateHash(), written to the hash log only. Nil for every other input.
12+
componentHashes map[string][]byte
1013
}
1114

1215
// Asks the builder goroutine to complete setup at the storage layer height blockHeight.

‎giga/apphash/builder/hash_input.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ type pendingBlock struct {
7777

7878
// Whether each input has been reported, indexed by hashInput.
7979
reported [hashInputCount]bool
80+
81+
// The debugging hashes reported with the state hash, written to the hash log only. May be nil.
82+
componentHashes map[string][]byte
8083
}
8184

8285
// Returns whether every input has been reported.

‎giga/apphash/builder/standard_apphash_builder.go‎

Lines changed: 112 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,21 @@ import (
1010

1111
"github.com/sei-protocol/sei-chain/giga/apphash"
1212
"github.com/sei-protocol/sei-chain/giga/apphash/vault"
13+
"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/hashlog"
1314
)
1415

1516
var logger = seilog.NewLogger("giga", "apphash", "builder")
1617

18+
// Hash log columns holding the fields of each app hash the builder builds.
19+
const (
20+
appHashColumn = "appHash"
21+
previousAppHashColumn = "previousAppHash"
22+
blockHashColumn = "blockHash"
23+
stateHashColumn = "stateHash"
24+
budColumn = "bud"
25+
receiptHashColumn = "receiptHash"
26+
)
27+
1728
var _ AppHashBuilder = (*StandardAppHashBuilder)(nil)
1829

1930
// StandardAppHashBuilder is an AppHashBuilder that stores app hashes in a StandardHashVault it owns, and closes the
@@ -29,6 +40,10 @@ type StandardAppHashBuilder struct {
2940
// Holds the app hash data of every block built. Closed by the builder goroutine as it exits.
3041
vault vault.HashVault
3142

43+
// Receives each app hash built, as a debugging aid. Nil if hash logging is disabled or has failed. Owned by the
44+
// builder goroutine, which closes it as it exits.
45+
hashLog *hashlog.HashLogWriter
46+
3247
// Requests for the builder goroutine, run in order. Each is one of the *Request types.
3348
requests chan any
3449

@@ -142,6 +157,7 @@ func NewStandardAppHashBuilder(
142157
chainID: chainID,
143158
gigaActivationHeight: gigaActivationHeight,
144159
vault: hashVault,
160+
hashLog: openHashLog(config),
145161
requests: make(chan any, config.InputBufferSize),
146162
maxBatchSize: config.MaxBatchSize,
147163
ctx: ctx,
@@ -165,8 +181,18 @@ func (b *StandardAppHashBuilder) ReportBlockHash(ctx context.Context, blockHeigh
165181
return nil
166182
}
167183

168-
func (b *StandardAppHashBuilder) ReportStateHash(ctx context.Context, blockHeight uint64, stateHash [32]byte) error {
169-
request := reportRequest{input: stateHashInput, blockHeight: blockHeight, value: stateHash}
184+
func (b *StandardAppHashBuilder) ReportStateHash(
185+
ctx context.Context,
186+
blockHeight uint64,
187+
stateHash [32]byte,
188+
componentHashes map[string][]byte,
189+
) error {
190+
request := reportRequest{
191+
input: stateHashInput,
192+
blockHeight: blockHeight,
193+
value: stateHash,
194+
componentHashes: componentHashes,
195+
}
170196
if err := b.submit(ctx, request); err != nil {
171197
return fmt.Errorf("failed to report the state hash of block %d: %w", blockHeight, err)
172198
}
@@ -253,6 +279,20 @@ func (b *StandardAppHashBuilder) Close() error {
253279
return nil
254280
}
255281

282+
// Opens the hash log if config enables one. Returns nil if it does not, or if the hash log fails to open, which
283+
// does not stop the builder since the hash log is a debugging aid.
284+
func openHashLog(config AppHashBuilderConfig) *hashlog.HashLogWriter {
285+
if !config.HashLogEnabled {
286+
return nil
287+
}
288+
hashLog, err := hashlog.NewHashLogWriter(&config.HashLogConfig)
289+
if err != nil {
290+
logger.Error("failed to open the app hash log; disabling hash logging", "err", err)
291+
return nil
292+
}
293+
return hashLog
294+
}
295+
256296
// Reads the builder's startup state from hashVault. Its head is the newest stored record if that is at or above
257297
// gigaActivationHeight, and the block before gigaActivationHeight otherwise. Errors if the stored records belong to
258298
// a chain other than chainID.
@@ -342,9 +382,9 @@ func awaitReply[T any](b *StandardAppHashBuilder, reply <-chan callResult[T]) (T
342382
}
343383
}
344384

345-
// The builder goroutine. Runs requests until Close() or a request fails, then closes the hash vault.
385+
// The builder goroutine. Runs requests until Close() or a request fails, then closes the hash vault and hash log.
346386
func (b *StandardAppHashBuilder) run() {
347-
defer b.closeVault()
387+
defer b.closeStores()
348388
for {
349389
select {
350390
case <-b.ctx.Done():
@@ -391,7 +431,7 @@ func (b *StandardAppHashBuilder) runBatch(first any) error {
391431
func (b *StandardAppHashBuilder) handleRequest(request any) error {
392432
switch r := request.(type) {
393433
case reportRequest:
394-
if err := b.handleReport(r.input, r.blockHeight, r.value); err != nil {
434+
if err := b.handleReport(r.input, r.blockHeight, r.value, r.componentHashes); err != nil {
395435
return fmt.Errorf("failed to take in the %s of block %d: %w", r.input, r.blockHeight, err)
396436
}
397437
return nil
@@ -418,9 +458,15 @@ func (b *StandardAppHashBuilder) handleRequest(request any) error {
418458
}
419459
}
420460

421-
// Takes in one report (i.e. one call to a Report*() method): an input's value for one block. A report below the
422-
// giga activation height is ignored. Before setup, a value for a stored block is checked against the stored record.
423-
func (b *StandardAppHashBuilder) handleReport(input hashInput, blockHeight uint64, value [32]byte) error {
461+
// Takes in one report (i.e. one call to a Report*() method): an input's value for one block, and the debugging
462+
// hashes reported with it. A report below the giga activation height is ignored. Before setup, a value for a
463+
// stored block is checked against the stored record.
464+
func (b *StandardAppHashBuilder) handleReport(
465+
input hashInput,
466+
blockHeight uint64,
467+
value [32]byte,
468+
componentHashes map[string][]byte,
469+
) error {
424470
if blockHeight < b.gigaActivationHeight {
425471
return nil
426472
}
@@ -446,6 +492,9 @@ func (b *StandardAppHashBuilder) handleReport(input hashInput, blockHeight uint6
446492
}
447493
block.values[input] = value
448494
block.reported[input] = true
495+
if componentHashes != nil {
496+
block.componentHashes = componentHashes
497+
}
449498
return nil
450499
}
451500

@@ -473,7 +522,7 @@ func (b *StandardAppHashBuilder) compareStoredInput(
473522
return fmt.Errorf("failed to read the stored record of block %d: %w", blockHeight, err)
474523
}
475524
if storedValue := input.of(stored); storedValue != value {
476-
panicOnMismatch(blockHeight, input.String(), storedValue, value)
525+
b.panicOnMismatch(blockHeight, input.String(), storedValue, value)
477526
}
478527
if blockHeight == b.vaultHighest {
479528
err := tracker.storedRecordIterator.Close()
@@ -551,7 +600,7 @@ func (b *StandardAppHashBuilder) storeSetupBlocks(
551600
return fmt.Errorf("cannot compute the app hash of block %d: not every input was reported",
552601
height)
553602
}
554-
record := b.newRecord(height, block)
603+
record := b.buildRecord(height, block)
555604
records = append(records, record)
556605
b.headHeight = height
557606
b.headAppHash = record.AppHash()
@@ -617,7 +666,7 @@ func (b *StandardAppHashBuilder) publishCompleteBlocks() error {
617666
break
618667
}
619668
delete(b.pendingBlocks, height)
620-
record := b.newRecord(height, block)
669+
record := b.buildRecord(height, block)
621670
if b.storedAboveHeadIterator != nil {
622671
if err := b.compareStoredRecord(record); err != nil {
623672
return fmt.Errorf("failed to check block %d against the hash vault: %w", height, err)
@@ -657,7 +706,7 @@ func (b *StandardAppHashBuilder) compareStoredRecord(record *apphash.AppHashData
657706
return fmt.Errorf("failed to read the stored record of block %d: %w", record.BlockHeight(), err)
658707
}
659708
if stored.AppHash() != record.AppHash() {
660-
panicOnMismatch(record.BlockHeight(), "app hash", stored.AppHash(), record.AppHash())
709+
b.panicOnMismatch(record.BlockHeight(), "app hash", stored.AppHash(), record.AppHash())
661710
}
662711
if record.BlockHeight() == b.vaultHighest {
663712
err := b.storedAboveHeadIterator.Close()
@@ -670,9 +719,9 @@ func (b *StandardAppHashBuilder) compareStoredRecord(record *apphash.AppHashData
670719
}
671720

672721
// Returns the app hash data of a block whose inputs are all reported (i.e. passed to the Report*() methods),
673-
// chained from the app hash of the block before it.
674-
func (b *StandardAppHashBuilder) newRecord(blockHeight uint64, block *pendingBlock) *apphash.AppHashData {
675-
return apphash.NewAppHashData(
722+
// chained from the app hash of the block before it, and writes it to the hash log.
723+
func (b *StandardAppHashBuilder) buildRecord(blockHeight uint64, block *pendingBlock) *apphash.AppHashData {
724+
record := apphash.NewAppHashData(
676725
b.chainID,
677726
blockHeight,
678727
block.values[blockHashInput],
@@ -681,6 +730,36 @@ func (b *StandardAppHashBuilder) newRecord(blockHeight uint64, block *pendingBlo
681730
block.values[receiptHashInput],
682731
b.headAppHash,
683732
)
733+
b.writeToHashLog(record, block.componentHashes)
734+
return record
735+
}
736+
737+
// Writes record's fields and componentHashes to the hash log as one row. The hash log is a debugging aid, so a
738+
// failure disables it rather than stopping the builder.
739+
func (b *StandardAppHashBuilder) writeToHashLog(record *apphash.AppHashData, componentHashes map[string][]byte) {
740+
if b.hashLog == nil {
741+
return
742+
}
743+
hashes := make(map[string][]byte, len(componentHashes)+6)
744+
for name, hash := range componentHashes {
745+
hashes[name] = hash
746+
}
747+
hashes[appHashColumn] = hashBytes(record.AppHash())
748+
hashes[previousAppHashColumn] = hashBytes(record.PreviousAppHash())
749+
hashes[blockHashColumn] = hashBytes(record.BlockHash())
750+
hashes[stateHashColumn] = hashBytes(record.StateHash())
751+
hashes[budColumn] = hashBytes(record.BUD())
752+
hashes[receiptHashColumn] = hashBytes(record.ReceiptHash())
753+
if err := b.hashLog.Write(record.BlockHeight(), hashes); err != nil {
754+
logger.Error("failed to write to the app hash log; disabling hash logging", "err", err)
755+
_ = b.hashLog.Close()
756+
b.hashLog = nil
757+
}
758+
}
759+
760+
// Returns a slice holding a copy of hash.
761+
func hashBytes(hash [32]byte) []byte {
762+
return hash[:]
684763
}
685764

686765
// Appends records to the hash vault, durably if HashVaultConfig.FsyncOnFlush is set, and updates the builder's
@@ -711,8 +790,15 @@ func (b *StandardAppHashBuilder) applyPrune() error {
711790
return nil
712791
}
713792

714-
// Closes the vault iterators the builder goroutine holds, then the hash vault, and records the error for Close().
715-
func (b *StandardAppHashBuilder) closeVault() {
793+
// Closes the vault iterators the builder goroutine holds, the hash log, and the hash vault, and records the error
794+
// for Close(). A hash log failure is logged rather than recorded, since the hash log is a debugging aid.
795+
func (b *StandardAppHashBuilder) closeStores() {
796+
if b.hashLog != nil {
797+
if err := b.hashLog.Close(); err != nil {
798+
logger.Error("failed to close the app hash log", "err", err)
799+
}
800+
b.hashLog = nil
801+
}
716802
b.closeErr = errors.Join(b.closeIterators(), b.vault.Close())
717803
}
718804

@@ -753,14 +839,21 @@ func nextStoredRecord(it apphash.AppHashIterator, blockHeight uint64) (*apphash.
753839
return record, nil
754840
}
755841

756-
// Crashes the node because a value computed for blockHeight differs from the one it stored earlier.
757-
func panicOnMismatch(blockHeight uint64, what string, stored [32]byte, computed [32]byte) {
842+
// Crashes the node because a value computed for blockHeight differs from the one it stored earlier, after flushing
843+
// the hash log so that the rows written so far, the computed app hash among them when it is the one that changed,
844+
// survive the crash as evidence.
845+
func (b *StandardAppHashBuilder) panicOnMismatch(blockHeight uint64, what string, stored [32]byte, computed [32]byte) {
758846
logger.Error("App hash builder detected a changed hash; the node attempted to change its mind about a block "+
759847
"it already committed to. DO NOT RESTART WITHOUT HUMAN INVESTIGATION.",
760848
"blockHeight", blockHeight,
761849
"value", what,
762850
"stored", fmt.Sprintf("%x", stored),
763851
"computed", fmt.Sprintf("%x", computed),
764852
)
853+
if b.hashLog != nil {
854+
if err := b.hashLog.Flush(); err != nil {
855+
logger.Error("failed to flush the app hash log", "err", err)
856+
}
857+
}
765858
panic(fmt.Sprintf("%s of block %d changed: stored %x, computed %x", what, blockHeight, stored, computed))
766859
}

0 commit comments

Comments
 (0)