diff --git a/consensus/late_signature.go b/consensus/late_signature.go index 796f0824c1..c5dfe394cd 100644 --- a/consensus/late_signature.go +++ b/consensus/late_signature.go @@ -42,12 +42,34 @@ func (consensus *Consensus) checkOwnCommitInclusion(blockNum uint64, blockHash c localPubs = append(localPubs, key.Pub.Bytes) } - for _, pub := range excludedLocalCommitKeys(mask, localPubs) { + reportLocalCommitInclusions(mask, localPubs, func(pub bls.SerializedPublicKey) { consensus.getLogger().Warn(). Uint64("blockNum", blockNum). Str("blockHash", blockHash.Hex()). Str("blsPubKey", pub.Hex()). Msg("[OnCommitted] local commit signature not included in final commit bitmap") + }) +} + +// reportLocalCommitInclusions counts each local committee key present in the +// COMMITTED participant set toward signature_total, and late_signature when +// the key is disabled in the bitmap. +func reportLocalCommitInclusions(mask *bls.Mask, localPubs []bls.SerializedPublicKey, onMissing func(bls.SerializedPublicKey)) { + for _, pub := range localPubs { + ok, err := mask.KeyEnabled(pub) + if err != nil { + continue + } + consensusSignatureTotalCounterVec.With(prometheus.Labels{ + "role": "validator", + "phase": "committed", + }).Inc() + if ok { + continue + } + if onMissing != nil { + onMissing(pub) + } consensusLateSignatureCounterVec.With(prometheus.Labels{ "role": "validator", "phase": "committed", @@ -71,6 +93,14 @@ func excludedLocalCommitKeys(mask *bls.Mask, localPubs []bls.SerializedPublicKey return excluded } +// reportAcceptedVote counts a prepare/commit vote accepted on time by the leader. +func (consensus *Consensus) reportAcceptedVote(phase string) { + consensusSignatureTotalCounterVec.With(prometheus.Labels{ + "role": "leader", + "phase": phase, + }).Inc() +} + // reportLateVoteIfPastFinalized logs and counts a prepare/commit vote whose // block number is exactly one behind the leader's current block number. func (consensus *Consensus) reportLateVoteIfPastFinalized(recvMsg *FBFTMessage, myBlockNum uint64) { @@ -82,6 +112,10 @@ func (consensus *Consensus) reportLateVoteIfPastFinalized(recvMsg *FBFTMessage, "role": "leader", "phase": phase, }).Inc() + consensusSignatureTotalCounterVec.With(prometheus.Labels{ + "role": "leader", + "phase": phase, + }).Inc() consensus.getLogger().Info(). Uint64("msgBlockNum", recvMsg.BlockNum). diff --git a/consensus/late_signature_test.go b/consensus/late_signature_test.go index 6c88dde077..ab3145624c 100644 --- a/consensus/late_signature_test.go +++ b/consensus/late_signature_test.go @@ -6,6 +6,7 @@ import ( "github.com/ethereum/go-ethereum/common" msg_pb "github.com/harmony-one/harmony/api/proto/message" "github.com/harmony-one/harmony/crypto/bls" + "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" ) @@ -36,7 +37,7 @@ func TestExcludedLocalCommitKeys(t *testing.T) { require.Empty(t, excluded) } -// TestReportLateVoteIfPastFinalized increments the metric only for the prior block. +// TestReportLateVoteIfPastFinalized increments late and total only for the prior block. func TestReportLateVoteIfPastFinalized(t *testing.T) { c := &Consensus{current: NewState(Normal, 0)} initMetrics() @@ -50,10 +51,51 @@ func TestReportLateVoteIfPastFinalized(t *testing.T) { c.reportLateVoteIfPastFinalized(recvMsg, 12) c.reportLateVoteIfPastFinalized(nil, 11) - before := lateSignatureCount(t, "leader", msg_pb.MessageType_COMMIT.String()) + phase := msg_pb.MessageType_COMMIT.String() + beforeLate := lateSignatureCount(t, "leader", phase) + beforeTotal := signatureTotalCount(t, "leader", phase) c.reportLateVoteIfPastFinalized(recvMsg, 11) - after := lateSignatureCount(t, "leader", msg_pb.MessageType_COMMIT.String()) - require.Equal(t, before+1, after) + require.Equal(t, beforeLate+1, lateSignatureCount(t, "leader", phase)) + require.Equal(t, beforeTotal+1, signatureTotalCount(t, "leader", phase)) +} + +// TestReportAcceptedVote increments the total counter for on-time votes. +func TestReportAcceptedVote(t *testing.T) { + c := &Consensus{current: NewState(Normal, 0)} + initMetrics() + + phase := msg_pb.MessageType_PREPARE.String() + beforeLate := lateSignatureCount(t, "leader", phase) + beforeTotal := signatureTotalCount(t, "leader", phase) + c.reportAcceptedVote(phase) + require.Equal(t, beforeLate, lateSignatureCount(t, "leader", phase)) + require.Equal(t, beforeTotal+1, signatureTotalCount(t, "leader", phase)) +} + +// TestReportLocalCommitInclusionsCountsTotal increments total for included and excluded local keys. +func TestReportLocalCommitInclusionsCountsTotal(t *testing.T) { + pub1 := bls.PublicKeyWrapper{Object: bls.RandPrivateKey().GetPublicKey()} + pub2 := bls.PublicKeyWrapper{Object: bls.RandPrivateKey().GetPublicKey()} + pub3 := bls.PublicKeyWrapper{Object: bls.RandPrivateKey().GetPublicKey()} + pub1.Bytes.FromLibBLSPublicKey(pub1.Object) + pub2.Bytes.FromLibBLSPublicKey(pub2.Object) + pub3.Bytes.FromLibBLSPublicKey(pub3.Object) + + mask := bls.NewMask([]bls.PublicKeyWrapper{pub1, pub2}) + require.NoError(t, mask.SetKey(pub1.Bytes, true)) // pub2 excluded + + initMetrics() + beforeLate := lateSignatureCount(t, "validator", "committed") + beforeTotal := signatureTotalCount(t, "validator", "committed") + + var missing []bls.SerializedPublicKey + reportLocalCommitInclusions(mask, []bls.SerializedPublicKey{pub1.Bytes, pub2.Bytes, pub3.Bytes}, func(pub bls.SerializedPublicKey) { + missing = append(missing, pub) + }) + + require.Equal(t, []bls.SerializedPublicKey{pub2.Bytes}, missing) + require.Equal(t, beforeLate+1, lateSignatureCount(t, "validator", "committed")) + require.Equal(t, beforeTotal+2, signatureTotalCount(t, "validator", "committed")) } // TestRecordLastCommitSentGatesInclusionCheck skips checks without a matching sent COMMIT. @@ -84,7 +126,17 @@ func TestRecordLastCommitSentGatesInclusionCheck(t *testing.T) { func lateSignatureCount(t *testing.T, role, phase string) float64 { t.Helper() - metric, err := consensusLateSignatureCounterVec.GetMetricWithLabelValues(role, phase) + return counterValue(t, consensusLateSignatureCounterVec, role, phase) +} + +func signatureTotalCount(t *testing.T, role, phase string) float64 { + t.Helper() + return counterValue(t, consensusSignatureTotalCounterVec, role, phase) +} + +func counterValue(t *testing.T, vec *prometheus.CounterVec, role, phase string) float64 { + t.Helper() + metric, err := vec.GetMetricWithLabelValues(role, phase) require.NoError(t, err) var m dto.Metric require.NoError(t, metric.Write(&m)) diff --git a/consensus/leader.go b/consensus/leader.go index ba27d1f0c3..b9e79818a6 100644 --- a/consensus/leader.go +++ b/consensus/leader.go @@ -198,6 +198,7 @@ func (consensus *Consensus) onPrepare(recvMsg *FBFTMessage) { consensus.getLogger().Warn().Err(err).Msg("[OnPrepare] prepareBitmap.SetKey failed") return } + consensus.reportAcceptedVote(recvMsg.MessageType.String()) //// Write - End //// Read - Start @@ -309,6 +310,7 @@ func (consensus *Consensus) onCommit(recvMsg *FBFTMessage) { Msg("[OnCommit] commitBitmap.SetKey failed") return } + consensus.reportAcceptedVote(recvMsg.MessageType.String()) //// Write - End //// Read - Start diff --git a/consensus/metrics.go b/consensus/metrics.go index 1f8ba2758d..266a238630 100644 --- a/consensus/metrics.go +++ b/consensus/metrics.go @@ -141,6 +141,21 @@ var ( "phase", }, ) + // consensusSignatureTotalCounterVec counts all prepare/commit votes that are + // either accepted on time or classified as late, and all local commit + // inclusion checks. Pair with late_signature to derive on-time vs late rates. + consensusSignatureTotalCounterVec = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "hmy", + Subsystem: "consensus", + Name: "signature_total", + Help: "total prepare/commit votes (accepted or late) and local commit inclusion checks", + }, + []string{ + "role", + "phase", + }, + ) onceMetrics sync.Once @@ -203,6 +218,7 @@ func initMetrics() { consensusPubkeyVec, consensusFinalityHistogram, consensusLateSignatureCounterVec, + consensusSignatureTotalCounterVec, lastPreimageImportGauge, preimageEndGauge, preimageStartGauge, diff --git a/go.mod b/go.mod index 42658cd200..cd80085178 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/olekukonko/tablewriter v0.0.5 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 + github.com/prometheus/client_model v0.6.1 golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c golang.org/x/term v0.28.0 golang.org/x/text v0.21.0 @@ -250,7 +251,6 @@ require ( github.com/pion/webrtc/v3 v3.3.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/tsdb v0.7.1 // indirect