Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions pkg/frost/signing/roast_retry_metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package signing

import (
"sync/atomic"

"github.com/keep-network/keep-core/pkg/clientinfo"
"github.com/keep-network/keep-core/pkg/frost/roast/attempt"
"github.com/keep-network/keep-core/pkg/protocol/group"
)

// roastRetryEvidenceCounters holds cumulative event counts across
// the entire process lifetime. They are bumped whenever a
// metrics-emitting recorder records an event. Exposed to keep-
// core's clientinfo registry via RegisterRoastRetryMetrics, which
// operators invoke at process startup.
//
// The counters are intentionally process-wide rather than per-
// session: operators want to see "how many overflow events did
// the node observe today?" rather than "what was the count for
// the third attempt of session 0x1234?". Per-attempt detail is
// already visible in the TransitionMessage payload.
var (
roastRetryOverflowEvents atomic.Uint64
roastRetryRejectEvents atomic.Uint64
roastRetryConflictEvents atomic.Uint64
)

// Application label prefix used by RegisterRoastRetryMetrics when
// registering with clientinfo.Registry.ObserveApplicationSource.
// The registry concatenates this with each per-source name, so the
// final metric labels look like "frost_roast_retry_overflow_events_total".
const roastRetryMetricsApplication = "frost_roast_retry"

const (
overflowEventsMetricName = "overflow_events_total"
rejectEventsMetricName = "reject_events_total"
conflictEventsMetricName = "conflict_events_total"
)

// RegisterRoastRetryMetrics registers the cumulative ROAST-retry
// evidence counters with the supplied clientinfo registry.
// Operators call this from the node's startup sequence so the
// counters appear in the Prometheus scrape alongside the other
// keep-core metrics.
//
// The metrics are emitted in every build but only increment when
// the receive loops actually call into the metrics-emitting
// recorder, which happens only when the ROAST-retry registry is
// populated (i.e. the operator has opted in). In default builds
// the counters stay at zero.
func RegisterRoastRetryMetrics(registry *clientinfo.Registry) {
if registry == nil {
return
}
registry.ObserveApplicationSource(
roastRetryMetricsApplication,
map[string]clientinfo.Source{
overflowEventsMetricName: func() float64 {
return float64(roastRetryOverflowEvents.Load())
},
rejectEventsMetricName: func() float64 {
return float64(roastRetryRejectEvents.Load())
},
conflictEventsMetricName: func() float64 {
return float64(roastRetryConflictEvents.Load())
},
},
)
}

// metricsEmittingRecorder wraps an attempt.EvidenceRecorder with
// the process-wide cumulative counters declared above. Each
// Record*-class method bumps the matching counter and then
// delegates to the inner recorder so the per-attempt bounded
// snapshot still reflects the event for the NextAttempt policy.
//
// Use newMetricsEmittingRecorder to construct; do not instantiate
// directly.
type metricsEmittingRecorder struct {
inner attempt.EvidenceRecorder
}

func newMetricsEmittingRecorder(
inner attempt.EvidenceRecorder,
) attempt.EvidenceRecorder {
if inner == nil {
return attempt.NoOpRecorder()
}
return &metricsEmittingRecorder{inner: inner}
}

func (m *metricsEmittingRecorder) RecordOverflow(sender group.MemberIndex) {
roastRetryOverflowEvents.Add(1)
m.inner.RecordOverflow(sender)
}

func (m *metricsEmittingRecorder) RecordReject(
sender group.MemberIndex,
reason string,
) {
roastRetryRejectEvents.Add(1)
m.inner.RecordReject(sender, reason)
}

func (m *metricsEmittingRecorder) RecordConflict(sender group.MemberIndex) {
roastRetryConflictEvents.Add(1)
m.inner.RecordConflict(sender)
}

func (m *metricsEmittingRecorder) Snapshot() attempt.Evidence {
return m.inner.Snapshot()
}

// resetRoastRetryMetricsForTest clears the cumulative counters.
// Exposed only for the package's own tests; not a production
// helper.
func resetRoastRetryMetricsForTest() {
roastRetryOverflowEvents.Store(0)
roastRetryRejectEvents.Store(0)
roastRetryConflictEvents.Store(0)
}
116 changes: 116 additions & 0 deletions pkg/frost/signing/roast_retry_metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
package signing

import (
"sync"
"testing"

"github.com/keep-network/keep-core/pkg/frost/roast/attempt"
)

func TestMetricsEmittingRecorder_IncrementsOnEachCategory(t *testing.T) {
resetRoastRetryMetricsForTest()
t.Cleanup(resetRoastRetryMetricsForTest)

rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder())
rec.RecordOverflow(1)
rec.RecordOverflow(2)
rec.RecordReject(3, "validation_gate_rejected")
rec.RecordConflict(4)
rec.RecordConflict(5)
rec.RecordConflict(6)

if got := roastRetryOverflowEvents.Load(); got != 2 {
t.Fatalf("overflow counter: got %d want 2", got)
}
if got := roastRetryRejectEvents.Load(); got != 1 {
t.Fatalf("reject counter: got %d want 1", got)
}
if got := roastRetryConflictEvents.Load(); got != 3 {
t.Fatalf("conflict counter: got %d want 3", got)
}
}

func TestMetricsEmittingRecorder_DelegatesSnapshotToInner(t *testing.T) {
resetRoastRetryMetricsForTest()
t.Cleanup(resetRoastRetryMetricsForTest)

rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder())
rec.RecordOverflow(7)
rec.RecordOverflow(7)

snap := rec.Snapshot()
if snap.Overflows[7] != 2 {
t.Fatalf(
"inner snapshot must reflect events; got %d want 2",
snap.Overflows[7],
)
}
}

func TestMetricsEmittingRecorder_NilInnerFallsBackToNoOp(t *testing.T) {
resetRoastRetryMetricsForTest()
t.Cleanup(resetRoastRetryMetricsForTest)

rec := newMetricsEmittingRecorder(nil)
// Defensive guard: a nil inner recorder must produce a recorder
// that does not panic on Record* calls. The wrapper substitutes
// a NoOp inner.
rec.RecordOverflow(1)
rec.RecordReject(1, "x")
rec.RecordConflict(1)
// Counters STILL increment with the recommended call sites...
// wait, that's wrong. If inner is nil and we substitute NoOp,
// the wrapper is the NoOp recorder, no counters bumped.
if roastRetryOverflowEvents.Load() != 0 {
t.Fatal("nil inner -> NoOp; counters should stay at zero")
}
}

func TestRoastRetryRecorderForCollect_WrapsBoundedWithMetricsWhenRegistered(t *testing.T) {
resetRoastRetryMetricsForTest()
ResetRoastRetryRegistrationForTest()
t.Cleanup(resetRoastRetryMetricsForTest)
t.Cleanup(ResetRoastRetryRegistrationForTest)

// Without registration, the recorder is NoOp -- recording does
// not bump the cumulative counters.
rec := roastRetryRecorderForCollect()
rec.RecordOverflow(1)
if roastRetryOverflowEvents.Load() != 0 {
t.Fatal("no registration -> NoOp recorder -> no counter bump")
}
}

func TestMetricsEmittingRecorder_ConcurrentCountersAreRaceSafe(t *testing.T) {
resetRoastRetryMetricsForTest()
t.Cleanup(resetRoastRetryMetricsForTest)

rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder())
const workers = 16
const callsPerWorker = 100

var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < callsPerWorker; j++ {
rec.RecordOverflow(1)
}
}()
}
wg.Wait()

if got := roastRetryOverflowEvents.Load(); got != uint64(workers*callsPerWorker) {
t.Fatalf(
"concurrent counter: got %d want %d",
got, workers*callsPerWorker,
)
}
}

func TestRegisterRoastRetryMetrics_NilRegistryIsNoOp(t *testing.T) {
// Defensive: RegisterRoastRetryMetrics(nil) must not panic so
// optional integration paths can pass through nil.
RegisterRoastRetryMetrics(nil)
}
6 changes: 5 additions & 1 deletion pkg/frost/signing/roast_retry_recorder.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,9 @@ func roastRetryRecorderForCollect() attempt.EvidenceRecorder {
if _, ok := RegisteredRoastRetryCoordinator(); !ok {
return attempt.NoOpRecorder()
}
return attempt.NewBoundedRecorder()
// Wrap the bounded recorder with the metrics-emitting
// decorator so RecordOverflow/Reject/Conflict bump the
// process-wide cumulative counters that
// RegisterRoastRetryMetrics exposes to clientinfo.
return newMetricsEmittingRecorder(attempt.NewBoundedRecorder())
}
Loading