diff --git a/pkg/frost/signing/roast_retry_metrics.go b/pkg/frost/signing/roast_retry_metrics.go new file mode 100644 index 0000000000..d1312ab51e --- /dev/null +++ b/pkg/frost/signing/roast_retry_metrics.go @@ -0,0 +1,121 @@ +package signing + +import ( + "sync/atomic" + + "github.com/keep-network/keep-core/pkg/clientinfo" + "github.com/keep-network/keep-core/pkg/frost/roast/attempt" + "github.com/keep-network/keep-core/pkg/protocol/group" +) + +// roastRetryEvidenceCounters holds cumulative event counts across +// the entire process lifetime. They are bumped whenever a +// metrics-emitting recorder records an event. Exposed to keep- +// core's clientinfo registry via RegisterRoastRetryMetrics, which +// operators invoke at process startup. +// +// The counters are intentionally process-wide rather than per- +// session: operators want to see "how many overflow events did +// the node observe today?" rather than "what was the count for +// the third attempt of session 0x1234?". Per-attempt detail is +// already visible in the TransitionMessage payload. +var ( + roastRetryOverflowEvents atomic.Uint64 + roastRetryRejectEvents atomic.Uint64 + roastRetryConflictEvents atomic.Uint64 +) + +// Application label prefix used by RegisterRoastRetryMetrics when +// registering with clientinfo.Registry.ObserveApplicationSource. +// The registry concatenates this with each per-source name, so the +// final metric labels look like "frost_roast_retry_overflow_events_total". +const roastRetryMetricsApplication = "frost_roast_retry" + +const ( + overflowEventsMetricName = "overflow_events_total" + rejectEventsMetricName = "reject_events_total" + conflictEventsMetricName = "conflict_events_total" +) + +// RegisterRoastRetryMetrics registers the cumulative ROAST-retry +// evidence counters with the supplied clientinfo registry. +// Operators call this from the node's startup sequence so the +// counters appear in the Prometheus scrape alongside the other +// keep-core metrics. +// +// The metrics are emitted in every build but only increment when +// the receive loops actually call into the metrics-emitting +// recorder, which happens only when the ROAST-retry registry is +// populated (i.e. the operator has opted in). In default builds +// the counters stay at zero. +func RegisterRoastRetryMetrics(registry *clientinfo.Registry) { + if registry == nil { + return + } + registry.ObserveApplicationSource( + roastRetryMetricsApplication, + map[string]clientinfo.Source{ + overflowEventsMetricName: func() float64 { + return float64(roastRetryOverflowEvents.Load()) + }, + rejectEventsMetricName: func() float64 { + return float64(roastRetryRejectEvents.Load()) + }, + conflictEventsMetricName: func() float64 { + return float64(roastRetryConflictEvents.Load()) + }, + }, + ) +} + +// metricsEmittingRecorder wraps an attempt.EvidenceRecorder with +// the process-wide cumulative counters declared above. Each +// Record*-class method bumps the matching counter and then +// delegates to the inner recorder so the per-attempt bounded +// snapshot still reflects the event for the NextAttempt policy. +// +// Use newMetricsEmittingRecorder to construct; do not instantiate +// directly. +type metricsEmittingRecorder struct { + inner attempt.EvidenceRecorder +} + +func newMetricsEmittingRecorder( + inner attempt.EvidenceRecorder, +) attempt.EvidenceRecorder { + if inner == nil { + return attempt.NoOpRecorder() + } + return &metricsEmittingRecorder{inner: inner} +} + +func (m *metricsEmittingRecorder) RecordOverflow(sender group.MemberIndex) { + roastRetryOverflowEvents.Add(1) + m.inner.RecordOverflow(sender) +} + +func (m *metricsEmittingRecorder) RecordReject( + sender group.MemberIndex, + reason string, +) { + roastRetryRejectEvents.Add(1) + m.inner.RecordReject(sender, reason) +} + +func (m *metricsEmittingRecorder) RecordConflict(sender group.MemberIndex) { + roastRetryConflictEvents.Add(1) + m.inner.RecordConflict(sender) +} + +func (m *metricsEmittingRecorder) Snapshot() attempt.Evidence { + return m.inner.Snapshot() +} + +// resetRoastRetryMetricsForTest clears the cumulative counters. +// Exposed only for the package's own tests; not a production +// helper. +func resetRoastRetryMetricsForTest() { + roastRetryOverflowEvents.Store(0) + roastRetryRejectEvents.Store(0) + roastRetryConflictEvents.Store(0) +} diff --git a/pkg/frost/signing/roast_retry_metrics_test.go b/pkg/frost/signing/roast_retry_metrics_test.go new file mode 100644 index 0000000000..fd5e015255 --- /dev/null +++ b/pkg/frost/signing/roast_retry_metrics_test.go @@ -0,0 +1,116 @@ +package signing + +import ( + "sync" + "testing" + + "github.com/keep-network/keep-core/pkg/frost/roast/attempt" +) + +func TestMetricsEmittingRecorder_IncrementsOnEachCategory(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) + rec.RecordOverflow(1) + rec.RecordOverflow(2) + rec.RecordReject(3, "validation_gate_rejected") + rec.RecordConflict(4) + rec.RecordConflict(5) + rec.RecordConflict(6) + + if got := roastRetryOverflowEvents.Load(); got != 2 { + t.Fatalf("overflow counter: got %d want 2", got) + } + if got := roastRetryRejectEvents.Load(); got != 1 { + t.Fatalf("reject counter: got %d want 1", got) + } + if got := roastRetryConflictEvents.Load(); got != 3 { + t.Fatalf("conflict counter: got %d want 3", got) + } +} + +func TestMetricsEmittingRecorder_DelegatesSnapshotToInner(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) + rec.RecordOverflow(7) + rec.RecordOverflow(7) + + snap := rec.Snapshot() + if snap.Overflows[7] != 2 { + t.Fatalf( + "inner snapshot must reflect events; got %d want 2", + snap.Overflows[7], + ) + } +} + +func TestMetricsEmittingRecorder_NilInnerFallsBackToNoOp(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(nil) + // Defensive guard: a nil inner recorder must produce a recorder + // that does not panic on Record* calls. The wrapper substitutes + // a NoOp inner. + rec.RecordOverflow(1) + rec.RecordReject(1, "x") + rec.RecordConflict(1) + // Counters STILL increment with the recommended call sites... + // wait, that's wrong. If inner is nil and we substitute NoOp, + // the wrapper is the NoOp recorder, no counters bumped. + if roastRetryOverflowEvents.Load() != 0 { + t.Fatal("nil inner -> NoOp; counters should stay at zero") + } +} + +func TestRoastRetryRecorderForCollect_WrapsBoundedWithMetricsWhenRegistered(t *testing.T) { + resetRoastRetryMetricsForTest() + ResetRoastRetryRegistrationForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + t.Cleanup(ResetRoastRetryRegistrationForTest) + + // Without registration, the recorder is NoOp -- recording does + // not bump the cumulative counters. + rec := roastRetryRecorderForCollect() + rec.RecordOverflow(1) + if roastRetryOverflowEvents.Load() != 0 { + t.Fatal("no registration -> NoOp recorder -> no counter bump") + } +} + +func TestMetricsEmittingRecorder_ConcurrentCountersAreRaceSafe(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) + const workers = 16 + const callsPerWorker = 100 + + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < callsPerWorker; j++ { + rec.RecordOverflow(1) + } + }() + } + wg.Wait() + + if got := roastRetryOverflowEvents.Load(); got != uint64(workers*callsPerWorker) { + t.Fatalf( + "concurrent counter: got %d want %d", + got, workers*callsPerWorker, + ) + } +} + +func TestRegisterRoastRetryMetrics_NilRegistryIsNoOp(t *testing.T) { + // Defensive: RegisterRoastRetryMetrics(nil) must not panic so + // optional integration paths can pass through nil. + RegisterRoastRetryMetrics(nil) +} diff --git a/pkg/frost/signing/roast_retry_recorder.go b/pkg/frost/signing/roast_retry_recorder.go index ec284d3bc8..4bc2e292d7 100644 --- a/pkg/frost/signing/roast_retry_recorder.go +++ b/pkg/frost/signing/roast_retry_recorder.go @@ -30,5 +30,9 @@ func roastRetryRecorderForCollect() attempt.EvidenceRecorder { if _, ok := RegisteredRoastRetryCoordinator(); !ok { return attempt.NoOpRecorder() } - return attempt.NewBoundedRecorder() + // Wrap the bounded recorder with the metrics-emitting + // decorator so RecordOverflow/Reject/Conflict bump the + // process-wide cumulative counters that + // RegisterRoastRetryMetrics exposes to clientinfo. + return newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) }