From d8a1d7986b6e54c23ce5ec05b42840d5f5296d51 Mon Sep 17 00:00:00 2001 From: maclane Date: Fri, 22 May 2026 23:02:00 -0500 Subject: [PATCH] feat(frost/signing): expose ROAST-retry evidence counters via clientinfo Adds process-wide cumulative counters for the three evidence categories (overflow / reject / conflict) and exposes them through keep-core's clientinfo registry so operators can observe per- category event rates via the standard Prometheus scrape. The counters increment whenever a metrics-emitting recorder records an event. In default builds and in unregistered-coordinator states the recorder is NoOp, so the counters stay at zero. Operators only see non-zero values once the ROAST-retry registry is populated and live signing flows record evidence -- the "do I have ROAST retry running?" smoke test. * pkg/frost/signing/roast_retry_metrics.go (new, untagged) - Cumulative atomic counters: roastRetryOverflowEvents, roastRetryRejectEvents, roastRetryConflictEvents. - RegisterRoastRetryMetrics(*clientinfo.Registry) registers Source functions under the "frost_roast_retry" application prefix, producing metrics named: - frost_roast_retry_overflow_events_total - frost_roast_retry_reject_events_total - frost_roast_retry_conflict_events_total via the existing ObserveApplicationSource mechanism. - metricsEmittingRecorder wraps an attempt.EvidenceRecorder and bumps the matching counter on each Record* call before delegating to the inner recorder. - Nil-safe: a nil inner recorder collapses to NoOp; a nil clientinfo.Registry is a no-op registration. * pkg/frost/signing/roast_retry_recorder.go (modified) - roastRetryRecorderForCollect now wraps the bounded recorder with newMetricsEmittingRecorder when the registry is populated. NoOp path is unchanged (no metrics emission). Tests (6 cases in roast_retry_metrics_test.go): * Counters increment on Record* (with different per-category counts). * Snapshot delegates to the inner recorder. * Nil inner falls back to NoOp without panicking. * Unregistered coordinator -> NoOp recorder -> no counter bumps. * Concurrent counter increments are race-safe. * RegisterRoastRetryMetrics(nil) is a no-op (defensive guard). Operator wiring: The keep-core node's startup sequence should call RegisterRoastRetryMetrics(&clientinfo.Registry) alongside the existing registry observation calls. Documentation will be added in a follow-up to the rollout guide (docs/development/frost-roast-retry-rollout.adoc). Verification: * go build ./... -- clean * go test ./pkg/frost/... -- pass (5 packages) * go test -race ./pkg/frost/signing/... -- pass * go test -tags 'frost_native frost_tbtc_signer frost_roast_retry' ./pkg/frost/... -- pass (5 packages) * staticcheck -checks '-SA1019' ./pkg/frost/... -- silent * go vet ./pkg/frost/... -- clean * gofmt -l ./pkg/frost/signing/ -- silent Stacked on the AttemptContextHash enforcement PR. --- pkg/frost/signing/roast_retry_metrics.go | 121 ++++++++++++++++++ pkg/frost/signing/roast_retry_metrics_test.go | 116 +++++++++++++++++ pkg/frost/signing/roast_retry_recorder.go | 6 +- 3 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 pkg/frost/signing/roast_retry_metrics.go create mode 100644 pkg/frost/signing/roast_retry_metrics_test.go diff --git a/pkg/frost/signing/roast_retry_metrics.go b/pkg/frost/signing/roast_retry_metrics.go new file mode 100644 index 0000000000..d1312ab51e --- /dev/null +++ b/pkg/frost/signing/roast_retry_metrics.go @@ -0,0 +1,121 @@ +package signing + +import ( + "sync/atomic" + + "github.com/keep-network/keep-core/pkg/clientinfo" + "github.com/keep-network/keep-core/pkg/frost/roast/attempt" + "github.com/keep-network/keep-core/pkg/protocol/group" +) + +// roastRetryEvidenceCounters holds cumulative event counts across +// the entire process lifetime. They are bumped whenever a +// metrics-emitting recorder records an event. Exposed to keep- +// core's clientinfo registry via RegisterRoastRetryMetrics, which +// operators invoke at process startup. +// +// The counters are intentionally process-wide rather than per- +// session: operators want to see "how many overflow events did +// the node observe today?" rather than "what was the count for +// the third attempt of session 0x1234?". Per-attempt detail is +// already visible in the TransitionMessage payload. +var ( + roastRetryOverflowEvents atomic.Uint64 + roastRetryRejectEvents atomic.Uint64 + roastRetryConflictEvents atomic.Uint64 +) + +// Application label prefix used by RegisterRoastRetryMetrics when +// registering with clientinfo.Registry.ObserveApplicationSource. +// The registry concatenates this with each per-source name, so the +// final metric labels look like "frost_roast_retry_overflow_events_total". +const roastRetryMetricsApplication = "frost_roast_retry" + +const ( + overflowEventsMetricName = "overflow_events_total" + rejectEventsMetricName = "reject_events_total" + conflictEventsMetricName = "conflict_events_total" +) + +// RegisterRoastRetryMetrics registers the cumulative ROAST-retry +// evidence counters with the supplied clientinfo registry. +// Operators call this from the node's startup sequence so the +// counters appear in the Prometheus scrape alongside the other +// keep-core metrics. +// +// The metrics are emitted in every build but only increment when +// the receive loops actually call into the metrics-emitting +// recorder, which happens only when the ROAST-retry registry is +// populated (i.e. the operator has opted in). In default builds +// the counters stay at zero. +func RegisterRoastRetryMetrics(registry *clientinfo.Registry) { + if registry == nil { + return + } + registry.ObserveApplicationSource( + roastRetryMetricsApplication, + map[string]clientinfo.Source{ + overflowEventsMetricName: func() float64 { + return float64(roastRetryOverflowEvents.Load()) + }, + rejectEventsMetricName: func() float64 { + return float64(roastRetryRejectEvents.Load()) + }, + conflictEventsMetricName: func() float64 { + return float64(roastRetryConflictEvents.Load()) + }, + }, + ) +} + +// metricsEmittingRecorder wraps an attempt.EvidenceRecorder with +// the process-wide cumulative counters declared above. Each +// Record*-class method bumps the matching counter and then +// delegates to the inner recorder so the per-attempt bounded +// snapshot still reflects the event for the NextAttempt policy. +// +// Use newMetricsEmittingRecorder to construct; do not instantiate +// directly. +type metricsEmittingRecorder struct { + inner attempt.EvidenceRecorder +} + +func newMetricsEmittingRecorder( + inner attempt.EvidenceRecorder, +) attempt.EvidenceRecorder { + if inner == nil { + return attempt.NoOpRecorder() + } + return &metricsEmittingRecorder{inner: inner} +} + +func (m *metricsEmittingRecorder) RecordOverflow(sender group.MemberIndex) { + roastRetryOverflowEvents.Add(1) + m.inner.RecordOverflow(sender) +} + +func (m *metricsEmittingRecorder) RecordReject( + sender group.MemberIndex, + reason string, +) { + roastRetryRejectEvents.Add(1) + m.inner.RecordReject(sender, reason) +} + +func (m *metricsEmittingRecorder) RecordConflict(sender group.MemberIndex) { + roastRetryConflictEvents.Add(1) + m.inner.RecordConflict(sender) +} + +func (m *metricsEmittingRecorder) Snapshot() attempt.Evidence { + return m.inner.Snapshot() +} + +// resetRoastRetryMetricsForTest clears the cumulative counters. +// Exposed only for the package's own tests; not a production +// helper. +func resetRoastRetryMetricsForTest() { + roastRetryOverflowEvents.Store(0) + roastRetryRejectEvents.Store(0) + roastRetryConflictEvents.Store(0) +} diff --git a/pkg/frost/signing/roast_retry_metrics_test.go b/pkg/frost/signing/roast_retry_metrics_test.go new file mode 100644 index 0000000000..fd5e015255 --- /dev/null +++ b/pkg/frost/signing/roast_retry_metrics_test.go @@ -0,0 +1,116 @@ +package signing + +import ( + "sync" + "testing" + + "github.com/keep-network/keep-core/pkg/frost/roast/attempt" +) + +func TestMetricsEmittingRecorder_IncrementsOnEachCategory(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) + rec.RecordOverflow(1) + rec.RecordOverflow(2) + rec.RecordReject(3, "validation_gate_rejected") + rec.RecordConflict(4) + rec.RecordConflict(5) + rec.RecordConflict(6) + + if got := roastRetryOverflowEvents.Load(); got != 2 { + t.Fatalf("overflow counter: got %d want 2", got) + } + if got := roastRetryRejectEvents.Load(); got != 1 { + t.Fatalf("reject counter: got %d want 1", got) + } + if got := roastRetryConflictEvents.Load(); got != 3 { + t.Fatalf("conflict counter: got %d want 3", got) + } +} + +func TestMetricsEmittingRecorder_DelegatesSnapshotToInner(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) + rec.RecordOverflow(7) + rec.RecordOverflow(7) + + snap := rec.Snapshot() + if snap.Overflows[7] != 2 { + t.Fatalf( + "inner snapshot must reflect events; got %d want 2", + snap.Overflows[7], + ) + } +} + +func TestMetricsEmittingRecorder_NilInnerFallsBackToNoOp(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(nil) + // Defensive guard: a nil inner recorder must produce a recorder + // that does not panic on Record* calls. The wrapper substitutes + // a NoOp inner. + rec.RecordOverflow(1) + rec.RecordReject(1, "x") + rec.RecordConflict(1) + // Counters STILL increment with the recommended call sites... + // wait, that's wrong. If inner is nil and we substitute NoOp, + // the wrapper is the NoOp recorder, no counters bumped. + if roastRetryOverflowEvents.Load() != 0 { + t.Fatal("nil inner -> NoOp; counters should stay at zero") + } +} + +func TestRoastRetryRecorderForCollect_WrapsBoundedWithMetricsWhenRegistered(t *testing.T) { + resetRoastRetryMetricsForTest() + ResetRoastRetryRegistrationForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + t.Cleanup(ResetRoastRetryRegistrationForTest) + + // Without registration, the recorder is NoOp -- recording does + // not bump the cumulative counters. + rec := roastRetryRecorderForCollect() + rec.RecordOverflow(1) + if roastRetryOverflowEvents.Load() != 0 { + t.Fatal("no registration -> NoOp recorder -> no counter bump") + } +} + +func TestMetricsEmittingRecorder_ConcurrentCountersAreRaceSafe(t *testing.T) { + resetRoastRetryMetricsForTest() + t.Cleanup(resetRoastRetryMetricsForTest) + + rec := newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) + const workers = 16 + const callsPerWorker = 100 + + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < callsPerWorker; j++ { + rec.RecordOverflow(1) + } + }() + } + wg.Wait() + + if got := roastRetryOverflowEvents.Load(); got != uint64(workers*callsPerWorker) { + t.Fatalf( + "concurrent counter: got %d want %d", + got, workers*callsPerWorker, + ) + } +} + +func TestRegisterRoastRetryMetrics_NilRegistryIsNoOp(t *testing.T) { + // Defensive: RegisterRoastRetryMetrics(nil) must not panic so + // optional integration paths can pass through nil. + RegisterRoastRetryMetrics(nil) +} diff --git a/pkg/frost/signing/roast_retry_recorder.go b/pkg/frost/signing/roast_retry_recorder.go index ec284d3bc8..4bc2e292d7 100644 --- a/pkg/frost/signing/roast_retry_recorder.go +++ b/pkg/frost/signing/roast_retry_recorder.go @@ -30,5 +30,9 @@ func roastRetryRecorderForCollect() attempt.EvidenceRecorder { if _, ok := RegisteredRoastRetryCoordinator(); !ok { return attempt.NoOpRecorder() } - return attempt.NewBoundedRecorder() + // Wrap the bounded recorder with the metrics-emitting + // decorator so RecordOverflow/Reject/Conflict bump the + // process-wide cumulative counters that + // RegisterRoastRetryMetrics exposes to clientinfo. + return newMetricsEmittingRecorder(attempt.NewBoundedRecorder()) }