diff --git a/README.md b/README.md index a641b4e..199c400 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,10 @@ export GUARDRAIL_SERVER__PORT=18080 ./scripts/ci/runtime-smoke.sh ``` + +## Observability + +The runtime exposes Prometheus metrics at `/metrics`. See `guard-rail-engine/deploy/observability/` for: +- Alert rules (`prometheus-alerts.yml`) +- Grafana dashboard (`grafana-dashboard.json`) +- Operator runbooks (`RUNBOOKS.md`) diff --git a/guard-rail-engine/deploy/observability/README.md b/guard-rail-engine/deploy/observability/README.md new file mode 100644 index 0000000..add5271 --- /dev/null +++ b/guard-rail-engine/deploy/observability/README.md @@ -0,0 +1,28 @@ +# Guard Rail Observability + +This directory contains repo-native observability artifacts for the Guard Rail runtime. + +## Metrics Endpoint + +The runtime exposes Prometheus metrics at `/metrics` on the main listener when `observability.metrics_enabled` is `true` in the config. + +Example scrape config: + +```yaml +- job_name: 'guard-rail' + static_configs: + - targets: ['guard-rail:8080'] + metrics_path: '/metrics' +``` + +## Alert Rules + +Load `prometheus-alerts.yml` into your Prometheus configuration. See the alert rules for operational thresholds. + +## Dashboard + +Import `grafana-dashboard.json` into Grafana for a pre-built operational dashboard. + +## Runbooks + +See `RUNBOOKS.md` for operational procedures. \ No newline at end of file diff --git a/guard-rail-engine/deploy/observability/RUNBOOKS.md b/guard-rail-engine/deploy/observability/RUNBOOKS.md new file mode 100644 index 0000000..5eb8125 --- /dev/null +++ b/guard-rail-engine/deploy/observability/RUNBOOKS.md @@ -0,0 +1,222 @@ +# Guard Rail Runbooks + +This document contains operational procedures for common scenarios. + +## Deploy + +**Symptoms:** New version needs to be rolled out. + +**Immediate Checks:** +1. Verify the new container image is available +2. Check current deployment's health: `curl http://localhost:8080/ready` +3. Check metrics: `curl http://localhost:8080/metrics` + +**Mitigation:** +1. Update the container image reference in your deployment +2. Rolling update: `kubectl rollout restart deployment/guard-rail` or equivalent +3. Verify new pods become ready + +**Recovery Verification:** +1. Check readiness: `curl http://localhost:8080/ready` +2. Check logs for errors +3. Verify request throughput returns to normal + +--- + +## Rollback + +**Symptoms:** New version is causing issues (errors, latency, unexpected behavior). + +**Immediate Checks:** +1. Check `/ready` endpoint: `curl http://localhost:8080/ready` +2. Check logs for errors: `kubectl logs -l app=guard-rail --tail=100` +3. Check `guardrail_readiness` metric + +**Mitigation:** +1. Rollback to previous version: `kubectl rollout undo deployment/guard-rail` or equivalent +2. Verify pods return to ready state + +**Recovery Verification:** +1. Confirm `/ready` returns 200 +2. Confirm error rate returns to baseline + +--- + +## Readiness Down + +**Symptoms:** `GuardRailReadinessDown` fires and `/ready` returns non-200 responses. + +**Immediate Checks:** +1. Check `/ready`: `curl -i http://localhost:8080/ready` +2. Check readiness metrics: `guardrail_readiness` and `guardrail_readiness_failures_total` +3. Check recent shutdown and reload events in logs + +**Mitigation:** +1. If cause is `database_unavailable`, follow the **Database Outage** runbook +2. If cause is lifecycle draining during deploy, validate rollout health and finish/rollback deploy +3. Restart the instance if it is stuck in a non-ready state unexpectedly + +**Recovery Verification:** +1. `/ready` returns 200 consistently +2. `guardrail_readiness` returns to `1` + +--- + +## Rotate Admin Token + +**Symptoms:** Admin token needs to be rotated for security. + +**Immediate Checks:** +1. Identify current admin token in config + +**Mitigation:** +1. Generate new token (use a secure random generator) +2. Update config file or secret +3. For container: update secret and restart + +**Recovery Verification:** +1. Test new token: `curl -H "Authorization: Bearer " http://localhost:8081/v1/admin/tenants` +2. Verify old token is rejected + +--- + +## Rotate Tenant Keys + +**Symptoms:** Tenant API keys need rotation. + +**Immediate Checks:** +1. Identify affected tenants via tenant repository + +**Mitigation:** +1. Generate new keys via tenant API or database +2. Distribute new keys to tenants securely + +**Recovery Verification:** +1. Verify new keys work: request with new key returns 200 + +--- + +## Database Outage + +**Symptoms:** `/ready` returns failure with `database_unavailable`, `guardrail_readiness_failures_total{cause="database_unavailable"}` increasing. + +**Immediate Checks:** +1. Check database connectivity: `pg_isready -h -U guardrail` +2. Check database logs +3. Check `guardrail_readiness_failures_total` metric + +**Mitigation:** +1. If database is down, wait for recovery +2. If database is up but Guard Rail can't reach it, check network/firewall +3. Restart Guard Rail if needed + +**Recovery Verification:** +1. `/ready` returns 200 +2. Check `guardrail_readiness_failures_total` stops increasing + +--- + +## Audit Persistence Failures + +**Symptoms:** `GuardRailAuditPersistenceFailures` alert is firing and `guardrail_audit_persist_failures_total` is increasing. + +**Immediate Checks:** +1. Check runtime logs for `failed to persist execution` and `failed to persist execution bundle` +2. Check database health and connectivity +3. Check whether failures are continuous or bursty + +**Mitigation:** +1. If DB connectivity is degraded, follow **Database Outage** +2. If failures are tied to a recent rollout, rollback and re-check +3. If failures are sustained, temporarily reduce traffic while resolving DB/write-path errors + +**Recovery Verification:** +1. `guardrail_audit_persist_failures_total` stops increasing +2. New execution rows are present in `execution_audit` + +--- + +## Replay Persistence Failures + +**Symptoms:** `GuardRailReplayPersistenceFailures` alert is firing and `guardrail_replay_persist_failures_total` is increasing. + +**Immediate Checks:** +1. Check runtime logs for replay bundle persistence errors +2. Check DB health and storage capacity +3. Confirm whether replay capture is enabled in config + +**Mitigation:** +1. If DB path is degraded, follow **Database Outage** +2. If failures are new after rollout, rollback and compare config changes +3. Reduce replay capture pressure if needed while restoring DB health + +**Recovery Verification:** +1. `guardrail_replay_persist_failures_total` stops increasing +2. New replay artifacts are persisted successfully + +--- + +## Upstream Outage + +**Symptoms:** High `guardrail_upstream_failures_total`, requests returning 502/503. + +**Immediate Checks:** +1. Check upstream service health +2. Check `guardrail_upstream_failures_total` metric +3. Check logs for upstream errors + +**Mitigation:** +1. If upstream is down, wait for upstream recovery +2. If upstream is slow, consider adjusting timeout config + +**Recovery Verification:** +1. Request rate returns to normal +2. Upstream failure rate drops to near-zero + +--- + +## Auth Rejection Spike + +**Symptoms:** `GuardRailAuthRejectionSpike` alert is firing and `guardrail_auth_rejections_total` grows quickly. + +**Immediate Checks:** +1. Break down by reason label: `sum by (reason) (increase(guardrail_auth_rejections_total[5m]))` +2. Confirm whether spike is from a single tenant, route, or client rollout +3. Check recent tenant key rotation or route-binding changes + +**Mitigation:** +1. If `missing_api_key` or `invalid_api_key` dominates, coordinate client credential fix +2. If `tenant_route_mismatch` dominates, validate route bindings and tenant config +3. If `rate_limited` dominates, review traffic spike and tenant quotas + +**Recovery Verification:** +1. Auth rejection increase returns to baseline +2. Expected request success rates recover + +--- + +## Bad Policy Reload + +**Symptoms:** `guardrail_reload_events_total{outcome=~"rejected|failed"}` increasing, new policies not loading. + +**Immediate Checks:** +1. Check reload events: `guardrail_reload_events_total` +2. Check logs for reload errors + +**Mitigation:** +1. Fix policy files in `/etc/guard-rail-engine/policies/` +2. Trigger manual reload or wait for auto-reload + +**Recovery Verification:** +1. Check `guardrail_reload_events_total{outcome="succeeded"}` increments + +--- + +## Related Alerts + +- **GuardRailReadinessDown**: `guardrail_readiness == 0` +- **GuardRailAuditPersistenceFailures**: `guardrail_audit_persist_failures_total` increasing +- **GuardRailReplayPersistenceFailures**: `guardrail_replay_persist_failures_total` increasing +- **GuardRailReloadFailures**: `guardrail_reload_events_total{outcome=~"rejected|failed"}` increasing +- **GuardRailAuthRejectionSpike**: `guardrail_auth_rejections_total` elevated +- **GuardRailUpstreamFailures**: `guardrail_upstream_failures_total` elevated diff --git a/guard-rail-engine/deploy/observability/grafana-dashboard.json b/guard-rail-engine/deploy/observability/grafana-dashboard.json new file mode 100644 index 0000000..1c763ea --- /dev/null +++ b/guard-rail-engine/deploy/observability/grafana-dashboard.json @@ -0,0 +1,158 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "id": 1, + "type": "timeseries", + "title": "Request Rate by Verdict", + "targets": [ + { + "expr": "rate(guardrail_requests_total[1m])", + "legendFormat": "{{route_id}} - {{verdict}}", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "id": 2, + "type": "timeseries", + "title": "Request Latency (p95)", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(guardrail_request_latency_seconds_bucket[5m]))", + "legendFormat": "{{route_id}}", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "id": 3, + "type": "timeseries", + "title": "Policy Processing Latency (p95)", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(guardrail_policy_latency_seconds_bucket[5m]))", + "legendFormat": "{{route_id}}", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "id": 4, + "type": "timeseries", + "title": "Upstream Latency (p95)", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(guardrail_upstream_latency_seconds_bucket[5m]))", + "legendFormat": "{{route_id}}", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 6, "x": 0, "y": 16}, + "id": 5, + "type": "stat", + "title": "Readiness State", + "targets": [ + { + "expr": "guardrail_readiness", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 6, "x": 6, "y": 16}, + "id": 6, + "type": "stat", + "title": "In-Flight Requests", + "targets": [ + { + "expr": "guardrail_inflight_requests", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 6, "x": 12, "y": 16}, + "id": 7, + "type": "timeseries", + "title": "Upstream Failures", + "targets": [ + { + "expr": "rate(guardrail_upstream_failures_total[1m])", + "legendFormat": "{{route_id}}", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 6, "x": 18, "y": 16}, + "id": 8, + "type": "timeseries", + "title": "Auth Rejections", + "targets": [ + { + "expr": "rate(guardrail_auth_rejections_total[1m])", + "legendFormat": "{{reason}}", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 24}, + "id": 9, + "type": "timeseries", + "title": "Reload Events", + "targets": [ + { + "expr": "rate(guardrail_reload_events_total[1m])", + "legendFormat": "{{outcome}}", + "refId": "A" + } + ] + }, + { + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 24}, + "id": 10, + "type": "timeseries", + "title": "Persistence Failures", + "targets": [ + { + "expr": "rate(guardrail_audit_persist_failures_total[1m])", + "legendFormat": "audit - {{operation}}", + "refId": "A" + }, + { + "expr": "rate(guardrail_replay_persist_failures_total[1m])", + "legendFormat": "replay - {{operation}}", + "refId": "B" + } + ] + } + ], + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": ["guard-rail"], + "templating": {"list": []}, + "time": {"from": "now-1h", "to": "now"}, + "timepicker": {}, + "timezone": "", + "title": "Guard Rail Operations", + "uid": "guard-rail-ops", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/guard-rail-engine/deploy/observability/prometheus-alerts.yml b/guard-rail-engine/deploy/observability/prometheus-alerts.yml new file mode 100644 index 0000000..5751da3 --- /dev/null +++ b/guard-rail-engine/deploy/observability/prometheus-alerts.yml @@ -0,0 +1,58 @@ +groups: + - name: guard-rail + interval: 30s + rules: + - alert: GuardRailReadinessDown + expr: guardrail_readiness == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Guard Rail readiness endpoint is down" + description: "Guard Rail has been unresponsive for 1 minute" + runbook_url: "#readiness-down" + + - alert: GuardRailAuditPersistenceFailures + expr: increase(guardrail_audit_persist_failures_total[5m]) > 0 + labels: + severity: critical + annotations: + summary: "Guard Rail audit persistence is failing" + description: "Audit persistence has failed repeatedly in the last 5 minutes" + runbook_url: "#audit-persistence-failures" + + - alert: GuardRailReplayPersistenceFailures + expr: increase(guardrail_replay_persist_failures_total[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Guard Rail replay persistence is failing" + description: "Replay persistence has failed in the last 5 minutes" + runbook_url: "#replay-persistence-failures" + + - alert: GuardRailReloadFailures + expr: increase(guardrail_reload_events_total{outcome=~"rejected|failed"}[5m]) > 0 + labels: + severity: warning + annotations: + summary: "Guard Rail config reload is failing" + description: "Config reload has failed or been rejected in the last 5 minutes" + runbook_url: "#bad-policy-reload" + + - alert: GuardRailAuthRejectionSpike + expr: increase(guardrail_auth_rejections_total[5m]) > 100 + labels: + severity: warning + annotations: + summary: "Guard Rail auth rejections are elevated" + description: "More than 100 auth rejections in the last 5 minutes" + runbook_url: "#auth-rejection-spike" + + - alert: GuardRailUpstreamFailures + expr: sum(rate(guardrail_upstream_failures_total[5m])) / clamp_min(sum(rate(guardrail_requests_total{verdict=~"ALLOWED|UPSTREAM_FAILURE"}[5m])), 0.001) > 0.10 + labels: + severity: warning + annotations: + summary: "Guard Rail upstream failure rate is elevated" + description: "More than 10% of upstream requests are failing" + runbook_url: "#upstream-outage" diff --git a/guard-rail-engine/src/main.rs b/guard-rail-engine/src/main.rs index 6deae60..433f27d 100644 --- a/guard-rail-engine/src/main.rs +++ b/guard-rail-engine/src/main.rs @@ -141,6 +141,18 @@ async fn main() -> Result<(), Box> { let tenant_cache = TenantAuthCache::default(); tenant_cache.replace(auth_snapshot).await; + let metrics = if app_config.observability.metrics_enabled { + match observability::metrics::Metrics::new() { + Ok(metrics) => Some(Arc::new(metrics)), + Err(err) => { + tracing::error!(error = %err, "failed to initialize metrics"); + None + } + } + } else { + None + }; + let reload_routes = Arc::clone(&routes); let reload_policies = Arc::clone(&policies); reload::start_watcher( @@ -150,6 +162,7 @@ async fn main() -> Result<(), Box> { reload_policies, tenant_cache.clone(), app_config.environment, + metrics.clone(), )?; let http_client = Client::builder() @@ -160,17 +173,6 @@ async fn main() -> Result<(), Box> { pool.clone(), std::time::Duration::from_millis(250), ); - let metrics = if app_config.observability.metrics_enabled { - match observability::metrics::Metrics::new() { - Ok(metrics) => Some(Arc::new(metrics)), - Err(err) => { - tracing::error!(error = %err, "failed to initialize metrics"); - None - } - } - } else { - None - }; let lifecycle = shutdown::LifecycleState::new(); let route_config_path = PathBuf::from(&app_config.routes_file); diff --git a/guard-rail-engine/src/observability/metrics.rs b/guard-rail-engine/src/observability/metrics.rs index 59729e3..ee5e197 100644 --- a/guard-rail-engine/src/observability/metrics.rs +++ b/guard-rail-engine/src/observability/metrics.rs @@ -1,5 +1,6 @@ use prometheus::{ - Encoder, HistogramOpts, HistogramVec, IntCounterVec, IntGauge, Opts, Registry, TextEncoder, + Encoder, Gauge, HistogramOpts, HistogramVec, IntCounterVec, IntGauge, Opts, Registry, + TextEncoder, }; pub struct Metrics { @@ -12,6 +13,12 @@ pub struct Metrics { inflight_requests: IntGauge, readiness: IntGauge, shutdown_transitions_total: IntCounterVec, + policy_latency_seconds: HistogramVec, + upstream_latency_seconds: HistogramVec, + auth_rejections_total: IntCounterVec, + reload_events_total: IntCounterVec, + last_reload_success_timestamp_seconds: Gauge, + readiness_failures_total: IntCounterVec, } pub struct InflightGuard { @@ -76,6 +83,46 @@ impl Metrics { &["state"], )?; + let policy_latency_seconds = HistogramVec::new( + HistogramOpts::new( + "guardrail_policy_latency_seconds", + "Observed policy-processing latency in seconds.", + ), + &["route_id", "method", "verdict"], + )?; + let upstream_latency_seconds = HistogramVec::new( + HistogramOpts::new( + "guardrail_upstream_latency_seconds", + "Observed upstream forwarding latency in seconds.", + ), + &["route_id", "method"], + )?; + let auth_rejections_total = IntCounterVec::new( + Opts::new( + "guardrail_auth_rejections_total", + "Total auth rejections by reason.", + ), + &["reason"], + )?; + let reload_events_total = IntCounterVec::new( + Opts::new( + "guardrail_reload_events_total", + "Total reload events by outcome.", + ), + &["outcome"], + )?; + let last_reload_success_timestamp_seconds = Gauge::new( + "guardrail_last_reload_success_timestamp_seconds", + "Timestamp of last successful reload.", + )?; + let readiness_failures_total = IntCounterVec::new( + Opts::new( + "guardrail_readiness_failures_total", + "Total readiness check failures by cause.", + ), + &["cause"], + )?; + registry.register(Box::new(requests_total.clone()))?; registry.register(Box::new(request_latency_seconds.clone()))?; registry.register(Box::new(upstream_failures_total.clone()))?; @@ -84,6 +131,12 @@ impl Metrics { registry.register(Box::new(inflight_requests.clone()))?; registry.register(Box::new(readiness.clone()))?; registry.register(Box::new(shutdown_transitions_total.clone()))?; + registry.register(Box::new(policy_latency_seconds.clone()))?; + registry.register(Box::new(upstream_latency_seconds.clone()))?; + registry.register(Box::new(auth_rejections_total.clone()))?; + registry.register(Box::new(reload_events_total.clone()))?; + registry.register(Box::new(last_reload_success_timestamp_seconds.clone()))?; + registry.register(Box::new(readiness_failures_total.clone()))?; Ok(Self { registry, @@ -95,6 +148,12 @@ impl Metrics { inflight_requests, readiness, shutdown_transitions_total, + policy_latency_seconds, + upstream_latency_seconds, + auth_rejections_total, + reload_events_total, + last_reload_success_timestamp_seconds, + readiness_failures_total, }) } @@ -160,6 +219,56 @@ impl Metrics { .with_label_values(&[state]) .inc(); } + + pub fn record_policy_latency(&self, route_id: &str, method: &str, verdict: &str, seconds: f64) { + self.policy_latency_seconds + .with_label_values(&[route_id, method, verdict]) + .observe(seconds); + } + + pub fn record_upstream_latency(&self, route_id: &str, method: &str, seconds: f64) { + self.upstream_latency_seconds + .with_label_values(&[route_id, method]) + .observe(seconds); + } + + pub fn record_auth_rejection(&self, reason: &str) { + self.auth_rejections_total + .with_label_values(&[reason]) + .inc(); + } + + pub fn record_reload_event(&self, outcome: &str) { + self.reload_events_total.with_label_values(&[outcome]).inc(); + } + + pub fn record_reload_success_now(&self) { + self.last_reload_success_timestamp_seconds + .set(chrono::Utc::now().timestamp() as f64); + } + + pub fn record_readiness_failure(&self, cause: &str) { + self.readiness_failures_total + .with_label_values(&[cause]) + .inc(); + } + + #[allow(dead_code)] + pub fn record_policy_and_upstream_latency( + &self, + route_id: &str, + method: &str, + verdict: &str, + policy_seconds: f64, + upstream_seconds: f64, + ) { + self.policy_latency_seconds + .with_label_values(&[route_id, method, verdict]) + .observe(policy_seconds); + self.upstream_latency_seconds + .with_label_values(&[route_id, method]) + .observe(upstream_seconds); + } } #[cfg(test)] @@ -193,4 +302,31 @@ mod tests { assert!(snapshot.contains("operation=\"insert_execution_bundle\"")); assert!(snapshot.contains("state=\"ready\"")); } + + #[test] + fn test_metrics_snapshot_contains_phase6_series() { + let metrics = Metrics::new().unwrap(); + metrics.record_policy_latency("route-a", "POST", "ALLOWED", 0.010); + metrics.record_upstream_latency("route-a", "POST", 0.050); + metrics.record_auth_rejection("missing_api_key"); + metrics.record_reload_event("succeeded"); + metrics.record_reload_success_now(); + metrics.record_readiness_failure("database_unavailable"); + + let snapshot = metrics.render().unwrap(); + assert!(snapshot.contains("guardrail_policy_latency_seconds")); + assert!(snapshot.contains("guardrail_upstream_latency_seconds")); + assert!(snapshot.contains("guardrail_auth_rejections_total")); + assert!(snapshot.contains("guardrail_reload_events_total")); + assert!(snapshot.contains("guardrail_last_reload_success_timestamp_seconds")); + assert!(snapshot.contains("guardrail_readiness_failures_total")); + } + + #[test] + fn test_metrics_snapshot_contains_reload_success_timestamp() { + let metrics = Metrics::new().unwrap(); + metrics.record_reload_success_now(); + let snapshot = metrics.render().unwrap(); + assert!(snapshot.contains("guardrail_last_reload_success_timestamp_seconds")); + } } diff --git a/guard-rail-engine/src/proxy/mod.rs b/guard-rail-engine/src/proxy/mod.rs index 45dc204..dbf3d30 100644 --- a/guard-rail-engine/src/proxy/mod.rs +++ b/guard-rail-engine/src/proxy/mod.rs @@ -214,6 +214,9 @@ pub async fn handle_execute( let bound_tenant_id = match snapshot.route_bindings.get(&route_id) { Some(tenant_id) => *tenant_id, None => { + if let Some(metrics) = &state.metrics { + metrics.record_auth_rejection("route_unbound"); + } let record = ExecutionRecord { execution_id: execution_id.clone(), execution_started_at, @@ -277,6 +280,9 @@ pub async fn handle_execute( tracing::Span::current() .record("api_key_id", tracing::field::display(api_key_id)); if !state.rate_limiter.allow(tenant_id).await { + if let Some(metrics) = &state.metrics { + metrics.record_auth_rejection("rate_limited"); + } let record = ExecutionRecord { execution_id: execution_id.clone(), execution_started_at, @@ -334,6 +340,9 @@ pub async fn handle_execute( api_key_id, .. }) => { + if let Some(metrics) = &state.metrics { + metrics.record_auth_rejection("tenant_route_mismatch"); + } let record = ExecutionRecord { execution_id: execution_id.clone(), execution_started_at, @@ -383,6 +392,9 @@ pub async fn handle_execute( Ok(RequestAuthContext::Admin) => (None, None, Some("admin".to_string())), Err(auth_failure) => { let auth_outcome_str = auth_failure.as_str().to_string(); + if let Some(metrics) = &state.metrics { + metrics.record_auth_rejection(auth_failure.as_str()); + } let record = ExecutionRecord { execution_id: execution_id.clone(), execution_started_at, @@ -533,6 +545,7 @@ pub async fn handle_execute( let verdict = evaluate(&payload, body.len(), &route.policies, &policies); drop(policies); let latency_inspect_us = inspect_start.elapsed().as_micros(); + let latency_inspect_seconds = latency_inspect_us as f64 / 1_000_000.0; match verdict { Verdict::Block { @@ -595,6 +608,14 @@ pub async fn handle_execute( response_body_sha256: None, response_body_truncated: false, }; + if let Some(metrics) = &state.metrics { + metrics.record_policy_latency( + &route_id, + &method_str, + "BLOCKED", + latency_inspect_seconds, + ); + } record_execution_metric( state.metrics.as_ref(), &route_id, @@ -611,6 +632,14 @@ pub async fn handle_execute( state.metrics.clone(), ); } else { + if let Some(metrics) = &state.metrics { + metrics.record_policy_latency( + &route_id, + &method_str, + "BLOCKED", + latency_inspect_seconds, + ); + } record_execution_metric( state.metrics.as_ref(), &route_id, @@ -623,7 +652,16 @@ pub async fn handle_execute( } return response::block_response(&execution_id, &policy_name, &rule_field, &message); } - Verdict::Allow => {} + Verdict::Allow => { + if let Some(metrics) = &state.metrics { + metrics.record_policy_latency( + &route_id, + &method_str, + "ALLOWED", + latency_inspect_seconds, + ); + } + } } // 5. Forward to upstream @@ -653,6 +691,7 @@ pub async fn handle_execute( .await { Ok(result) => { + let forward_elapsed = forward_start.elapsed(); let record = ExecutionRecord { execution_id: execution_id.clone(), execution_started_at, @@ -679,11 +718,18 @@ pub async fn handle_execute( upstream_status: Some(result.status), forward_error: None, latency_inspect_us, - latency_forward_ms: Some(forward_start.elapsed().as_millis()), + latency_forward_ms: Some(forward_elapsed.as_millis()), latency_total_ms: total_start.elapsed().as_millis(), route_config_hash: state.route_config_hash.clone(), policy_set_hash: state.policy_set_hash.clone(), }; + if let Some(metrics) = &state.metrics { + metrics.record_upstream_latency( + &route_id, + &method_str, + forward_elapsed.as_secs_f64(), + ); + } record_execution_metric( state.metrics.as_ref(), &route_id, @@ -735,6 +781,7 @@ pub async fn handle_execute( result.response } Err(e) => { + let forward_elapsed = forward_start.elapsed(); let record = ExecutionRecord { execution_id: execution_id.clone(), execution_started_at, @@ -761,7 +808,7 @@ pub async fn handle_execute( upstream_status: None, forward_error: Some(e.clone()), latency_inspect_us, - latency_forward_ms: Some(forward_start.elapsed().as_millis()), + latency_forward_ms: Some(forward_elapsed.as_millis()), latency_total_ms: total_start.elapsed().as_millis(), route_config_hash: state.route_config_hash.clone(), policy_set_hash: state.policy_set_hash.clone(), @@ -776,6 +823,11 @@ pub async fn handle_execute( tracing::Span::current().record("verdict", "UPSTREAM_FAILURE"); if let Some(metrics) = &state.metrics { metrics.record_upstream_failure(&route_id); + metrics.record_upstream_latency( + &route_id, + &method_str, + forward_elapsed.as_secs_f64(), + ); } spawn_emit_and_persist(record, state.audit_store.clone(), state.metrics.clone()); response::bad_gateway_response(&execution_id, &e) @@ -799,6 +851,14 @@ async fn ready_handler(State(state): State) -> impl IntoResponse { let ready = lifecycle_ready && db_ready; if let Some(metrics) = &state.metrics { + if !ready { + if !lifecycle_ready { + metrics.record_readiness_failure("lifecycle_draining"); + } + if !db_ready { + metrics.record_readiness_failure("database_unavailable"); + } + } metrics.set_readiness(ready); } diff --git a/guard-rail-engine/src/reload.rs b/guard-rail-engine/src/reload.rs index 6f6b19e..4d32e21 100644 --- a/guard-rail-engine/src/reload.rs +++ b/guard-rail-engine/src/reload.rs @@ -1,3 +1,4 @@ +use crate::observability::metrics::Metrics; use crate::policy::PolicySet; use crate::routes::RouteTable; use crate::tenant::cache::TenantAuthCache; @@ -13,6 +14,7 @@ pub fn start_watcher( policies: Arc>, tenant_cache: TenantAuthCache, environment: crate::config::RuntimeEnvironment, + metrics: Option>, ) -> Result<(), Box> { let rt = tokio::runtime::Handle::current(); let rt_initial = rt.clone(); @@ -20,6 +22,7 @@ pub fn start_watcher( let routes_path = routes_file.clone(); let policies_path = policies_dir.clone(); let initial_tenant_cache = tenant_cache.clone(); + let initial_metrics = metrics.clone(); let callback_routes = Arc::clone(&routes); let callback_policies = Arc::clone(&policies); @@ -27,6 +30,7 @@ pub fn start_watcher( let callback_routes_path = routes_file.clone(); let callback_policies_path = policies_dir.clone(); let callback_environment = environment; + let callback_metrics = metrics; let mut watcher = notify::recommended_watcher(move |res: Result| { @@ -42,6 +46,7 @@ pub fn start_watcher( let routes_path = callback_routes_path.clone(); let policies_path = callback_policies_path.clone(); let environment = callback_environment; + let metrics = callback_metrics.clone(); rt.spawn(async move { reload_all( @@ -51,6 +56,7 @@ pub fn start_watcher( &policies, &tenant_cache, environment, + metrics, ) .await; }); @@ -66,6 +72,7 @@ pub fn start_watcher( let final_routes = Arc::clone(&routes); let final_policies = Arc::clone(&policies); + let final_metrics = initial_metrics; rt_initial.spawn(async move { reload_all( &routes_path, @@ -74,6 +81,7 @@ pub fn start_watcher( &final_policies, &initial_tenant_cache, environment, + final_metrics, ) .await; }); @@ -114,11 +122,19 @@ async fn reload_all( policies: &Arc>, tenant_cache: &TenantAuthCache, environment: crate::config::RuntimeEnvironment, + metrics: Option>, ) { + if let Some(metrics) = &metrics { + metrics.record_reload_event("started"); + } + let new_policies = match PolicySet::load_dir(policies_path) { Ok(p) => p, Err(e) => { tracing::warn!("Policies reload failed, keeping previous config: {}", e); + if let Some(metrics) = &metrics { + metrics.record_reload_event("failed"); + } return; } }; @@ -127,6 +143,9 @@ async fn reload_all( Ok(r) => r, Err(e) => { tracing::warn!("Routes reload failed, keeping previous config: {}", e); + if let Some(metrics) = &metrics { + metrics.record_reload_event("failed"); + } return; } }; @@ -142,8 +161,16 @@ async fn reload_all( .await { tracing::warn!("Reload rejected — {}", e); + if let Some(metrics) = &metrics { + metrics.record_reload_event("rejected"); + } return; } + if let Some(metrics) = &metrics { + metrics.record_reload_event("succeeded"); + metrics.record_reload_success_now(); + } + tracing::info!("Routes and policies reloaded successfully"); }