From 72ecb1ba0a83e123d791fa719d9791dda455f6e5 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Mon, 7 Sep 2026 08:31:19 -0400 Subject: [PATCH] Grafana dashboard for the sonic-ray release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SuperSONIC dashboard filters app=~"supersonic" in several panels, so it cannot show this release; rather than loosen a production dashboard, this is a separate one (uid sonic-ray) provisioned into the same Grafana. Two halves. Ray Serve: replicas, request rate by RPC, latency percentiles, the autoscaling signal (in-flight per replica) against what it produced (replicas, queued), errors. Triton: inferences by model and the queue/input/infer/output breakdown, both the SuperSONIC queries scoped to this release, plus pending requests and GPU. One panel subtracts Triton's own mean request duration from Serve's mean latency: the cost of the Ray hop, which is the thing worth measuring here. Every series name comes from a source rather than memory — Ray 2.52's metric definitions, the proxy's declared tag keys, the af-pod-monitor relabel rules — and the tests hold that, along with release scoping, guarded divisions, matching units on the overhead panel and a grid with no overlaps. Co-Authored-By: Claude Opus 5 --- .../grafana/dashboards/sonic-ray.json | 974 ++++++++++++++++++ deploy/core-production/kustomization.yaml | 1 + tests/manifests/test_sonic_ray_dashboard.py | 216 ++++ 3 files changed, 1191 insertions(+) create mode 100644 apps/monitoring/grafana/dashboards/sonic-ray.json create mode 100644 tests/manifests/test_sonic_ray_dashboard.py diff --git a/apps/monitoring/grafana/dashboards/sonic-ray.json b/apps/monitoring/grafana/dashboards/sonic-ray.json new file mode 100644 index 00000000..8f9a3de1 --- /dev/null +++ b/apps/monitoring/grafana/dashboards/sonic-ray.json @@ -0,0 +1,974 @@ +{ + "uid": "sonic-ray", + "title": "Triton on Ray", + "description": "Triton served through Ray Serve's gRPC proxy: the request path, the inference, and what the Ray hop costs.", + "tags": ["sonic", "ray", "triton"], + "timezone": "browser", + "editable": true, + "schemaVersion": 41, + "refresh": "10s", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "release", + "label": "Release name", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(ray_serve_deployment_replica_healthy,release)", + "query": { + "qryType": 1, + "query": "label_values(ray_serve_deployment_replica_healthy,release)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "includeAll": true, + "multi": true, + "refresh": 1, + "options": [], + "regex": "", + "allowCustomValue": false + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(ray_serve_deployment_replica_healthy,namespace)", + "query": { + "qryType": 1, + "query": "label_values(ray_serve_deployment_replica_healthy,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "includeAll": true, + "multi": true, + "refresh": 1, + "options": [], + "regex": "", + "allowCustomValue": true + } + ] + }, + "panels": [ + { + "id": 1, + "type": "text", + "title": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 3 + }, + "options": { + "mode": "markdown", + "content": "## Triton on Ray (`sonic-ray`)\n\nTriton does the inference; Ray Serve's gRPC proxy carries the requests and sizes the deployment from them. **Ray Serve** panels are the request path (what the proxy accepted, what Serve did with it, how it scaled). **Triton** panels are the same series the SuperSONIC dashboards plot, for this release. **Ray overhead** is the difference between the two latencies: what the extra hop costs." + } + }, + { + "id": 2, + "type": "row", + "title": "Ray Serve \u2014 the request path", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 3, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 3, + "type": "stat", + "title": "Replicas serving", + "description": "Healthy replicas of the forwarder. One replica is one GPU pod with one Triton in it.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 4, + "w": 4, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(ray_serve_deployment_replica_healthy{release=~\"$release\", namespace=~\"$namespace\"})", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Requests per second", + "description": "Accepted by Serve's gRPC proxy, every RPC included.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 4, + "y": 4, + "w": 4, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(ray_serve_num_grpc_requests_total{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "In flight per replica", + "description": "The autoscaling signal: Serve adds a replica when this stays above target_ongoing_requests (16 by default), and removes one when it falls well below.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 4, + "w": 4, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(ray_serve_replica_processing_queries{release=~\"$release\", namespace=~\"$namespace\"}) / clamp_min(sum(ray_serve_deployment_replica_healthy{release=~\"$release\", namespace=~\"$namespace\"}), 1)", + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 16 + }, + { + "color": "red", + "value": 64 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Requests per second by RPC", + "description": "Triton's own RPC names, as Serve's proxy sees them. ModelInfer is inference; the metadata calls are what a client does on connect.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 5 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(method)(rate(ray_serve_num_grpc_requests_total{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Serve latency", + "description": "Measured at the replica: the forwarder's call to Triton and back, queueing included.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 9, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by(le)(rate(ray_serve_deployment_processing_latency_ms_bucket{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le)(rate(ray_serve_deployment_processing_latency_ms_bucket{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le)(rate(ray_serve_deployment_processing_latency_ms_bucket{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(ray_serve_deployment_processing_latency_ms_sum{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])) / clamp_min(sum(rate(ray_serve_deployment_processing_latency_ms_count{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", + "legendFormat": "mean", + "range": true, + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Autoscaling", + "description": "What Serve decided and why. Queued requests rising while replicas are flat means the group is behind: either the ceiling is reached or pods are still starting (an image pull plus Triton's load from CVMFS takes minutes).", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 9, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(ray_serve_deployment_replica_healthy{release=~\"$release\", namespace=~\"$namespace\"})", + "legendFormat": "replicas", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(ray_serve_replica_processing_queries{release=~\"$release\", namespace=~\"$namespace\"})", + "legendFormat": "in flight", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(ray_serve_deployment_queued_queries{release=~\"$release\", namespace=~\"$namespace\"})", + "legendFormat": "queued", + "range": true, + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Errors per second", + "description": "Proxy-level and replica-level. A replica whose Triton died fails its health check and is restarted; that shows here first.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 17, + "w": 12, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(ray_serve_num_grpc_error_requests_total{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "proxy", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(ray_serve_deployment_error_counter_total{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "deployment", + "range": true, + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Ray overhead", + "description": "Serve's mean latency minus Triton's own mean request duration: the cost of the proxy hop, the Ray transport and the forwarder. The number this deployment exists to measure \u2014 compare it against a Triton reached directly.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 17, + "w": 12, + "h": 6 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(ray_serve_deployment_processing_latency_ms_sum{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])) / clamp_min(sum(rate(ray_serve_deployment_processing_latency_ms_count{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])), 0.001) - sum(rate(nv_inference_request_duration_us{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])) / clamp_min(sum(rate(nv_inference_request_success{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])), 0.001) / 1000", + "legendFormat": "overhead", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "row", + "title": "Triton \u2014 the inference", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 23, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 12, + "type": "timeseries", + "title": "Inferences per second by model", + "description": "Triton's own count, per model.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(model)(rate(nv_inference_count{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval]))", + "legendFormat": "{{model}}", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Latency breakdown", + "description": "Per execution, inside Triton. Queue time rising is the GPU falling behind; that is what the autoscaler is meant to fix by adding a server.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(nv_inference_queue_duration_us{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])) / clamp_min(sum(rate(nv_inference_exec_count{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", + "legendFormat": "queue", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(nv_inference_compute_input_duration_us{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])) / clamp_min(sum(rate(nv_inference_exec_count{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", + "legendFormat": "input", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(nv_inference_compute_infer_duration_us{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])) / clamp_min(sum(rate(nv_inference_exec_count{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", + "legendFormat": "infer", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(nv_inference_compute_output_duration_us{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])) / clamp_min(sum(rate(nv_inference_exec_count{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", + "legendFormat": "output", + "range": true, + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "unit": "\u00b5s", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Pending requests per server", + "description": "Triton's own queue depth. KEDA scales the SuperSONIC releases on this; here it is an observation, not an input.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 8, + "h": 7 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(pod)(nv_inference_pending_request_count{release=~\"$release\", namespace=~\"$namespace\"})", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "GPU utilization", + "description": "One GPU per pod.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 8, + "y": 32, + "w": 8, + "h": 7 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg_over_time(nv_gpu_utilization{release=~\"$release\", namespace=~\"$namespace\"}[$__rate_interval])", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "GPU memory", + "description": "Fraction of each GPU's memory Triton holds; a model repository that does not fit shows up here before it shows up as an error.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 16, + "y": 32, + "w": 8, + "h": 7 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "nv_gpu_memory_used_bytes{release=~\"$release\", namespace=~\"$namespace\"} / clamp_min(nv_gpu_memory_total_bytes{release=~\"$release\", namespace=~\"$namespace\"}, 1)", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 0, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/deploy/core-production/kustomization.yaml b/deploy/core-production/kustomization.yaml index dd7f8133..bc45765e 100644 --- a/deploy/core-production/kustomization.yaml +++ b/deploy/core-production/kustomization.yaml @@ -102,6 +102,7 @@ configMapGenerator: - ../../apps/monitoring/grafana/dashboards/agentic-interface.json - ../../apps/monitoring/grafana/dashboards/single-user.json - ../../apps/monitoring/grafana/dashboards/sonic.json + - ../../apps/monitoring/grafana/dashboards/sonic-ray.json - ../../apps/monitoring/grafana/dashboards/alerts.json - ../../apps/monitoring/grafana/dashboards/gpus.json - ../../apps/monitoring/grafana/dashboards/phys390.json diff --git a/tests/manifests/test_sonic_ray_dashboard.py b/tests/manifests/test_sonic_ray_dashboard.py new file mode 100644 index 00000000..b033a1d8 --- /dev/null +++ b/tests/manifests/test_sonic_ray_dashboard.py @@ -0,0 +1,216 @@ +"""Contracts for the Triton-on-Ray dashboard. + +A Grafana dashboard fails silently: a renamed metric, a label that does not +exist or a missing release filter renders an empty panel, or worse, one that +quietly sums two releases together. These hold the parts that were checked +against the sources rather than guessed: + + * Ray Serve metric names come from Ray 2.52's own definitions + (python/ray/serve/_private/{replica,router,proxy,deployment_state}.py). + * `method` is a declared tag key of the proxy's request counter; `pod`, + `release` and `namespace` are attached by the AF Prometheus itself + (af-pod-monitor relabel_configs in apps/monitoring/prometheus/values.yaml), + so panels group by those rather than by Ray's internal replica tags. + * The nv_* series and the datasource uid match what the SuperSONIC + dashboard already plots on this Grafana. +""" + +import json +import re + +import yaml +from common import REPO + +DASHBOARD = REPO / "apps/monitoring/grafana/dashboards/sonic-ray.json" +DASHBOARD_FILE = "sonic-ray.json" +KUSTOMIZATION = REPO / "deploy/core-production/kustomization.yaml" +PUBLIC_VALUES = REPO / "apps/monitoring/grafana/values.yaml" +PRIVATE_VALUES = REPO / "apps/monitoring/grafana/values-private.yaml" + +# Every metric the dashboard is allowed to name, and where the name was +# verified. Adding one means checking it the same way. +RAY_METRICS = { + "ray_serve_deployment_replica_healthy", # deployment_state.py + "ray_serve_deployment_processing_latency_ms", # replica.py (histogram) + "ray_serve_replica_processing_queries", # replica.py + "ray_serve_deployment_error_counter_total", # replica.py + "ray_serve_deployment_queued_queries", # router.py + "ray_serve_num_grpc_requests_total", # proxy.py: serve_num_{protocol}_requests + "ray_serve_num_grpc_error_requests_total", # proxy.py +} +TRITON_METRICS = { + "nv_inference_count", + "nv_inference_exec_count", + "nv_inference_request_duration_us", + "nv_inference_request_success", + "nv_inference_queue_duration_us", + "nv_inference_compute_input_duration_us", + "nv_inference_compute_infer_duration_us", + "nv_inference_compute_output_duration_us", + "nv_inference_pending_request_count", + "nv_gpu_utilization", + "nv_gpu_memory_used_bytes", + "nv_gpu_memory_total_bytes", +} +# Labels a panel may group by: declared Ray tag keys, or labels the AF +# Prometheus attaches during relabeling. +GROUPABLE = {"method", "model", "pod", "le"} + +METRIC = re.compile(r"\b(ray_serve_[a-z_]+|nv_[a-z_]+)\b") +BY = re.compile(r"\bby\s*\(([^)]*)\)") + + +def dashboard(): + return json.loads(DASHBOARD.read_text()) + + +def panels(): + return [p for p in dashboard()["panels"] if p["type"] not in ("row", "text")] + + +def exprs(): + return [ + (p["title"], t["expr"]) + for p in panels() + for t in p.get("targets", []) + if t.get("expr") + ] + + +def generated_dashboards(): + """{generator name: [dashboard filenames]} from the deploy kustomization.""" + doc = yaml.safe_load(KUSTOMIZATION.read_text()) + return { + cm["name"]: [f.rsplit("/", 1)[-1] for f in cm.get("files", [])] + for cm in doc["configMapGenerator"] + if cm["name"].startswith("grafana-") + } + + +def test_dashboard_is_provisioned_to_the_public_instance(): + """Grafana is two instances since the public/private split: the public one + is anonymous-viewer readable and carries the SuperSONIC dashboard, so its + sibling belongs there too. A dashboard in neither ConfigMap is a file in + git and nothing else; one in the private ConfigMap would be invisible to + the people watching a SONIC benchmark.""" + generated = generated_dashboards() + assert DASHBOARD_FILE in generated["grafana-public-dashboards"] + assert DASHBOARD_FILE not in generated["grafana-private-dashboards"] + # ...and it sits with the release it is the counterpart of. + assert "sonic.json" in generated["grafana-public-dashboards"] + + +def test_the_public_configmap_is_the_one_the_public_instance_mounts(): + """The ConfigMap name above is only meaningful if that instance mounts it; + the split gave each instance its own map and its own provider.""" + public = yaml.safe_load(PUBLIC_VALUES.read_text())["dashboardsConfigMaps"] + private = yaml.safe_load(PRIVATE_VALUES.read_text())["dashboardsConfigMaps"] + assert public["public"] == "grafana-public-dashboards" + assert private["private"] == "grafana-private-dashboards" + assert "grafana-public-dashboards" not in private.values() + + +def test_identity_is_stable(): + """The uid is what links and bookmarks resolve; renaming it orphans them.""" + doc = dashboard() + assert doc["uid"] == "sonic-ray" + assert doc["title"] == "Triton on Ray" + + +def test_every_series_is_one_the_exporters_actually_publish(subtests=None): + """Guards against plausible-looking names that no exporter emits.""" + known = RAY_METRICS | TRITON_METRICS + for title, expr in exprs(): + for metric in METRIC.findall(expr): + base = re.sub(r"_(bucket|sum|count)$", "", metric) + assert metric in known or base in known, f"{title}: unknown series {metric}" + + +def test_every_panel_is_scoped_to_one_release(): + """Without both filters a panel sums this release with any other Ray or + Triton release in the cluster — silently, and plausibly.""" + for title, expr in exprs(): + assert 'release=~"$release"' in expr, title + assert 'namespace=~"$namespace"' in expr, title + + +def test_panels_group_only_by_labels_that_exist(): + """Ray's internal tags (deployment, replica, application) are not what the + AF Prometheus keys on; pod/release/namespace are added by its relabeling, + and method/model are declared tag keys of the exporters.""" + for title, expr in exprs(): + for clause in BY.findall(expr): + for label in (label.strip() for label in clause.split(",")): + assert label in GROUPABLE, f"{title}: groups by {label!r}" + + +def test_rate_windows_follow_the_dashboard_interval(): + """A hardcoded window under-samples when someone zooms out to a week.""" + for title, expr in exprs(): + if "rate(" in expr: + assert "[$__rate_interval]" in expr, title + + +def test_divisions_cannot_produce_infinity(): + """An idle release divides by a zero rate; clamp_min keeps the panel from + rendering ±∞ (the failure the pixi panels already learned).""" + for title, expr in exprs(): + for denominator in expr.split(" / ")[1:]: + denominator = denominator.lstrip() + if re.match(r"^[0-9.]+\b", denominator): + continue # a constant (unit conversion) is never zero + assert denominator.startswith("clamp_min("), ( + f"{title}: unguarded division by {denominator.strip()[:60]}" + ) + + +def test_overhead_panel_compares_like_with_like(): + """The point of the dashboard: Serve's latency minus Triton's own. Serve + reports milliseconds, Triton microseconds — the conversion is what makes + the subtraction mean anything.""" + (panel,) = [p for p in panels() if p["title"] == "Ray overhead"] + (expr,) = [t["expr"] for t in panel["targets"]] + serve, _, triton = expr.partition(" - ") + assert "ray_serve_deployment_processing_latency_ms_sum" in serve + assert "nv_inference_request_duration_us" in triton + assert triton.rstrip().endswith("/ 1000"), "µs are not ms" + assert panel["fieldConfig"]["defaults"]["unit"] == "ms" + # Overhead is the only panel that may legitimately go negative (clock + # skew, different windows); it must not be clipped at zero. + assert "min" not in panel["fieldConfig"]["defaults"] + + +def test_autoscaling_panel_shows_the_signal_and_the_response(): + """Replicas alone do not explain a scaling decision; the in-flight average + is what Serve acts on and queued requests are what it failed to absorb.""" + (panel,) = [p for p in panels() if p["title"] == "Autoscaling"] + legends = {t["legendFormat"] for t in panel["targets"]} + assert {"replicas", "in flight", "queued"} == legends + + +def test_every_querying_panel_uses_the_af_datasource(): + """The AF Grafana provisions exactly one Prometheus, uid 'prometheus'. Rows + carry no datasource of their own, which is correct.""" + expected = {"type": "prometheus", "uid": "prometheus"} + for panel in panels(): + assert panel["datasource"] == expected, panel["title"] + for target in panel.get("targets", []): + assert target["datasource"] == expected, panel["title"] + assert all( + "datasource" not in p for p in dashboard()["panels"] if p["type"] == "row" + ) + + +def test_panels_do_not_overlap(): + """Two panels on the same square is a rendering bug that only shows in the + browser, never in review.""" + taken = {} + for panel in dashboard()["panels"]: + pos = panel["gridPos"] + for x in range(pos["x"], pos["x"] + pos["w"]): + for y in range(pos["y"], pos["y"] + pos["h"]): + assert (x, y) not in taken, ( + f"{panel['title']!r} overlaps {taken[(x, y)]!r} at {(x, y)}" + ) + taken[(x, y)] = panel["title"] + assert pos["x"] + pos["w"] <= 24, panel["title"]