diff --git a/.github/workflows/validate-manifests.sh b/.github/workflows/validate-manifests.sh index 2a968214..3d6fc395 100755 --- a/.github/workflows/validate-manifests.sh +++ b/.github/workflows/validate-manifests.sh @@ -12,6 +12,10 @@ # the chart version, HelmRepository URL and valuesFrom-ConfigMap values # resolved from that same stream — this is what catches breaking chart # schema changes when a chart version is bumped (e.g. by Renovate). +# Charts that live in this repository (sourced from its GitRepository, +# path ./apps/...) are rendered from the working tree with the same +# values, so a broken template or a values/template mismatch fails here +# rather than in the cluster. # 5. `promtool check rules` on Prometheus alerting/recording rules embedded # in Helm values — a typo'd PromQL expression otherwise deploys silently # and the alert simply never fires. @@ -114,25 +118,33 @@ validate_helmreleases() { src_name=$(yq "select(.kind==\"HelmRelease\" and .metadata.name==\"$name\") | .spec.chart.spec.sourceRef.name" "$rendered") src_kind=$(yq "select(.kind==\"HelmRelease\" and .metadata.name==\"$name\") | .spec.chart.spec.sourceRef.kind" "$rendered") - if [[ "$src_kind" != "HelmRepository" ]]; then + # A chart in this repository is rendered from the working tree. Only + # ./apps/... paths qualify: other GitRepository sources (supersonic-dev, + # servicex-dev) point at charts in someone else's repository. + local_chart="" + if [[ "$src_kind" == "GitRepository" && "$chart" == ./apps/* && -f "$chart/Chart.yaml" ]]; then + local_chart="$chart" + elif [[ "$src_kind" != "HelmRepository" ]]; then echo " skip ${name}: chart sourced from ${src_kind} '${src_name}' (no registry version to validate)" continue fi - if [[ -z "$version" || "$version" == "null" ]]; then - echo "✗ ${name}: chart version is not pinned (omitted version = Flux silently tracks latest)" >&2 - failed=1 - continue - fi + if [[ -z "$local_chart" ]]; then + if [[ -z "$version" || "$version" == "null" ]]; then + echo "✗ ${name}: chart version is not pinned (omitted version = Flux silently tracks latest)" >&2 + failed=1 + continue + fi - repo_line=$(grep -m1 "^${src_name}|" "$repos_file" || true) - if [[ -z "$repo_line" ]]; then - echo "✗ ${name}: HelmRepository '${src_name}' not found in any rendered environment" >&2 - failed=1 - continue + repo_line=$(grep -m1 "^${src_name}|" "$repos_file" || true) + if [[ -z "$repo_line" ]]; then + echo "✗ ${name}: HelmRepository '${src_name}' not found in any rendered environment" >&2 + failed=1 + continue + fi + repo_type=$(cut -d'|' -f2 <<<"$repo_line") + repo_url=$(cut -d'|' -f3- <<<"$repo_line") fi - repo_type=$(cut -d'|' -f2 <<<"$repo_line") - repo_url=$(cut -d'|' -f3- <<<"$repo_line") # Resolve valuesFrom ConfigMaps (generated by kustomize from values.yaml # files) into temp files, in order. @@ -167,6 +179,19 @@ validate_helmreleases() { done fi + if [[ -n "$local_chart" ]]; then + echo " helm template ${name} (${local_chart}, from this repository)" + if ! out=$(helm template "$name" "$local_chart" \ + --kube-version "$KUBE_VERSION" \ + --namespace cms \ + ${values_args[@]+"${values_args[@]}"} 2>&1); then + printf '%s\n' "$out" >&2 + echo "✗ ${name}: helm template failed" >&2 + failed=1 + fi + continue + fi + local fingerprint="${repo_url}|${chart}|${version}|${vhash}" if [[ "$seen_releases" == *"$fingerprint"* ]]; then continue diff --git a/README.md b/README.md index 9f115c4e..b2dbdb69 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ Whether each component on the cluster is running what is on `main` ![supersonic][experimental-sonic-supersonic] ![supersonic-dev][experimental-sonic-supersonic-dev] ![model-manager][experimental-sonic-model-manager] +![kuberay-operator][experimental-ray-operator] +![sonic-ray][experimental-ray-sonic-ray] **Images** — `purdue-af` is released on its own semver stream and pinned at ![AF image][af-image-version]. `agentic-interface` is auto-versioned (every @@ -136,6 +138,8 @@ How a change reaches the cluster, version rules and rollback: [experimental-sonic-model-manager]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/experimental-sonic-model-manager.json [experimental-sonic-supersonic]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/experimental-sonic-supersonic.json [experimental-sonic-supersonic-dev]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/experimental-sonic-supersonic-dev.json +[experimental-ray-operator]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/experimental-ray-operator.json +[experimental-ray-sonic-ray]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/experimental-ray-sonic-ray.json [image-purdue-af]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/image-purdue-af.json [image-agentic-interface]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/image-agentic-interface.json [image-af-pod-monitor]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/PurdueAF/purdue-af/status/badges/image-af-pod-monitor.json diff --git a/apps/ray/README.md b/apps/ray/README.md new file mode 100644 index 00000000..c75f2d8b --- /dev/null +++ b/apps/ray/README.md @@ -0,0 +1,188 @@ +# Ray on the Analysis Facility + +**Triton on Ray**: every worker pod carries NVIDIA's Triton Inference Server, +configured exactly as in the `supersonic` release (`apps/sonic/supersonic`) — +same image, arguments, resources, model repository — and **Ray Serve's gRPC proxy carries Triton's protocol** to it. +Serve speaks that protocol because it is handed Triton's own generated +servicer; the only code of ours is a forwarder that passes each RPC from the +proxy to the Triton in its pod. Serve counts every request on the way through, +sizes the deployment from that, and the Ray autoscaler adds a GPU pod for each +replica with nowhere to go. + +No custom image, no protocol code, no model code: official Ray, official +Triton, ~100 lines of glue shipped as a ConfigMap, Triton's Python stubs +pip-installed by an init container. + +| path | what it is | +| --- | --- | +| `helmrepo.yaml` | `HelmRepository` for the KubeRay charts | +| `operator/` | `kuberay-operator` 1.7.0 — the `ray.io` CRDs and the controller. Namespaced (`singleNamespaceInstall: true`), so both the watch and the RBAC stay in `cms`. | +| `sonic-ray/chart/` | the `sonic-ray` chart: a `RayService` with a Triton in every worker pod and the forwarder as its Serve application, the ConfigMap carrying the forwarder, two metrics Services | +| `sonic-ray/chart/files/sonic_ray/serve_app.py` | the forwarder — one replica per pod, every unary RPC of `GRPCInferenceService` handed to the pod's Triton unchanged | +| `sonic-ray/helmrelease.yaml`, `sonic-ray/values.yaml` | the AF release: `dependsOn` the operator, values with the `triton:` block of the `supersonic` release's values | +| [`tests/sonic_ray/`](../../tests/sonic_ray), [`tests/manifests/test_ray.py`](../../tests/manifests/test_ray.py) | source-level checks of the forwarder; rendered-chart checks incl. parity with `apps/sonic/supersonic/values.yaml` | + +The chart lives here (like `apps/sonic/model-manager`) rather than being a raw +`RayService` because of ordering: until the operator's chart has installed the +`ray.io` CRDs that is an unknown kind, and kustomize-controller aborts an apply +on the first one it meets — on a fresh cluster, before the HelmRelease that +would install them. `dependsOn: kuberay-operator` is the fix, and a +`HelmRelease` is the only object that can carry it. + +## Shape + +``` + clients ──▶ sonic-ray-serve (LoadBalancer, private pool) :8001 + │ Triton gRPC: ModelInfer, ModelMetadata, … + ▼ + Serve gRPC proxy (Triton's servicer) on head and every worker + │ counted, balanced, autoscaled by Serve + ▼ + ┌─────────────────────────────────────────────────┐ × 1…4 pods + │ worker pod │ + │ ray-worker raylet advertising triton: 1, │ + │ proxy, TritonProxy replica ──┐ │ + │ triton 1 GPU, 4 CPU, 16G, /cvmfs ro ◀┘ │ localhost:8001 + └─────────────────────────────────────────────────┘ + ▲ replica demand + ┌─────────────────────────────────────────────────┐ + │ head pod: Serve controller, Ray autoscaler │ (0 CPUs for work, no GPU) + └─────────────────────────────────────────────────┘ +``` + +A **pod is one Triton on one GPU**, the unit SuperSONIC scales by too. A +**replica is one pod**: every worker advertises one `triton` resource and +every replica claims one, so a replica lands next to its Triton and nowhere +else. Nothing else claims the resource, which is what leaves a pod without a +replica idle and therefore reclaimable, and a replica without a pod pending — +the request that grows the group. + +## What it serves and speaks + +Whatever Triton is pointed at. On the AF that is the models CMSSW ships: +`sonic-ray/values.yaml` mounts the cluster's CVMFS claim read-only at `/cvmfs` +and gives Triton four `--model-repository` directories inside a CMSSW release +(`CMSSW_17_0_0_pre2` today — RecoBTag, RecoEgamma, RecoTauTag, RecoMET) with +an explicit load list: + +`deepmet`, `deeptau_2018v2p5`, `particleNetFromMiniAODAK4CHSCentral`, +`particleNetFromMiniAODAK4PuppiCentral`, `particleNetFromMiniAODAK4PuppiForward`, +`particleNetFromMiniAODAK8`, `particlenet_AK8_MD-2prong_PT`, +`particlenet_AK8_MassRegression_PT`, `particlenet_PT`, +`unifiedparticletransformer_AK4_V01`. + +Nothing is uploaded anywhere and there is no model manager: a new CMSSW +release, or a different model set, is a path change in the values. Every +backend, `config.pbtxt` semantics, dynamic batching and the repository index +work as in any Triton, because it is Triton. The first load from CVMFS pulls +the files over the network into the node's cache, so the startup probe allows +four minutes. + +The wire protocol is Triton's gRPC. HTTP is **not** carried (Serve's HTTP +proxy on 8000 answers only its own `/-/healthz` and `/-/routes`); Triton's +HTTP port stays inside the pod. CMSSW's `TritonClient` speaks gRPC, so +`cmsRun` jobs point at `sonic-ray-serve:8001` as at any Triton endpoint — the +port is Triton's conventional one on purpose. `tritonclient.grpc` works the +same way. + +The one RPC not forwarded is `ModelStreamInfer`, Triton's bidirectional +stream: Serve's proxy carries unary and server-streaming calls only. CMSSW +uses the unary `ModelInfer`. + +## Autoscaling + +Two loops, both Ray's, nothing else in between: + +1. **Ray Serve** sizes the deployment from the requests its gRPC proxy + forwards. When the average number in flight per replica exceeds + `serve.targetOngoingRequests` (16) for `upscaleDelayS` (10 s) it adds a + replica; when it falls well below for `downscaleDelayS` (300 s) it removes + one, giving in-flight requests `gracefulShutdownTimeoutS` (60 s). Bounds are + `replicas.min`/`max` (1/4 on the AF). +2. **The Ray autoscaler** sizes the cluster. A new replica needs a `triton` + resource; if no worker has one free, that is a pending request and the + autoscaler adds a pod to `gpu-group` (ceiling: the same `replicas.max`; the + group's own floor is 0, since a pod with a replica on it is never idle and + Serve's minimum therefore keeps pods alive). A worker whose replica is gone idles for + `idleTimeoutSeconds` (60 s) and is reclaimed; the pod then gets + `terminationGracePeriodSeconds` against Triton's `--exit-timeout-secs` to + drain (the chart refuses to render if the first is not larger). + +One pair of numbers sizes both, because a replica *is* a pod. Raising the GPU +ceiling is one edit in `sonic-ray/values.yaml`: + +```yaml +replicas: { min: 1, max: 8 } +``` + +A replica only becomes ready once its Triton answers `ServerReady`, and it +polls `ServerLive` as its health check, so Serve never routes to a pod whose +Triton is still loading or has died — Serve restarts the replica, and Ray +reclaims a pod that stays broken. + +## How it lines up with SuperSONIC + +| SuperSONIC (`supersonic`) | Ray (`sonic-ray`) | +| --- | --- | +| Triton on a per-site PVC or CVMFS, explicit load list | Triton on CVMFS, explicit load list — a plain `--model-repository` path | +| Envoy: gRPC entry point behind a `LoadBalancer` on `geddes-private-pool`, `ROUND_ROBIN` | Serve's gRPC proxy behind KubeRay's serve Service, same pool, port 8001 | +| `ingress.enabled: false` — private pool only | no ingress; the head is `ClusterIP`, dashboard by port-forward only | +| KEDA `ScaledObject` on a Prometheus expression, 1–10 pods | Ray Serve request-based autoscaling, 1–4 pods — see above | +| `nodeSelector: cms-af-prod=true` + the `hub.jupyter.org/dedicated` toleration | same, head and workers | +| model repository from a PVC or CVMFS | the cluster's `cvmfs` claim, mounted **read-only** | +| Triton Service labelled `scrape_metrics: "true"` | `sonic-ray-triton-metrics` (`nv_*`) and `sonic-ray-metrics` (Ray, incl. `ray_serve_*`), same label, `release="sonic-ray"` | +| Envoy's Lua rate limiter on `RepositoryIndex` | none; Serve's `maxOngoingRequests` back-pressure instead | + +## Using it + +```bash +kubectl -n cms get svc sonic-ray-serve # MetalLB address on the private pool +SONIC=
:8001 +``` + +CMSSW clients point at `$SONIC`, exactly as they point at the supersonic +release's Envoy address. From Python: + +```python +import tritonclient.grpc as grpcclient + +client = grpcclient.InferenceServerClient("
:8001") +client.is_server_ready() +client.get_model_repository_index() +``` + +The Ray dashboard, for Serve and autoscaler state: + +```bash +kubectl -n cms port-forward svc/sonic-ray-head-svc 8265:8265 +``` + +## What is not an image + +The Ray containers run `rayproject/ray:2.52.0-py312-cpu` (through the geddes +Docker Hub proxy cache) exactly as published; the Triton container runs the +image the values name (the chart default is `nvcr.io/nvidia/tritonserver`; +the AF values use the lighter `docexoty/tritonserver:light`). Two things +are added at deploy time instead of build time: + +- **the forwarder** — `files/sonic_ray/*.py` become the `sonic-ray-code` + ConfigMap, mounted at `/serve_app/sonic_ray` on head and workers. Its hash + is annotated onto both pod templates, so a code change rolls the cluster. +- **Triton's Python stubs** — `python.pip` (`tritonclient==2.48.0`, the last + release whose generated stubs match the protobuf 4 in the Ray image, plus + `python-rapidjson`) is pip-installed `--no-deps --target` into an emptyDir + by an init container on every pod, and that directory is on `PYTHONPATH`. + Serve's proxies import the servicer from it at startup, on every node, + which is why a `runtime_env` (replicas only) would not do. + +The price is a small pip download per pod start and a dependency on PyPI +being reachable from the nodes — chosen over maintaining an image. + +## Cost + +One GPU idles (`replicas.min: 1`) on the same `cms-af-prod` nodes +SuperSONIC and the user sessions compete for. An upgrade costs a second set +for its duration: `upgradeStrategy: NewCluster` brings a second cluster up +before cutting over, and if no GPU is free it waits while the old one keeps +serving. A GPU node here has 128 cores, so the two extra CPUs the Ray +container adds to each pod change nothing about what fits. diff --git a/apps/ray/helmrepo.yaml b/apps/ray/helmrepo.yaml new file mode 100644 index 00000000..1d3261c7 --- /dev/null +++ b/apps/ray/helmrepo.yaml @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: kuberay +spec: + interval: 1h + url: https://ray-project.github.io/kuberay-helm/ diff --git a/apps/ray/operator/helmrelease.yaml b/apps/ray/operator/helmrelease.yaml new file mode 100644 index 00000000..176748e4 --- /dev/null +++ b/apps/ray/operator/helmrelease.yaml @@ -0,0 +1,29 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: kuberay-operator +spec: + suspend: false + interval: 1m + chart: + spec: + chart: kuberay-operator + version: "1.7.0" + sourceRef: + kind: HelmRepository + name: kuberay + interval: 1m + install: + # The ray.io CRDs (RayCluster/RayJob/RayService/RayCronJob) ship in the + # chart's crds/ directory; nothing else in the cluster installs them. + crds: Create + remediation: + retries: -1 + upgrade: + crds: CreateReplace + remediation: + retries: -1 + valuesFrom: + - kind: ConfigMap + name: kuberay-operator-config + valuesKey: values.yaml diff --git a/apps/ray/operator/values.yaml b/apps/ray/operator/values.yaml new file mode 100644 index 00000000..3d1b0d7a --- /dev/null +++ b/apps/ray/operator/values.yaml @@ -0,0 +1,24 @@ +# KubeRay operator for the AF. Deployed only so RayCluster CRs can be +# reconciled — see apps/ray/sonic-ray for the cluster itself. + +# Watch the release namespace (cms) only, and create namespaced Role/RoleBinding +# instead of cluster-wide ClusterRole/ClusterRoleBinding. The AF has no Ray +# workloads outside cms, so there is no reason to grant cluster scope. +singleNamespaceInstall: true + +# The operator is a controller, not a data-plane component: keep it off the +# GPU nodes' dedicated taint and let it land anywhere in the cluster. +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + +metrics: + enabled: true + # The AF Prometheus scrapes by Service label (scrape_metrics: "true"), + # not through prometheus-operator ServiceMonitors. + serviceMonitor: + enabled: false diff --git a/apps/ray/sonic-ray/chart/Chart.yaml b/apps/ray/sonic-ray/chart/Chart.yaml new file mode 100644 index 00000000..66068849 --- /dev/null +++ b/apps/ray/sonic-ray/chart/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: sonic-ray +description: Triton Inference Server on Ray — Ray Serve's gRPC proxy carries Triton's protocol to a Triton in every GPU pod, and Ray autoscales the pods from the requests. +type: application +version: 0.1.0 +appVersion: "26.04" +home: https://github.com/PurdueAF/purdue-af/tree/main/apps/ray +annotations: + artifacthub.io/category: ai-machine-learning diff --git a/apps/ray/sonic-ray/chart/files/sonic_ray/__init__.py b/apps/ray/sonic-ray/chart/files/sonic_ray/__init__.py new file mode 100644 index 00000000..be6b0d52 --- /dev/null +++ b/apps/ray/sonic-ray/chart/files/sonic_ray/__init__.py @@ -0,0 +1,12 @@ +"""sonic-ray: Triton on Ray, with Ray Serve carrying the gRPC traffic. + +One module, ``serve_app``: a Ray Serve deployment that forwards every unary +RPC of Triton's ``GRPCInferenceService`` to the Triton running beside it in +the same pod. Ray Serve's gRPC proxy speaks Triton's protocol because it is +handed Triton's own generated servicer (from the ``tritonclient`` package); +Triton does every bit of the inference. Nothing here parses a request. + +The file ships to the cluster as a ConfigMap rendered by the chart (see +templates/configmap.yaml) and lands on PYTHONPATH in the stock Ray image; +there is no custom image. +""" diff --git a/apps/ray/sonic-ray/chart/files/sonic_ray/serve_app.py b/apps/ray/sonic-ray/chart/files/sonic_ray/serve_app.py new file mode 100644 index 00000000..89f59204 --- /dev/null +++ b/apps/ray/sonic-ray/chart/files/sonic_ray/serve_app.py @@ -0,0 +1,101 @@ +"""Triton behind Ray Serve's gRPC proxy. + +Every worker pod runs two containers: Triton and Ray. This deployment runs on the Ray side, +one replica per pod (pinned there by the ``triton`` resource each worker +advertises), and forwards each RPC it receives to Triton on localhost. + +Ray Serve's gRPC proxy is configured (in the chart's serveConfigV2) with +Triton's generated ``add_GRPCInferenceServiceServicer_to_server``, so it +accepts exactly Triton's protocol; it dispatches each call to the method of +this class with the RPC's name, and that method hands the protobuf message +to Triton and returns Triton's protobuf answer. Serve counts the request on +the way through, which is what its autoscaler and load balancing key on. + +``ModelStreamInfer`` — Triton's one bidirectional stream — is not forwarded: +Serve's proxy carries unary and server-streaming calls only. CMSSW's client +uses the unary ``ModelInfer``. +""" + +from __future__ import annotations + +import logging +import os +import time +from typing import Any + +import grpc +from ray import serve +from tritonclient.grpc import service_pb2, service_pb2_grpc + +LOGGER = logging.getLogger("sonic_ray") + +# Triton's gRPC endpoint in this pod, and how long a fresh Triton may take to +# load the repository before the replica gives up on it. +TRITON = os.environ.get("TRITON_GRPC", "localhost:8001") +READY_TIMEOUT_S = float(os.environ.get("TRITON_READY_TIMEOUT_S", "900")) +# Inference payloads are large; match Serve's own proxy limit rather than +# grpc's 4 MB default. +MAX_MESSAGE_BYTES = 2**31 - 1 +CHANNEL_OPTIONS = [ + ("grpc.max_send_message_length", MAX_MESSAGE_BYTES), + ("grpc.max_receive_message_length", MAX_MESSAGE_BYTES), +] + +# Every RPC of Triton's service except the bidirectional stream. +RPCS = tuple( + name + for name in vars(service_pb2_grpc.GRPCInferenceServiceServicer) + if not name.startswith("_") and name != "ModelStreamInfer" +) + + +class TritonProxy: + """One replica = one pod = one Triton; every RPC goes to it unchanged.""" + + def __init__(self) -> None: + self._sync = service_pb2_grpc.GRPCInferenceServiceStub( + grpc.insecure_channel(TRITON, options=CHANNEL_OPTIONS) + ) + self._stub = service_pb2_grpc.GRPCInferenceServiceStub( + grpc.aio.insecure_channel(TRITON, options=CHANNEL_OPTIONS) + ) + self._wait_for_triton() + + def _wait_for_triton(self) -> None: + """Block until Triton answers ServerReady: the replica is not ready + until its Triton is, so Serve never routes to a still-loading pod.""" + deadline = time.monotonic() + READY_TIMEOUT_S + while True: + try: + if self._sync.ServerReady( + service_pb2.ServerReadyRequest(), timeout=5 + ).ready: + LOGGER.info("triton at %s is ready", TRITON) + return + except grpc.RpcError as exc: + if time.monotonic() > deadline: + raise RuntimeError( + f"triton at {TRITON} not ready after {READY_TIMEOUT_S}s" + ) from exc + time.sleep(2) + + def check_health(self) -> None: + """Serve restarts the replica — and Ray then reclaims the pod — when + its Triton stops answering.""" + if not self._sync.ServerLive(service_pb2.ServerLiveRequest(), timeout=5).live: + raise RuntimeError(f"triton at {TRITON} is not live") + + +def _forwarder(rpc: str) -> Any: + async def forward(self: TritonProxy, request: Any) -> Any: + return await getattr(self._stub, rpc)(request) + + forward.__name__ = rpc + return forward + + +for _rpc in RPCS: + setattr(TritonProxy, _rpc, _forwarder(_rpc)) + +# What serveConfigV2's import_path points at: `sonic_ray.serve_app:triton`. +triton = serve.deployment(TritonProxy).bind() diff --git a/apps/ray/sonic-ray/chart/templates/_helpers.tpl b/apps/ray/sonic-ray/chart/templates/_helpers.tpl new file mode 100644 index 00000000..fa711959 --- /dev/null +++ b/apps/ray/sonic-ray/chart/templates/_helpers.tpl @@ -0,0 +1,130 @@ +{{/* +Instance name (equal to release name unless overridden) +*/}} +{{- define "sonic-ray.name" -}} +{{- default .Release.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "sonic-ray.labels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ include "sonic-ray.name" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +What the Services select pods by. Deliberately not app.kubernetes.io/name: +KubeRay stamps its own value for that key onto every pod it creates, and +whether a template's value survives is a detail of the operator version. +app.kubernetes.io/instance and /component are ours alone. +*/}} +{{- define "sonic-ray.podSelector" -}} +app.kubernetes.io/instance: {{ include "sonic-ray.name" . }} +{{- end -}} + +{{- define "sonic-ray.rayImage" -}} +{{ .Values.ray.image.repository }}:{{ .Values.ray.version }}-{{ .Values.ray.image.flavor }} +{{- end -}} + +{{- define "sonic-ray.tritonImage" -}} +{{ .Values.triton.image.repository }}:{{ .Values.triton.image.tag }} +{{- end -}} + +{{/* +Where the forwarder's ConfigMap and the pip-installed stubs are mounted; both +are on PYTHONPATH. +*/}} +{{- define "sonic-ray.codeDir" -}}/serve_app{{- end -}} +{{- define "sonic-ray.depsDir" -}}/python-deps{{- end -}} + +{{/* +Hash of the forwarder's source, annotated onto both pod templates so a code +change rolls the cluster like any other change to it would. +*/}} +{{- define "sonic-ray.codeChecksum" -}} +{{ (.Files.Glob "files/sonic_ray/*.py").AsConfig | sha256sum }} +{{- end -}} + +{{/* +Shared by head and worker Ray containers: environment, the code + deps +mounts, and the init container that pip-installs Triton's stubs. +*/}} +{{- define "sonic-ray.rayEnv" -}} +- name: PYTHONPATH + value: {{ printf "%s:%s" (include "sonic-ray.codeDir" .) (include "sonic-ray.depsDir" .) | quote }} +{{- end -}} + +{{- define "sonic-ray.rayMounts" -}} +- { name: log-volume, mountPath: /tmp/ray } +- name: code + mountPath: {{ include "sonic-ray.codeDir" . }}/sonic_ray + readOnly: true +- name: python-deps + mountPath: {{ include "sonic-ray.depsDir" . }} + readOnly: true +{{- end -}} + +{{- define "sonic-ray.rayVolumes" -}} +- name: log-volume + emptyDir: {} +- name: code + configMap: + name: {{ include "sonic-ray.name" . }}-code +- name: python-deps + emptyDir: {} +{{- end -}} + +{{- define "sonic-ray.pipInitContainer" -}} +# Triton's generated gRPC servicer must be importable by Serve's proxies, +# which run outside any runtime_env — so it goes on PYTHONPATH for the +# whole pod. --no-deps keeps the image's grpcio/protobuf/numpy in charge. +- name: pip-install + image: {{ include "sonic-ray.rayImage" . }} + imagePullPolicy: {{ .Values.ray.image.pullPolicy }} + command: ["pip", "install", "--no-cache-dir", "--no-deps", "--target", {{ include "sonic-ray.depsDir" . | quote }}] + args: + {{- toYaml .Values.python.pip | nindent 4 }} + volumeMounts: + - name: python-deps + mountPath: {{ include "sonic-ray.depsDir" . }} + resources: + limits: { cpu: "1", memory: 1Gi } + requests: { cpu: 100m, memory: 256Mi } +{{- end -}} + +{{/* +Refuse to render what cannot work. +*/}} +{{- define "sonic-ray.validate" -}} +{{- if not (.Files.Glob "files/sonic_ray/*.py") -}} + {{- fail "files/sonic_ray/*.py is empty: nothing to serve." -}} +{{- end -}} +{{- if not (regexMatch "(^| )tritonclient==" (join " " .Values.python.pip)) -}} + {{- fail "python.pip must pin tritonclient==: it carries Triton's gRPC servicer for Serve's proxy." -}} +{{- end -}} +{{- if not .Values.triton.modelRepository.claimName -}} + {{- fail "triton.modelRepository.claimName is required: the PVC holding the Triton model repository." -}} +{{- end -}} +{{- if gt (int .Values.replicas.min) (int .Values.replicas.max) -}} + {{- fail "replicas.min exceeds replicas.max." -}} +{{- end -}} +{{- if lt (int .Values.replicas.min) 0 -}} + {{- fail "replicas.min is negative." -}} +{{- end -}} +{{- $gpus := index .Values.triton.resources.limits "nvidia.com/gpu" | default 0 | int -}} +{{- if ne $gpus 1 -}} + {{- fail "triton.resources.limits must request exactly one nvidia.com/gpu: a pod is one Triton on one GPU." -}} +{{- end -}} +{{- $args := join " " .Values.triton.args -}} +{{- if not (contains .Values.triton.modelRepository.mountPath $args) -}} + {{- fail (printf "triton.args never mention triton.modelRepository.mountPath (%s): Triton would not see the repository that is mounted." .Values.triton.modelRepository.mountPath) -}} +{{- end -}} +{{- with regexFind "--exit-timeout-secs=[0-9]+" $args -}} + {{- $exit := trimPrefix "--exit-timeout-secs=" . | int -}} + {{- if le (int $.Values.ray.worker.terminationGracePeriodSeconds) $exit -}} + {{- fail (printf "ray.worker.terminationGracePeriodSeconds (%d) must exceed Triton's --exit-timeout-secs (%d), or a scale-down kills in-flight requests." (int $.Values.ray.worker.terminationGracePeriodSeconds) $exit) -}} + {{- end -}} +{{- end -}} +{{- if le (int .Values.ray.worker.terminationGracePeriodSeconds) (int .Values.serve.gracefulShutdownTimeoutS) -}} + {{- fail (printf "ray.worker.terminationGracePeriodSeconds (%d) must exceed serve.gracefulShutdownTimeoutS (%d)." (int .Values.ray.worker.terminationGracePeriodSeconds) (int .Values.serve.gracefulShutdownTimeoutS)) -}} +{{- end -}} +{{- end -}} diff --git a/apps/ray/sonic-ray/chart/templates/configmap.yaml b/apps/ray/sonic-ray/chart/templates/configmap.yaml new file mode 100644 index 00000000..d06e404d --- /dev/null +++ b/apps/ray/sonic-ray/chart/templates/configmap.yaml @@ -0,0 +1,16 @@ +# The server code (files/sonic_ray), mounted into every pod as the package +# directory {{ include "sonic-ray.codeDir" . }}/sonic_ray. The checksum of +# these files is annotated onto the pod templates, so a code change rolls the +# cluster like any other change to it would. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "sonic-ray.name" . }}-code + namespace: {{ .Release.Namespace }} + labels: + {{- include "sonic-ray.labels" . | nindent 4 }} +data: +{{- range $path, $_ := .Files.Glob "files/sonic_ray/*.py" }} + {{ base $path }}: | +{{ $.Files.Get $path | indent 4 }} +{{- end }} diff --git a/apps/ray/sonic-ray/chart/templates/rayservice.yaml b/apps/ray/sonic-ray/chart/templates/rayservice.yaml new file mode 100644 index 00000000..d791e91e --- /dev/null +++ b/apps/ray/sonic-ray/chart/templates/rayservice.yaml @@ -0,0 +1,216 @@ +{{- include "sonic-ray.validate" . -}} +{{- $name := include "sonic-ray.name" . -}} +# The cluster, its Tritons, and the Serve application that fronts them. +# +# A RayService rather than a RayCluster because the forwarder is a Ray Serve +# application, and RayService is what runs one declaratively — with a health +# check on the application, and with NewCluster upgrades that only cut over +# once it answers. +# +# One custom Ray resource does the placement: +# triton — one per worker pod. A forwarding replica claims it, so exactly +# one replica lands in each pod, next to that pod's Triton. A +# replica with no `triton` left to claim is a pending resource +# request, and the Ray autoscaler adds a worker pod for it. +apiVersion: ray.io/v1 +kind: RayService +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sonic-ray.labels" . | nindent 4 }} +spec: + upgradeStrategy: + type: {{ .Values.ray.upgradeStrategy }} + + serveConfigV2: | + # Serve's gRPC proxy, speaking Triton's protocol: the servicer function is + # Triton's own generated one (pip-installed onto PYTHONPATH by the init + # container). Each RPC is dispatched to the replica method of that name. + grpc_options: + port: 9000 + grpc_servicer_functions: + - tritonclient.grpc.service_pb2_grpc.add_GRPCInferenceServiceServicer_to_server + applications: + - name: sonic + import_path: sonic_ray.serve_app:triton + route_prefix: / + deployments: + - name: TritonProxy + # One replica per pod: it claims the pod's `triton` resource and + # forwards to the Triton beside it. num_cpus 0 keeps the worker's + # CPU count out of the placement decision. + ray_actor_options: + num_cpus: 0 + resources: + triton: 1 + max_ongoing_requests: {{ .Values.serve.maxOngoingRequests }} + graceful_shutdown_timeout_s: {{ .Values.serve.gracefulShutdownTimeoutS }} + health_check_period_s: {{ .Values.serve.healthCheckPeriodS }} + health_check_timeout_s: {{ .Values.serve.healthCheckTimeoutS }} + autoscaling_config: + min_replicas: {{ .Values.replicas.min }} + max_replicas: {{ .Values.replicas.max }} + target_ongoing_requests: {{ .Values.serve.targetOngoingRequests }} + upscale_delay_s: {{ .Values.serve.upscaleDelayS }} + downscale_delay_s: {{ .Values.serve.downscaleDelayS }} + + # KubeRay's Service for the Serve proxies — the inference entry point. It + # selects pods whose proxy is healthy, so a pod whose Triton is still + # loading is not in rotation. + serveService: + metadata: + name: {{ $name }}-serve + labels: + {{- include "sonic-ray.labels" . | nindent 8 }} + app.kubernetes.io/component: serve + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + type: {{ .Values.service.type }} + ports: + - name: grpc + port: 8001 + targetPort: 9000 + protocol: TCP + - name: http + port: 8000 + targetPort: 8000 + protocol: TCP + + rayClusterConfig: + rayVersion: {{ .Values.ray.version | quote }} + # Serve's replica demand lands here: a replica with no `triton` resource + # to claim is a pending request, and the autoscaler adds a worker for it. + enableInTreeAutoscaling: true + autoscalerOptions: + upscalingMode: Default + idleTimeoutSeconds: {{ .Values.ray.autoscaler.idleTimeoutSeconds }} + resources: + {{- toYaml .Values.ray.autoscaler.resources | nindent 8 }} + + headGroupSpec: + # Never exposed: the dashboard has no authentication. Inference has + # its own Service above. + serviceType: ClusterIP + rayStartParams: + num-cpus: "0" + dashboard-host: 0.0.0.0 + template: + metadata: + labels: + {{- include "sonic-ray.podSelector" . | nindent 12 }} + app.kubernetes.io/component: head + annotations: + checksum/code: {{ include "sonic-ray.codeChecksum" . }} + spec: + initContainers: + {{- include "sonic-ray.pipInitContainer" . | nindent 12 }} + containers: + - name: ray-head + image: {{ include "sonic-ray.rayImage" . }} + imagePullPolicy: {{ .Values.ray.image.pullPolicy }} + env: + {{- include "sonic-ray.rayEnv" . | nindent 16 }} + ports: + - { name: gcs-server, containerPort: 6379 } + - { name: client, containerPort: 10001 } + - { name: dashboard, containerPort: 8265 } + - { name: metrics, containerPort: 8080 } + - { name: serve, containerPort: 8000 } + - { name: serve-grpc, containerPort: 9000 } + resources: + {{- toYaml .Values.ray.head.resources | nindent 16 }} + volumeMounts: + {{- include "sonic-ray.rayMounts" . | nindent 16 }} + volumes: + {{- include "sonic-ray.rayVolumes" . | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} + + workerGroupSpecs: + - groupName: gpu-group + # One pod per replica, so the group is bounded by `replicas` too. It + # starts with as many pods as Serve will immediately ask for, and may + # shrink to none of its own accord: Serve's minimum is what keeps pods + # alive, since a pod with a replica on it is never idle. + replicas: {{ .Values.replicas.min }} + minReplicas: 0 + maxReplicas: {{ .Values.replicas.max }} + rayStartParams: + resources: '"{\"triton\": 1}"' + template: + metadata: + labels: + {{- include "sonic-ray.podSelector" . | nindent 14 }} + app.kubernetes.io/component: worker + annotations: + checksum/code: {{ include "sonic-ray.codeChecksum" . }} + spec: + terminationGracePeriodSeconds: {{ .Values.ray.worker.terminationGracePeriodSeconds }} + initContainers: + {{- include "sonic-ray.pipInitContainer" . | nindent 14 }} + containers: + # The Ray side of the pod: a raylet advertising the triton + # resource, a Serve proxy, and the forwarding replica. + - name: ray-worker + image: {{ include "sonic-ray.rayImage" . }} + imagePullPolicy: {{ .Values.ray.image.pullPolicy }} + env: + {{- include "sonic-ray.rayEnv" . | nindent 18 }} + ports: + - { name: metrics, containerPort: 8080 } + - { name: serve, containerPort: 8000 } + - { name: serve-grpc, containerPort: 9000 } + resources: + {{- toYaml .Values.ray.worker.resources | nindent 18 }} + volumeMounts: + {{- include "sonic-ray.rayMounts" . | nindent 18 }} + + # The inference server. Holds the pod's GPU; Ray never sees it. + - name: triton + image: {{ include "sonic-ray.tritonImage" . }} + imagePullPolicy: {{ .Values.triton.image.pullPolicy }} + command: + {{- toYaml .Values.triton.command | nindent 18 }} + args: + {{- toYaml .Values.triton.args | nindent 18 }} + ports: + - { name: http, containerPort: 8000 } + - { name: grpc, containerPort: 8001 } + - { name: triton-metrics, containerPort: 8002 } + resources: + {{- toYaml .Values.triton.resources | nindent 18 }} + startupProbe: + {{- toYaml .Values.triton.startupProbe | nindent 18 }} + readinessProbe: + {{- toYaml .Values.triton.readinessProbe | nindent 18 }} + volumeMounts: + - name: model-repository + mountPath: {{ .Values.triton.modelRepository.mountPath }} + readOnly: true + {{- with .Values.triton.modelRepository.mountPropagation }} + mountPropagation: {{ . }} + {{- end }} + volumes: + {{- include "sonic-ray.rayVolumes" . | nindent 14 }} + - name: model-repository + persistentVolumeClaim: + claimName: {{ .Values.triton.modelRepository.claimName }} + readOnly: true + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 14 }} + {{- end }} diff --git a/apps/ray/sonic-ray/chart/templates/services.yaml b/apps/ray/sonic-ray/chart/templates/services.yaml new file mode 100644 index 00000000..0b2b7c62 --- /dev/null +++ b/apps/ray/sonic-ray/chart/templates/services.yaml @@ -0,0 +1,51 @@ +{{- $name := include "sonic-ray.name" . -}} +# What the AF Prometheus scrapes, by the scrape_metrics label. Two Services +# because Ray and Triton expose /metrics on different ports and only the +# workers run Triton: one Service carrying both ports would have 8002 scraped +# on the head, where nothing listens. Neither can be the serve Service — a +# scrape job keyed on the label hits /metrics on every port of the Service it +# keeps. +# +# The app/instance labels become the Prometheus app/release labels, so the +# ray_serve_* and nv_* series here carry release="{{ $name }}". +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "sonic-ray.labels" . | nindent 4 }} + app.kubernetes.io/component: ray + scrape_metrics: "true" +spec: + type: ClusterIP + clusterIP: None + selector: + {{- include "sonic-ray.podSelector" . | nindent 4 }} + ports: + # Ray's own metrics — including Serve's per-deployment request counts, + # latencies and queue depths — from head and workers alike. + - name: metrics + port: 8080 + targetPort: metrics +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-triton-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "sonic-ray.labels" . | nindent 4 }} + app.kubernetes.io/component: triton + scrape_metrics: "true" +spec: + type: ClusterIP + clusterIP: None + selector: + {{- include "sonic-ray.podSelector" . | nindent 4 }} + app.kubernetes.io/component: worker + ports: + # Triton's nv_* metrics — the series the Triton dashboards plot. + - name: triton-metrics + port: 8002 + targetPort: triton-metrics diff --git a/apps/ray/sonic-ray/chart/values.yaml b/apps/ray/sonic-ray/chart/values.yaml new file mode 100644 index 00000000..ac1b265b --- /dev/null +++ b/apps/ray/sonic-ray/chart/values.yaml @@ -0,0 +1,147 @@ +# Default values for sonic-ray. +# +# A RayService whose worker pods each carry a Triton Inference Server beside +# the Ray container, and whose Serve application forwards every Triton gRPC +# call to the Triton in its own pod. Ray Serve's gRPC proxy speaks Triton's +# protocol (it is handed Triton's generated servicer), counts every request, +# and sizes the deployment from that; the Ray autoscaler adds a GPU pod for +# each replica with nowhere to go. Triton does all of the inference. +# +# No custom image: the Ray containers run the official Ray image, the Triton +# container the official Triton image, the forwarder reaches the pods as a +# ConfigMap on PYTHONPATH, and Triton's Python stubs are pip-installed into an +# emptyDir by an init container. + +# -- Instance name (release name by default). Names every resource and is the +# app.kubernetes.io/instance label the Services select pods by. +nameOverride: "" + +# -- How many Tritons: the one pair of numbers that sizes the release. A +# replica of the forwarder is one GPU pod with one Triton in it, so this +# bounds Serve's replica count and the worker group alike (the group's own +# minimum is always 0 — Serve's minimum is what keeps pods alive). Serve adds +# a replica when the average number of in-flight requests per replica exceeds +# serve.targetOngoingRequests, and removes one when it falls well below. +# min: 0 (scale-to-zero) works, but the first request then waits for a pod, +# two image pulls and Triton's model load. +replicas: + min: 1 + max: 4 + +ray: + # -- Ray version: selects the image tag below and pins the autoscaler sidecar. + version: "2.52.0" + image: + # -- The official Ray image, through the geddes Docker Hub proxy cache. + # Tag is -. No Ray process touches a GPU, so the CPU + # flavor; Triton brings its own CUDA. + repository: geddes-registry.rcac.purdue.edu/docker-hub-cache/rayproject/ray + flavor: py312-cpu + pullPolicy: IfNotPresent + # -- NewCluster brings a second cluster up and cuts over once its Serve + # application is healthy — zero downtime, at the cost of a second set of + # GPUs for the duration; if none are free it waits and the old cluster + # keeps serving. None never replaces the running cluster. + upgradeStrategy: NewCluster + head: + # -- GCS, dashboard, Serve controller and the proxies. No replicas run + # here: num-cpus is pinned to 0 and there is no Triton. + resources: + limits: { cpu: "2", memory: 8Gi } + requests: { cpu: "2", memory: 8Gi } + worker: + # -- The Ray side of a worker pod: a raylet, a Serve proxy and the one + # forwarding replica. Triton's own resources are under triton.resources. + resources: + limits: { cpu: "2", memory: 4Gi } + requests: { cpu: "2", memory: 4Gi } + # -- Scaling down deletes the pod. Triton drains in-flight requests for + # --exit-timeout-secs after SIGTERM; the kubelet must wait at least that + # long (the chart refuses to render otherwise). + terminationGracePeriodSeconds: 90 + autoscaler: + # -- How long a worker with no replica on it lingers before the Ray + # autoscaler reclaims it. + idleTimeoutSeconds: 60 + resources: + limits: { cpu: 500m, memory: 512Mi } + requests: { cpu: 500m, memory: 512Mi } + +python: + # -- Installed by an init container into an emptyDir that every Ray process + # sees on PYTHONPATH — proxies included, which is where Triton's generated + # servicer has to be importable. --no-deps: grpcio, protobuf, numpy and + # packaging come from the image, and tritonclient 2.48 is the last release + # whose stubs match the image's protobuf 4. + pip: + - tritonclient==2.48.0 + - python-rapidjson==1.24 + +serve: + # -- In-flight requests per replica above which Serve adds one (bounds are + # `replicas` above). Set near the concurrency at which one Triton saturates + # its GPU. + targetOngoingRequests: 16 + # -- Requests a replica queues before the proxy backs off to another. + maxOngoingRequests: 128 + # -- How fast Serve reacts. Up quickly; down slowly, since a replica gone is + # a GPU pod gone. + upscaleDelayS: 10 + downscaleDelayS: 300 + # -- In-flight requests get this long to finish when a replica is retired. + gracefulShutdownTimeoutS: 60 + # -- The forwarder polls its Triton's ServerLive on this schedule. + healthCheckPeriodS: 10 + healthCheckTimeoutS: 30 + +triton: + image: + repository: nvcr.io/nvidia/tritonserver + tag: "26.04-py3" + pullPolicy: IfNotPresent + # -- Passed to the Triton container as-is; the model repository path in args + # must be modelRepository.mountPath. + command: ["tritonserver"] + args: + - --model-repository=/models + - --exit-timeout-secs=60 + # -- One GPU per Triton is the unit of scaling: a pod is one Triton, and a + # Serve replica is one pod. + resources: + limits: { nvidia.com/gpu: 1 } + requests: { nvidia.com/gpu: 1 } + modelRepository: + # -- An existing claim holding the model repositories named in args — + # typically the cluster's CVMFS claim, with Triton pointed at model + # directories inside a CMSSW release. Mounted read-only: Triton only + # reads it, and this release does not own it. + claimName: "" + mountPath: /models + # -- Set to HostToContainer for CVMFS and other FUSE/autofs mounts, so + # repositories mounted on the host after the pod starts still appear. + mountPropagation: "" + # -- A large repository takes minutes to load; the startup probe buys that + # time without loosening readiness afterwards. + startupProbe: + httpGet: { path: /v2/health/ready, port: http } + periodSeconds: 10 + timeoutSeconds: 15 + failureThreshold: 60 + readinessProbe: + httpGet: { path: /v2/health/ready, port: http } + periodSeconds: 10 + timeoutSeconds: 15 + failureThreshold: 3 + successThreshold: 1 + +# -- Placement for every pod of the release, head and workers alike. +nodeSelector: {} +tolerations: [] + +# -- The inference entry point: KubeRay's serve Service, fronting Serve's +# proxies. gRPC on 8001 (Triton's conventional port, so clients keep their +# address:port habit) to the gRPC proxy; HTTP 8000 carries only Serve's own +# /-/healthz and /-/routes. The head's own Service is never exposed. +service: + type: LoadBalancer + annotations: {} diff --git a/apps/ray/sonic-ray/helmrelease.yaml b/apps/ray/sonic-ray/helmrelease.yaml new file mode 100644 index 00000000..41f03824 --- /dev/null +++ b/apps/ray/sonic-ray/helmrelease.yaml @@ -0,0 +1,33 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: sonic-ray +spec: + suspend: false + interval: 1m + # The chart renders a RayService, so the ray.io CRDs — and the operator that + # reconciles them — have to exist first. A raw RayService in the Flux + # Kustomization could not wait for that: kustomize-controller aborts an + # apply on the first kind the API server does not know, which on a fresh + # cluster is this one, before the HelmRelease that would install it. + dependsOn: + - name: kuberay-operator + chart: + spec: + # Chart lives in this repository, so it is sourced from the same + # GitRepository Flux already syncs for the experimental environment. + chart: ./apps/ray/sonic-ray/chart + sourceRef: + kind: GitRepository + name: purdue-af-experimental + interval: 1m + install: + remediation: + retries: -1 + upgrade: + remediation: + retries: -1 + valuesFrom: + - kind: ConfigMap + name: sonic-ray-config + valuesKey: values.yaml diff --git a/apps/ray/sonic-ray/values.yaml b/apps/ray/sonic-ray/values.yaml new file mode 100644 index 00000000..8318f369 --- /dev/null +++ b/apps/ray/sonic-ray/values.yaml @@ -0,0 +1,84 @@ +# Values for the `sonic-ray` release: Triton on Ray in the cms namespace on +# Geddes, with Ray Serve carrying its gRPC. Autoscaling is Ray's alone: no +# KEDA, no Prometheus in the loop. See the chart. +# +# The models are the ones CMSSW ships: Triton is pointed at model directories +# inside a CMSSW release on CVMFS, with an explicit load list. Nothing is +# uploaded anywhere; a new CMSSW release is a path change here. + +triton: + image: + repository: docker.io/docexoty/tritonserver + tag: light + command: ["/bin/sh", "-c"] + args: + - | + /opt/tritonserver/bin/tritonserver \ + --model-repository=/cvmfs/cms.cern.ch/el9_amd64_gcc13/cms/cmssw/CMSSW_17_0_0_pre2/external/el9_amd64_gcc13/data/RecoBTag/Combined/data/models/ \ + --model-repository=/cvmfs/cms.cern.ch/el9_amd64_gcc13/cms/cmssw/CMSSW_17_0_0_pre2/external/el9_amd64_gcc13/data/RecoEgamma/EgammaPhotonProducers/data/models/ \ + --model-repository=/cvmfs/cms.cern.ch/el9_amd64_gcc13/cms/cmssw/CMSSW_17_0_0_pre2/external/el9_amd64_gcc13/data/RecoTauTag/TrainingFiles/data/DeepTauIdSONIC/ \ + --model-repository=/cvmfs/cms.cern.ch/el9_amd64_gcc13/cms/cmssw/CMSSW_17_0_0_pre2/external/el9_amd64_gcc13/data/RecoMET/METPUSubtraction/data/models/ \ + --model-control-mode=explicit \ + --load-model=deepmet \ + --load-model=deeptau_2018v2p5 \ + --load-model=particleNetFromMiniAODAK4CHSCentral \ + --load-model=particleNetFromMiniAODAK4PuppiCentral \ + --load-model=particleNetFromMiniAODAK4PuppiForward \ + --load-model=particleNetFromMiniAODAK8 \ + --load-model=particlenet_AK8_MD-2prong_PT \ + --load-model=particlenet_AK8_MassRegression_PT \ + --load-model=particlenet_PT \ + --load-model=unifiedparticletransformer_AK4_V01 \ + --allow-gpu-metrics=true \ + --log-verbose=0 \ + --strict-model-config=false \ + --exit-timeout-secs=60 + resources: + limits: + nvidia.com/gpu: 1 + cpu: 4 + memory: 16G + requests: + nvidia.com/gpu: 1 + cpu: 4 + memory: 16G + # Loading from CVMFS is slow the first time (the files come over the + # network into the local cache): the startup probe allows four minutes. + startupProbe: + periodSeconds: 10 + failureThreshold: 24 + timeoutSeconds: 15 + httpGet: { path: /v2/health/ready, port: http } + readinessProbe: + timeoutSeconds: 15 + periodSeconds: 10 + failureThreshold: 10 + successThreshold: 1 + httpGet: { path: /v2/health/ready, port: http } + # The cluster's CVMFS claim (apps/infrastructure/cvmfs-pvc.yaml), read-only. + # HostToContainer: CVMFS repositories are autofs mounts on the node, and the + # pod must see the ones mounted after it started. + modelRepository: + claimName: cvmfs + mountPath: /cvmfs + mountPropagation: HostToContainer + +# One Triton idling so the first request does not wait for a pod, two image +# pulls and a model load from CVMFS; at most four, since the GPUs are shared. +replicas: + min: 1 + max: 4 + +# The CMS GPU nodes, for every pod here. +nodeSelector: { "cms-af-prod": "true" } +tolerations: + - key: hub.jupyter.org/dedicated + operator: Equal + value: cms-af + effect: NoSchedule + +# The private address pool: reachable from the AF, never a public ingress. +service: + type: LoadBalancer + annotations: + metallb.universe.tf/address-pool: geddes-private-pool diff --git a/deploy/experimental/kustomization.yaml b/deploy/experimental/kustomization.yaml index 68ac8083..75fb3034 100644 --- a/deploy/experimental/kustomization.yaml +++ b/deploy/experimental/kustomization.yaml @@ -38,6 +38,13 @@ resources: # supersonic-model-manager-auth Secret to exist in the namespace. - ../../apps/sonic/model-manager/helmrelease.yaml + # Triton on Ray: Ray Serve's gRPC proxy in front of a Triton per GPU pod, + # plus the KubeRay operator that reconciles it. Models come from a CMSSW + # release on CVMFS (the cvmfs claim). See apps/ray/README.md. + - ../../apps/ray/helmrepo.yaml + - ../../apps/ray/operator/helmrelease.yaml + - ../../apps/ray/sonic-ray/helmrelease.yaml + - ../../apps/interlink/helmrepo.yaml - ../../apps/interlink/hammer/helmrelease.yaml - ../../apps/interlink/gautschi/helmrelease.yaml @@ -150,6 +157,14 @@ configMapGenerator: - ../../apps/af-utils/slurm-probes/probe.sh - ../../apps/af-utils/slurm-probes/serve.py + - name: kuberay-operator-config + files: + - values.yaml=../../apps/ray/operator/values.yaml + + - name: sonic-ray-config + files: + - values.yaml=../../apps/ray/sonic-ray/values.yaml + - name: interlink-hammer-config files: - values.yaml=../../apps/interlink/hammer/values.yaml diff --git a/mypy.ini b/mypy.ini index ed0e0788..37427ef1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -26,11 +26,12 @@ files = docker/af-pod-monitor, docker/agentic-interface, docker/purdue-af/jupyter, + apps/ray/sonic-ray/chart/files, pixi/check-env.py, pixi/global/check-gpu.py ; our own flat-import packages (agentic-interface uses `from context import ...`; ; gpu_queries is shared with the hub snippets and copied into the image) -mypy_path = docker/agentic-interface:apps/jupyterhub/jupyterhub/extraFiles +mypy_path = docker/agentic-interface:apps/jupyterhub/jupyterhub/extraFiles:apps/ray/sonic-ray/chart/files explicit_package_bases = True namespace_packages = True diff --git a/tests/manifests/test_ray.py b/tests/manifests/test_ray.py new file mode 100644 index 00000000..e24b2b27 --- /dev/null +++ b/tests/manifests/test_ray.py @@ -0,0 +1,534 @@ +"""Tests for apps/ray — Triton on Ray with Ray Serve carrying its gRPC. + +The properties the deployment depends on, which no schema can express: the +Triton container is the one the values describe, on a read-only model +repository; one forwarding replica per Triton pod (the `triton` resource), Serve +never asking for more replicas than the worker group may hold, nothing on the +head, Serve's gRPC proxy handed Triton's own servicer from a package every +pod installs, the Services selecting on labels KubeRay leaves alone. Those +run against the rendered chart when helm is on PATH (it is in CI), and +against the values and source otherwise. +""" + +import re +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[2] +RAY = REPO / "apps" / "ray" +CHART = RAY / "sonic-ray" / "chart" +VALUES = RAY / "sonic-ray" / "values.yaml" +CODE = CHART / "files" / "sonic_ray" +SERVE_APP = CODE / "serve_app.py" +EXPERIMENTAL = REPO / "deploy" / "experimental" / "kustomization.yaml" +VALIDATOR = REPO / ".github" / "workflows" / "validate-manifests.sh" + + +def load(path): + return yaml.safe_load(path.read_text()) + + +@pytest.fixture(scope="module") +def values(): + return load(VALUES) + + +@pytest.fixture(scope="module") +def chart_defaults(): + return load(CHART / "values.yaml") + + +@pytest.fixture(scope="module") +def rendered(): + """Every object the chart renders with the AF values, keyed by kind/name.""" + if shutil.which("helm") is None: + pytest.skip("helm not on PATH; validate-manifests.sh renders this chart in CI") + out = subprocess.run( + [ + "helm", + "template", + "sonic-ray", + str(CHART), + "--namespace", + "cms", + "-f", + str(VALUES), + ], + capture_output=True, + text=True, + check=True, + ).stdout + return { + (doc["kind"], doc["metadata"]["name"]): doc + for doc in yaml.safe_load_all(out) + if doc + } + + +@pytest.fixture(scope="module") +def rayservice(rendered): + return rendered[("RayService", "sonic-ray")] + + +@pytest.fixture(scope="module") +def cluster(rayservice): + return rayservice["spec"]["rayClusterConfig"] + + +@pytest.fixture(scope="module") +def serve_config(rayservice): + return yaml.safe_load(rayservice["spec"]["serveConfigV2"]) + + +@pytest.fixture(scope="module") +def deployment(serve_config): + apps = serve_config["applications"] + assert len(apps) == 1 and len(apps[0]["deployments"]) == 1 + return apps[0]["deployments"][0] + + +@pytest.fixture(scope="module") +def head_pod(cluster): + return cluster["headGroupSpec"]["template"] + + +@pytest.fixture(scope="module") +def worker_group(cluster): + groups = cluster["workerGroupSpecs"] + assert len(groups) == 1, ( + "one replica is one Triton pod; more groups needs a rethink" + ) + return groups[0] + + +def container(pod_template, name, kind="containers"): + return next(c for c in pod_template["spec"][kind] if c["name"] == name) + + +def env_of(container_spec): + return {e["name"]: e["value"] for e in container_spec["env"]} + + +# -- Flux wiring ----------------------------------------------------------- + + +def test_flux_deploys_operator_and_release(): + text = EXPERIMENTAL.read_text() + for resource in ( + "../../apps/ray/helmrepo.yaml", + "../../apps/ray/operator/helmrelease.yaml", + "../../apps/ray/sonic-ray/helmrelease.yaml", + ): + assert resource in text + for line in text.splitlines(): + if resource in line: + assert not line.strip().startswith("#"), line + + +def test_values_reach_both_releases(): + """A valuesFrom ConfigMap nobody generates leaves a release on chart defaults.""" + generated = { + cm["name"]: cm["files"] for cm in load(EXPERIMENTAL)["configMapGenerator"] + } + for app, config in ( + ("operator", "kuberay-operator-config"), + ("sonic-ray", "sonic-ray-config"), + ): + hr = load(RAY / app / "helmrelease.yaml") + assert [v["name"] for v in hr["spec"]["valuesFrom"]] == [config] + assert generated[config] == [f"values.yaml=../../apps/ray/{app}/values.yaml"] + + +def test_release_waits_for_the_crds(): + """The chart renders a RayService. Until the operator's chart has installed + the ray.io CRDs that is an unknown kind, and a raw manifest in the same + Kustomization would have blocked the apply that installs them.""" + release = load(RAY / "sonic-ray" / "helmrelease.yaml") + assert release["spec"]["dependsOn"] == [{"name": "kuberay-operator"}] + assert release["spec"]["chart"]["spec"]["chart"] == "./apps/ray/sonic-ray/chart" + assert release["spec"]["chart"]["spec"]["sourceRef"]["kind"] == "GitRepository" + + operator = load(RAY / "operator" / "helmrelease.yaml") + assert operator["spec"]["install"]["crds"] == "Create" + assert operator["spec"]["upgrade"]["crds"] == "CreateReplace" + # singleNamespaceInstall keeps the watch and the RBAC inside cms. + assert load(RAY / "operator" / "values.yaml")["singleNamespaceInstall"] is True + + +def test_validator_renders_this_chart(): + """Nothing else validates a chart sourced from this repository: kubeconform + never sees what helm renders, and ray.io has no schema anyway.""" + text = VALIDATOR.read_text() + assert "from this repository" in text + assert "RayService" not in text, "the kubeconform skip is gone; keep it gone" + + +# -- no custom image ----------------------------------------------------------- + + +def test_pods_run_stock_images(head_pod, worker_group, chart_defaults, values): + """Official Ray (CPU flavour: no Ray process touches a GPU) through the + Docker Hub proxy cache; the Triton image is whatever the values name. The + Ray tag's version is ray.version, + which also pins the autoscaler sidecar KubeRay adds.""" + ray = chart_defaults["ray"] + ray_image = ( + f"{ray['image']['repository']}:{ray['version']}-{ray['image']['flavor']}" + ) + assert ray_image.startswith( + "geddes-registry.rcac.purdue.edu/docker-hub-cache/rayproject/ray:" + ) + assert ray_image.endswith("-cpu") + for template, name in ( + (head_pod, "ray-head"), + (worker_group["template"], "ray-worker"), + ): + c = container(template, name) + assert c["image"] == ray_image + assert c["imagePullPolicy"] == "IfNotPresent" # immutable tags + assert ( + container(template, "pip-install", "initContainers")["image"] == ray_image + ) + triton = container(worker_group["template"], "triton") + assert ( + triton["image"] + == f"{values['triton']['image']['repository']}:{values['triton']['image']['tag']}" + ) + assert not (REPO / "docker" / "sonic-ray").exists(), "no custom image, by decision" + + +def test_tritons_servicer_is_installed_where_the_proxies_run( + head_pod, worker_group, chart_defaults +): + """Serve's gRPC proxy imports Triton's generated servicer at startup, on + every node, outside any runtime_env — so the package must be on PYTHONPATH + for the whole pod. --no-deps keeps the image's grpcio/protobuf in charge; + tritonclient 2.48 is the last release whose stubs match protobuf 4.""" + pins = chart_defaults["python"]["pip"] + (tc,) = [p for p in pins if p.startswith("tritonclient==")] + major, minor = map(int, tc.removeprefix("tritonclient==").split(".")[:2]) + assert (major, minor) <= (2, 48) + for template, name in ( + (head_pod, "ray-head"), + (worker_group["template"], "ray-worker"), + ): + init = container(template, "pip-install", "initContainers") + assert init["args"] == pins + assert "--no-deps" in init["command"] and "--target" in init["command"] + deps_dir = init["command"][init["command"].index("--target") + 1] + deps_mount = next(m for m in init["volumeMounts"] if m["name"] == "python-deps") + assert deps_mount["mountPath"] == deps_dir + c = container(template, name) + assert deps_dir in env_of(c)["PYTHONPATH"].split(":") + assert any( + m["name"] == "python-deps" and m["mountPath"] == deps_dir + for m in c["volumeMounts"] + ) + assert any( + v["name"] == "python-deps" and "emptyDir" in v + for v in template["spec"]["volumes"] + ) + + +def test_code_reaches_every_pod(rendered, head_pod, worker_group): + """import_path resolves only if the ConfigMap holds the package and lands + on PYTHONPATH — on the head (where Serve builds the application) and the + workers (where replicas run). A code change must roll the cluster.""" + configmap = rendered[("ConfigMap", "sonic-ray-code")] + files = {p.name: p.read_text() for p in CODE.glob("*.py")} + assert files and configmap["data"] == files + for template, name in ( + (head_pod, "ray-head"), + (worker_group["template"], "ray-worker"), + ): + volume = next(v for v in template["spec"]["volumes"] if v["name"] == "code") + assert volume["configMap"]["name"] == "sonic-ray-code" + c = container(template, name) + mount = next(m for m in c["volumeMounts"] if m["name"] == "code") + assert mount["readOnly"] is True + code_dir = mount["mountPath"].removesuffix("/sonic_ray") + assert code_dir in env_of(c)["PYTHONPATH"].split(":") + assert template["metadata"]["annotations"]["checksum/code"] + assert ( + head_pod["metadata"]["annotations"]["checksum/code"] + == worker_group["template"]["metadata"]["annotations"]["checksum/code"] + ) + + +# -- the Triton in the pod is the one in the values --------------------------- + + +def test_rendered_triton_is_the_one_in_values(worker_group, values): + """The values are only parity if the template actually uses them.""" + triton = container(worker_group["template"], "triton") + assert triton["command"] == values["triton"]["command"] + assert triton["args"] == values["triton"]["args"] + assert triton["resources"] == values["triton"]["resources"] + assert triton["readinessProbe"] == values["triton"]["readinessProbe"] + assert triton["startupProbe"] == values["triton"]["startupProbe"] + assert {p["name"]: p["containerPort"] for p in triton["ports"]} == { + "http": 8000, + "grpc": 8001, + "triton-metrics": 8002, + } + + +def test_models_come_from_cvmfs_read_only(worker_group, values): + """The cluster's CVMFS claim, mounted read-only with host-to-container + propagation (CVMFS repositories are autofs mounts on the node), and every + --model-repository Triton is given lives under that mount, loaded from an + explicit list: what is served is exactly what the values say.""" + repo = values["triton"]["modelRepository"] + assert repo["claimName"] == "cvmfs" + volume = next( + v + for v in worker_group["template"]["spec"]["volumes"] + if v["name"] == "model-repository" + ) + assert volume["persistentVolumeClaim"] == {"claimName": "cvmfs", "readOnly": True} + mount = next( + m + for m in container(worker_group["template"], "triton")["volumeMounts"] + if m["name"] == "model-repository" + ) + assert mount == { + "name": "model-repository", + "mountPath": repo["mountPath"], + "readOnly": True, + "mountPropagation": "HostToContainer", + } + args = values["triton"]["args"][0].split() + repositories = [ + a.removeprefix("--model-repository=") + for a in args + if a.startswith("--model-repository=") + ] + assert repositories, "Triton is given no model repository" + assert all(r.startswith(repo["mountPath"] + "/") for r in repositories) + assert "--model-control-mode=explicit" in args + assert [a for a in args if a.startswith("--load-model=")], "no --load-model" + + +def test_placement_applies_to_every_pod(head_pod, worker_group, values): + for template in (head_pod, worker_group["template"]): + assert template["spec"]["nodeSelector"] == values["nodeSelector"] + assert template["spec"]["tolerations"] == values["tolerations"] + + +# -- scaling: one replica per Triton pod, all of it Ray's ------------------- + + +def test_no_keda_resources(): + """A ScaledObject would be a second controller fighting Ray over the + group — and could not drive it anyway: RayCluster has no scale subresource.""" + for path in sorted(RAY.rglob("*.yaml")): + text = path.read_text() + if "templates" in path.parts: # Go templates, not YAML until rendered + assert "ScaledObject" not in text and "keda.sh" not in text, path + continue + for doc in yaml.safe_load_all(text): + if isinstance(doc, dict): + assert doc.get("kind") != "ScaledObject", path + assert "keda.sh" not in str(doc.get("apiVersion", "")), path + + +def test_one_replica_per_triton_pod(deployment, worker_group, cluster): + """Each worker advertises one `triton`; each replica claims one. Nothing + else does, so a pod without a replica is idle and reclaimable, and a + replica without a pod is the pending request that grows the group.""" + assert deployment["ray_actor_options"] == { + "num_cpus": 0, + "resources": {"triton": 1}, + } + assert '\\"triton\\": 1' in worker_group["rayStartParams"]["resources"] + assert "resources" not in cluster["headGroupSpec"]["rayStartParams"] + assert cluster["enableInTreeAutoscaling"] is True + # The GPU is Triton's; Ray never sees it and never schedules onto it. + ray_worker = container(worker_group["template"], "ray-worker") + assert "nvidia.com/gpu" not in ray_worker["resources"]["limits"] + assert ( + container(worker_group["template"], "triton")["resources"]["limits"][ + "nvidia.com/gpu" + ] + == 1 + ) + + +def test_one_setting_bounds_serve_and_the_worker_group( + deployment, worker_group, values +): + """`replicas` is the only pair of numbers: Serve's bounds are it, the + group's ceiling is it (a replica needs a pod), the group starts with as + many pods as Serve will immediately ask for, and the group's own minimum + is 0 — Serve's minimum keeps pods alive, a pod with a replica is never + idle.""" + bounds = values["replicas"] + autoscaling = deployment["autoscaling_config"] + assert autoscaling["min_replicas"] == bounds["min"] + assert autoscaling["max_replicas"] == bounds["max"] + assert worker_group["maxReplicas"] == bounds["max"] + assert worker_group["replicas"] == bounds["min"] + assert worker_group["minReplicas"] == 0 + + +def test_scale_down_is_slower_than_scale_up(deployment): + autoscaling = deployment["autoscaling_config"] + assert autoscaling["downscale_delay_s"] > autoscaling["upscale_delay_s"] + assert autoscaling["target_ongoing_requests"] < deployment["max_ongoing_requests"] + + +def test_triton_is_given_time_to_drain(worker_group, deployment): + """Scale-down deletes the pod; --exit-timeout-secs and Serve's graceful + shutdown are worth nothing if the kubelet does not wait for them. (The + chart refuses to render otherwise.)""" + args = container(worker_group["template"], "triton")["args"][0] + exit_timeout = int(args.split("--exit-timeout-secs=")[1].split()[0]) + grace = worker_group["template"]["spec"]["terminationGracePeriodSeconds"] + assert grace > exit_timeout + assert grace > deployment["graceful_shutdown_timeout_s"] + + +def test_nothing_runs_on_the_head(cluster, head_pod): + """The head holds the Serve controller and proxies; a replica there would + have no Triton to forward to.""" + assert cluster["headGroupSpec"]["rayStartParams"]["num-cpus"] == "0" + assert all(c["name"] != "triton" for c in head_pod["spec"]["containers"]) + + +# -- Serve speaks Triton's protocol ------------------------------------------- + + +def test_grpc_proxy_is_handed_tritons_servicer(serve_config): + """The proxy accepts exactly the RPCs of Triton's GRPCInferenceService and + dispatches each to the replica method of the same name — which the + forwarder defines for every one of them (tests/sonic_ray).""" + grpc = serve_config["grpc_options"] + assert grpc["grpc_servicer_functions"] == [ + "tritonclient.grpc.service_pb2_grpc.add_GRPCInferenceServiceServicer_to_server" + ] + assert grpc["port"] == 9000 + + +def test_serve_import_path_resolves(serve_config, deployment): + """import_path names a module in the ConfigMap and an attribute in it; the + deployment name is the class serve.deployment wraps.""" + module_path, _, attribute = serve_config["applications"][0][ + "import_path" + ].partition(":") + assert module_path == "sonic_ray.serve_app" + source = SERVE_APP.read_text() + assert re.search( + rf"^{attribute} = serve\.deployment\({deployment['name']}\)\.bind\(\)", + source, + re.MULTILINE, + ) + assert re.search(rf"^class {deployment['name']}\b", source, re.MULTILINE) + + +# -- services --------------------------------------------------------------- + + +def test_inference_entry_point_is_kuberays_serve_service( + rayservice, head_pod, worker_group +): + """One gRPC address on the private pool, on Triton's conventional port. Behind it is Serve's gRPC proxy, so + every request is counted. KubeRay keeps the Service pointed at pods whose + proxy is healthy.""" + svc = rayservice["spec"]["serveService"] + assert svc["metadata"]["name"] == "sonic-ray-serve" + assert ( + svc["metadata"]["annotations"]["metallb.universe.tf/address-pool"] + == "geddes-private-pool" + ) + assert svc["spec"]["type"] == "LoadBalancer" + ports = {p["name"]: (p["port"], p["targetPort"]) for p in svc["spec"]["ports"]} + assert ports["grpc"] == (8001, 9000) + assert "scrape_metrics" not in svc["metadata"]["labels"] + for template, name in ( + (head_pod, "ray-head"), + (worker_group["template"], "ray-worker"), + ): + assert {p["containerPort"] for p in container(template, name)["ports"]} >= { + 8000, + 9000, + } + + +def test_head_is_not_exposed(cluster): + """The dashboard has no auth. Inference has its own address.""" + assert cluster["headGroupSpec"]["serviceType"] == "ClusterIP" + assert "headService" not in cluster["headGroupSpec"] + + +def test_metrics_services_select_labels_kuberay_leaves_alone( + rendered, head_pod, worker_group +): + """KubeRay stamps app.kubernetes.io/name onto every pod it creates and + names the cluster -raycluster-, renamed on each upgrade. + Selecting on either would match nothing, silently.""" + head_labels = head_pod["metadata"]["labels"] + worker_labels = worker_group["template"]["metadata"]["labels"] + + ray_metrics = rendered[("Service", "sonic-ray-metrics")] + assert ray_metrics["metadata"]["labels"]["scrape_metrics"] == "true" + assert [p["port"] for p in ray_metrics["spec"]["ports"]] == [8080] + assert ray_metrics["spec"]["selector"].items() <= head_labels.items() + assert ray_metrics["spec"]["selector"].items() <= worker_labels.items() + + triton_metrics = rendered[("Service", "sonic-ray-triton-metrics")] + assert triton_metrics["metadata"]["labels"]["scrape_metrics"] == "true" + assert [p["port"] for p in triton_metrics["spec"]["ports"]] == [8002] + assert triton_metrics["spec"]["selector"].items() <= worker_labels.items() + # Only the workers run Triton; scraping 8002 on the head would just fail. + assert not triton_metrics["spec"]["selector"].items() <= head_labels.items() + + for svc in (ray_metrics, triton_metrics): + assert "app.kubernetes.io/name" not in svc["spec"]["selector"] + assert "ray.io/cluster" not in svc["spec"]["selector"] + # release="sonic-ray" is how dashboards select this release's series. + assert svc["metadata"]["labels"]["app.kubernetes.io/instance"] == "sonic-ray" + + +# -- the chart refuses what cannot work ---------------------------------------- + + +@pytest.mark.parametrize( + "override, message", + [ + ("triton.modelRepository.claimName=", "claimName is required"), + ("replicas.min=5", "replicas.min exceeds replicas.max"), + ("triton.resources.limits.nvidia\\.com/gpu=2", "exactly one nvidia.com/gpu"), + ( + "ray.worker.terminationGracePeriodSeconds=30", + "must exceed Triton's --exit-timeout-secs", + ), + ( + "triton.modelRepository.mountPath=/elsewhere", + "never mention triton.modelRepository.mountPath", + ), + ("python.pip={grpcio}", "must pin tritonclient=="), + ], +) +def test_chart_fails_on_values_that_cannot_work(override, message): + if shutil.which("helm") is None: + pytest.skip("helm not on PATH") + result = subprocess.run( + [ + "helm", + "template", + "sonic-ray", + str(CHART), + "-f", + str(VALUES), + "--set", + override, + ], + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert message in result.stderr diff --git a/tests/sonic_ray/test_sonic_serve_app.py b/tests/sonic_ray/test_sonic_serve_app.py new file mode 100644 index 00000000..3a824bd1 --- /dev/null +++ b/tests/sonic_ray/test_sonic_serve_app.py @@ -0,0 +1,73 @@ +"""The Ray Serve layer, checked without Ray or tritonclient (neither is a test +dependency): the module is read as source. What matters is small — it must +forward every unary RPC of Triton's service and nothing else, be importable +under the name the chart uses, and gate readiness on Triton's.""" + +import ast +from pathlib import Path + +import pytest + +SERVE_APP = ( + Path(__file__).resolve().parents[2] + / "apps" + / "ray" + / "sonic-ray" + / "chart" + / "files" + / "sonic_ray" + / "serve_app.py" +) + + +@pytest.fixture(scope="module") +def source(): + return SERVE_APP.read_text() + + +@pytest.fixture(scope="module") +def module(source): + return ast.parse(source) + + +def test_forwards_every_rpc_but_the_bidirectional_stream(source): + """The RPC list is derived from Triton's generated servicer at import, so + a new Triton RPC is forwarded without an edit here; the one stream Serve + cannot carry is the only exclusion.""" + assert "vars(service_pb2_grpc.GRPCInferenceServiceServicer)" in source + assert 'name != "ModelStreamInfer"' in source + assert "setattr(TritonProxy, _rpc, _forwarder(_rpc))" in source + assert "await getattr(self._stub, rpc)(request)" in source + + +def test_bound_under_the_name_the_chart_imports(module): + """serveConfigV2's import_path is `sonic_ray.serve_app:triton`.""" + bound = next( + n + for n in module.body + if isinstance(n, ast.Assign) + and any(isinstance(t, ast.Name) and t.id == "triton" for t in n.targets) + ) + assert ast.unparse(bound.value) == "serve.deployment(TritonProxy).bind()" + + +def test_replica_readiness_is_tritons(module): + """A replica that came up before its Triton finished loading would be + routed to; __init__ blocks on ServerReady and check_health polls ServerLive.""" + cls = next( + n + for n in module.body + if isinstance(n, ast.ClassDef) and n.name == "TritonProxy" + ) + methods = {n.name for n in cls.body if isinstance(n, ast.FunctionDef)} + assert {"__init__", "_wait_for_triton", "check_health"} <= methods + init = next( + n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "__init__" + ) + assert "self._wait_for_triton()" in ast.unparse(init) + + +def test_nothing_here_parses_a_request(source): + """Triton does the inference; this file must stay a pass-through.""" + for forbidden in ("numpy", "onnxruntime", "fastapi", "json", "InferInput"): + assert forbidden not in source, forbidden