orca-chart: Add container probes

orca-chart/README.md

@@ -62,15 +62,13 @@ orca:
 | `ingress.hosts[0].paths[0].pathType` | string | `"Prefix"` |  |
 | `ingress.tls` | list | `[]` |  |
 | `kind` | string | `"Deployment"` | Workload kind, either `"Deployment"` or `"StatefulSet"`. `StatefulSet` provides stable per-pod DNS via a headless companion service and is the safe choice for horizontally scaling a persistent cache. |
-| `livenessProbe.httpGet.path` | string | `"/"` |  |
-| `livenessProbe.httpGet.port` | string | `"http"` |  |
+| `livenessProbe` | object | `{"httpGet":{"path":"/healthz","port":"http"}}` | Liveness probe for the Orca container. Set to `null` to drop it. Suspended while the startup probe is still failing. |
 | `nameOverride` | string | `""` |  |
 | `nodeSelector` | object | `{}` |  |
 | `podAnnotations` | object | `{}` |  |
 | `podLabels` | object | `{}` |  |
 | `podSecurityContext` | object | `{}` |  |
-| `readinessProbe.httpGet.path` | string | `"/"` |  |
-| `readinessProbe.httpGet.port` | string | `"http"` |  |
+| `readinessProbe` | object | `{"httpGet":{"path":"/readyz","port":"http"}}` | Readiness probe for the Orca container. Set to `null` to drop it. Suspended while the startup probe is still failing. |
 | `replicaCount` | int | `1` | Pod replicas |
 | `resources` | object | `{}` | CPU and memory resources to allocate to the pod |
 | `securityContext` | object | `{}` |  |
@@ -84,6 +82,7 @@ orca:
 | `serviceAccount.automount` | bool | `true` |  |
 | `serviceAccount.create` | bool | `true` |  |
 | `serviceAccount.name` | string | `""` |  |
+| `startupProbe` | object | See [values.yaml](values.yaml) | Startup probe for the Orca container, with a 5 minute budget (`failureThreshold` times `periodSeconds`). Raise `failureThreshold` if you add virtual registries or run on a slow or contended node. Set to `null` to drop it. |
 | `storage.accessModes` | list | `["ReadWriteOnce"]` | Access modes applied to every cache PVC the chart creates |
 | `storage.annotations` | object | `{}` | Extra annotations applied to every cache PVC the chart creates |
 | `storage.labels` | object | `{}` | Extra labels applied to every cache PVC the chart creates |
@@ -143,6 +142,23 @@ orca:
       - url: https://gitlab.com
 ```
 
+## Startup time and probes
+
+A cold *Varnish Orca* pod is not ready the moment the container starts. It compiles one VCL group per entry in `orca.virtual_registry.registries`, roughly 4 seconds each, and the artifact firewall performs an initial ruleset sync that blocks startup. That sync takes seconds against a warm cache, but it can take several minutes on a cold one, longer still when several replicas compete for the same CPU.
+
+The chart handles this with a startup probe rather than by padding the liveness and readiness probes. Kubernetes suspends both of those for as long as a startup probe is still failing, so a slow first boot never trips a restart, and the pod is kept out of the Service until it can actually serve.
+
+The default budget is 5 minutes, `failureThreshold: 60` at `periodSeconds: 5`. If your pods are killed mid-boot with `Startup probe failed`, raise `failureThreshold`:
+
+```sh
+helm install varnish-orca oci://docker.io/varnish/orca-chart \
+ --set "startupProbe.failureThreshold=180"
+```
+
+The liveness probe asks `/healthz`, which reports that the process is up. The readiness and startup probes ask `/readyz`, which reports that the pod can actually serve traffic. Keeping the startup probe on `/readyz` also means a pod that never finishes booting is eventually restarted, rather than sitting live but useless because `/healthz` keeps answering.
+
+All three probes address the listener by name as `http`, which always refers to the first entry in `orca.varnish.http`. Any probe can be dropped by setting it to `null`.
+
 ## Deploying a custom license
 
 To deploy a custom license to *Varnish Orca*, you first need to create a secret in Kubernetes which contains the license file.

orca-chart/templates/_pod.tpl

@@ -31,24 +31,36 @@ spec:
       imagePullPolicy: {{ .Values.image.pullPolicy }}
       command: ["/usr/bin/varnish-supervisor","--config","/etc/varnish-supervisor/config.yaml"]
       ports:
-        {{- $httpPorts := .Values.orca.varnish.http }}
-        {{- $numHttpPorts := len $httpPorts }}
-        {{- range $index, $portConfig := $httpPorts }}
-        - name: {{- if eq $numHttpPorts 1 }} http {{- else }} http-{{ $portConfig.port }} {{- end }}
-          containerPort: {{ $portConfig.port | default 80 }}
+        {{- /* The first listener of each scheme keeps the bare name, so that
+        "http" and "https" always resolve no matter how many listeners are
+        configured. The Services target them by name, as do the default
+        probes. */}}
+        {{- range $index, $portConfig := .Values.orca.varnish.http }}
+        {{- $port := $portConfig.port | default 80 }}
+        - name: {{ if eq $index 0 }}http{{ else }}http-{{ $port }}{{ end }}
+          containerPort: {{ $port }}
           protocol: TCP
         {{- end }}
-        {{- if .Values.orca.varnish.https }}
-        {{- $httpsPorts := .Values.orca.varnish.https }}
-        {{- $numHttpsPorts := len $httpsPorts }}
-        {{- range $index, $portConfig := $httpsPorts }}
-        - name: {{- if eq $numHttpsPorts 1 }} https {{- else }} https-{{ $portConfig.port }} {{- end }}
-          containerPort: {{ $portConfig.port | default 443 }}
+        {{- range $index, $portConfig := .Values.orca.varnish.https }}
+        {{- $port := $portConfig.port | default 443 }}
+        - name: {{ if eq $index 0 }}https{{ else }}https-{{ $port }}{{ end }}
+          containerPort: {{ $port }}
           protocol: TCP
         {{- end }}
-        {{- end }}
       resources:
         {{- toYaml .Values.resources | nindent 8 }}
+      {{- with .Values.startupProbe }}
+      startupProbe:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.livenessProbe }}
+      livenessProbe:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.readinessProbe }}
+      readinessProbe:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- if and .Values.extraEnvs (not (empty .Values.extraEnvs)) }}
       env:
         {{- include "orca.toEnv" (merge (dict "envs" .Values.extraEnvs) .) | nindent 8 }}

orca-chart/test/unit_common/common.bats

@@ -104,7 +104,7 @@ load _helpers
     [ "${actual}" = "8080" ]
 }
 
-@test "${kind}: multiple HTTP ports get suffixed names" {
+@test "${kind}: extra HTTP ports get suffixed names" {
     cd "$(chart_dir)"
     local actual=$((helm template \
         --set "kind=${kind}" \
@@ -113,7 +113,7 @@ load _helpers
         --namespace default \
         --show-only "${template}" \
         .) | yqj '[.spec.template.spec.containers[0].ports[].name]')
-    [ "${actual}" = '["http-80","http-8080"]' ]
+    [ "${actual}" = '["http","http-8080"]' ]
 }
 
 @test "${kind}: HTTPS port renders when configured" {
@@ -127,6 +127,18 @@ load _helpers
     [ "${actual}" = '{"name":"https","containerPort":443,"protocol":"TCP"}' ]
 }
 
+@test "${kind}: extra HTTPS ports get suffixed names" {
+    cd "$(chart_dir)"
+    local actual=$((helm template \
+        --set "kind=${kind}" \
+        --set 'orca.varnish.https[0].port=443' \
+        --set 'orca.varnish.https[1].port=8443' \
+        --namespace default \
+        --show-only "${template}" \
+        .) | yqj '[.spec.template.spec.containers[0].ports[].name]')
+    [ "${actual}" = '["http","https","https-8443"]' ]
+}
+
 @test "${kind}: resources applied" {
     cd "$(chart_dir)"
     local actual=$((helm template \
@@ -139,6 +151,91 @@ load _helpers
     [ "${actual}" = '{"limits":{"cpu":"500m"},"requests":{"memory":"256Mi"}}' ]
 }
 
+@test "${kind}: startupProbe rendered from values" {
+    cd "$(chart_dir)"
+    local actual=$((helm template \
+        --set "kind=${kind}" \
+        --namespace default \
+        --show-only "${template}" \
+        .) | yqj '.spec.template.spec.containers[0].startupProbe')
+    [ "${actual}" = '{"failureThreshold":60,"httpGet":{"path":"/readyz","port":"http"},"periodSeconds":5}' ]
+}
+
+@test "${kind}: livenessProbe rendered from values" {
+    cd "$(chart_dir)"
+    local actual=$((helm template \
+        --set "kind=${kind}" \
+        --namespace default \
+        --show-only "${template}" \
+        .) | yqj '.spec.template.spec.containers[0].livenessProbe')
+    [ "${actual}" = '{"httpGet":{"path":"/healthz","port":"http"}}' ]
+}
+
+@test "${kind}: readinessProbe rendered from values" {
+    cd "$(chart_dir)"
+    local actual=$((helm template \
+        --set "kind=${kind}" \
+        --namespace default \
+        --show-only "${template}" \
+        .) | yqj '.spec.template.spec.containers[0].readinessProbe')
+    [ "${actual}" = '{"httpGet":{"path":"/readyz","port":"http"}}' ]
+}
+
+@test "${kind}: startup budget overridable" {
+    cd "$(chart_dir)"
+    local actual=$((helm template \
+        --set "kind=${kind}" \
+        --set 'startupProbe.failureThreshold=180' \
+        --namespace default \
+        --show-only "${template}" \
+        .) | yq -r '.spec.template.spec.containers[0].startupProbe.failureThreshold')
+    [ "${actual}" = "180" ]
+}
+
+@test "${kind}: probes omitted when set to null" {
+    cd "$(chart_dir)"
+    local rendered
+    rendered=$(helm template \
+        --set "kind=${kind}" \
+        --set 'startupProbe=null' \
+        --set 'livenessProbe=null' \
+        --set 'readinessProbe=null' \
+        --namespace default \
+        --show-only "${template}" \
+        .)
+    local probe
+    for probe in startupProbe livenessProbe readinessProbe; do
+        local actual
+        actual=$(echo "${rendered}" |
+            yq -r ".spec.template.spec.containers[0].${probe}")
+        [ "${actual}" = "null" ]
+    done
+}
+
+# The probes address the listener by name, so the name has to survive a
+# multi-listener config. See the port naming in _pod.tpl.
+@test "${kind}: probe port is declared by the container" {
+    cd "$(chart_dir)"
+    local rendered
+    rendered=$(helm template \
+        --set "kind=${kind}" \
+        --set 'orca.varnish.http[0].port=80' \
+        --set 'orca.varnish.http[1].port=8080' \
+        --namespace default \
+        --show-only "${template}" \
+        .)
+    local probe_port
+    probe_port=$(echo "${rendered}" |
+        yq -r '.spec.template.spec.containers[0].startupProbe.httpGet.port')
+    [ "${probe_port}" = "http" ]
+
+    local declared
+    declared=$(echo "${rendered}" |
+        yq -r "[.spec.template.spec.containers[0].ports[].name] |
+            contains([\"${probe_port}\"])")
+    [ "${declared}" = "true" ]
+}
+
 @test "${kind}: securityContext applied" {
     cd "$(chart_dir)"
     local actual=$((helm template \

orca-chart/values.yaml

@@ -91,13 +91,37 @@ resources: {}
   #   cpu: 100m
   #   memory: 128Mi
 
+# Probes. `port: http` refers to the first entry in `orca.varnish.http`, which
+# always carries that name. Set any probe to `null` to drop it from the pod.
+#
+# Liveness asks /healthz, which reports that the process is up. Readiness and
+# startup ask /readyz, which reports that it can actually serve traffic.
+#
+# The startup probe owns the slow part of boot. Kubernetes suspends the
+# liveness and readiness probes for as long as a startup probe is still
+# failing, so neither begins evaluating until the pod has come up once, and
+# the startup budget is what has to cover a cold start.
+#
+# That budget is deliberately generous, because it scales with things the
+# chart cannot see: roughly 4s per `orca.virtual_registry` VCL group, plus the
+# artifact firewall's initial ruleset sync, which blocks startup. The sync
+# takes seconds against a warm cache, but minutes on a cold one, more still
+# when several replicas compete for the same CPU. failureThreshold multiplied
+# by periodSeconds is the total, 5 minutes here. Raise failureThreshold if you
+# add registries or run on a slow or contended node.
+startupProbe:
+  httpGet:
+    path: /readyz
+    port: http
+  periodSeconds: 5
+  failureThreshold: 60
 livenessProbe:
   httpGet:
     path: /healthz
     port: http
 readinessProbe:
   httpGet:
-    path: /healthz
+    path: /readyz
     port: http
 
 # Autoscaling is supported but generally a poor fit for caches: scale-up adds