diff --git a/orca-chart/README.md b/orca-chart/README.md index 1cc99ca..98b8ebc 100644 --- a/orca-chart/README.md +++ b/orca-chart/README.md @@ -62,15 +62,13 @@ orca: | `ingress.hosts[0].paths[0].pathType` | string | `"Prefix"` | | | `ingress.tls` | list | `[]` | | | `kind` | string | `"Deployment"` | Workload kind, either `"Deployment"` or `"StatefulSet"`. `StatefulSet` provides stable per-pod DNS via a headless companion service and is the safe choice for horizontally scaling a persistent cache. | -| `livenessProbe.httpGet.path` | string | `"/"` | | -| `livenessProbe.httpGet.port` | string | `"http"` | | +| `livenessProbe` | object | `{"httpGet":{"path":"/healthz","port":"http"}}` | Liveness probe for the Orca container. Set to `null` to drop it. Suspended while the startup probe is still failing. | | `nameOverride` | string | `""` | | | `nodeSelector` | object | `{}` | | | `podAnnotations` | object | `{}` | | | `podLabels` | object | `{}` | | | `podSecurityContext` | object | `{}` | | -| `readinessProbe.httpGet.path` | string | `"/"` | | -| `readinessProbe.httpGet.port` | string | `"http"` | | +| `readinessProbe` | object | `{"httpGet":{"path":"/readyz","port":"http"}}` | Readiness probe for the Orca container. Set to `null` to drop it. Suspended while the startup probe is still failing. | | `replicaCount` | int | `1` | Pod replicas | | `resources` | object | `{}` | CPU and memory resources to allocate to the pod | | `securityContext` | object | `{}` | | @@ -84,6 +82,7 @@ orca: | `serviceAccount.automount` | bool | `true` | | | `serviceAccount.create` | bool | `true` | | | `serviceAccount.name` | string | `""` | | +| `startupProbe` | object | See [values.yaml](values.yaml) | Startup probe for the Orca container, with a 5 minute budget (`failureThreshold` times `periodSeconds`). Raise `failureThreshold` if you add virtual registries or run on a slow or contended node. Set to `null` to drop it. | | `storage.accessModes` | list | `["ReadWriteOnce"]` | Access modes applied to every cache PVC the chart creates | | `storage.annotations` | object | `{}` | Extra annotations applied to every cache PVC the chart creates | | `storage.labels` | object | `{}` | Extra labels applied to every cache PVC the chart creates | @@ -143,6 +142,25 @@ orca: - url: https://gitlab.com ``` +## Startup time and probes + +A cold *Varnish Orca* pod is not ready the moment the container starts. It compiles one VCL group per entry in `orca.virtual_registry.registries`, roughly 4 seconds each, and the artifact firewall performs an initial ruleset sync that blocks startup. That sync takes seconds against a warm cache, but it can take several minutes on a cold one, longer still when several replicas compete for the same CPU. + +The chart handles this with a startup probe rather than by padding the liveness and readiness probes. Kubernetes suspends both of those for as long as a startup probe is still failing, so a slow first boot never trips a restart, and readiness keeps the pod out of the main Service until it can actually serve. + +The headless companion Service that `kind: StatefulSet` creates is deliberately exempt: it sets `publishNotReadyAddresses: true`, so each pod's stable DNS name resolves throughout startup. That is what makes a pod addressable before it is ready, which is the point of the headless Service. Only the main Service gates on readiness. + +The default budget is 5 minutes, `failureThreshold: 60` at `periodSeconds: 5`. If your pods are killed mid-boot with `Startup probe failed`, raise `failureThreshold`: + +```sh +helm install varnish-orca oci://docker.io/varnish/orca-chart \ + --set "startupProbe.failureThreshold=180" +``` + +The liveness probe asks `/healthz`, which reports that the process is up. The readiness and startup probes ask `/readyz`, which reports that the pod can actually serve traffic. Keeping the startup probe on `/readyz` also means a pod that never finishes booting is eventually restarted, rather than sitting live but useless because `/healthz` keeps answering. + +All three probes address the listener by name as `http`, which always refers to the first entry in `orca.varnish.http`. Any probe can be dropped by setting it to `null`. + ## Deploying a custom license To deploy a custom license to *Varnish Orca*, you first need to create a secret in Kubernetes which contains the license file. diff --git a/orca-chart/templates/_pod.tpl b/orca-chart/templates/_pod.tpl index 7a35dbd..efbf284 100644 --- a/orca-chart/templates/_pod.tpl +++ b/orca-chart/templates/_pod.tpl @@ -31,24 +31,36 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["/usr/bin/varnish-supervisor","--config","/etc/varnish-supervisor/config.yaml"] ports: - {{- $httpPorts := .Values.orca.varnish.http }} - {{- $numHttpPorts := len $httpPorts }} - {{- range $index, $portConfig := $httpPorts }} - - name: {{- if eq $numHttpPorts 1 }} http {{- else }} http-{{ $portConfig.port }} {{- end }} - containerPort: {{ $portConfig.port | default 80 }} + {{- /* The first listener of each scheme keeps the bare name, so that + "http" and "https" always resolve no matter how many listeners are + configured. The Services target them by name, as do the default + probes. */}} + {{- range $index, $portConfig := .Values.orca.varnish.http }} + {{- $port := $portConfig.port | default 80 }} + - name: {{ if eq $index 0 }}http{{ else }}http-{{ $port }}{{ end }} + containerPort: {{ $port }} protocol: TCP {{- end }} - {{- if .Values.orca.varnish.https }} - {{- $httpsPorts := .Values.orca.varnish.https }} - {{- $numHttpsPorts := len $httpsPorts }} - {{- range $index, $portConfig := $httpsPorts }} - - name: {{- if eq $numHttpsPorts 1 }} https {{- else }} https-{{ $portConfig.port }} {{- end }} - containerPort: {{ $portConfig.port | default 443 }} + {{- range $index, $portConfig := .Values.orca.varnish.https }} + {{- $port := $portConfig.port | default 443 }} + - name: {{ if eq $index 0 }}https{{ else }}https-{{ $port }}{{ end }} + containerPort: {{ $port }} protocol: TCP {{- end }} - {{- end }} resources: {{- toYaml .Values.resources | nindent 8 }} + {{- with .Values.startupProbe }} + startupProbe: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if and .Values.extraEnvs (not (empty .Values.extraEnvs)) }} env: {{- include "orca.toEnv" (merge (dict "envs" .Values.extraEnvs) .) | nindent 8 }} diff --git a/orca-chart/test/unit_common/common.bats b/orca-chart/test/unit_common/common.bats index 840fe8e..8da431a 100644 --- a/orca-chart/test/unit_common/common.bats +++ b/orca-chart/test/unit_common/common.bats @@ -104,7 +104,7 @@ load _helpers [ "${actual}" = "8080" ] } -@test "${kind}: multiple HTTP ports get suffixed names" { +@test "${kind}: extra HTTP ports get suffixed names" { cd "$(chart_dir)" local actual=$((helm template \ --set "kind=${kind}" \ @@ -113,7 +113,7 @@ load _helpers --namespace default \ --show-only "${template}" \ .) | yqj '[.spec.template.spec.containers[0].ports[].name]') - [ "${actual}" = '["http-80","http-8080"]' ] + [ "${actual}" = '["http","http-8080"]' ] } @test "${kind}: HTTPS port renders when configured" { @@ -127,6 +127,18 @@ load _helpers [ "${actual}" = '{"name":"https","containerPort":443,"protocol":"TCP"}' ] } +@test "${kind}: extra HTTPS ports get suffixed names" { + cd "$(chart_dir)" + local actual=$((helm template \ + --set "kind=${kind}" \ + --set 'orca.varnish.https[0].port=443' \ + --set 'orca.varnish.https[1].port=8443' \ + --namespace default \ + --show-only "${template}" \ + .) | yqj '[.spec.template.spec.containers[0].ports[].name]') + [ "${actual}" = '["http","https","https-8443"]' ] +} + @test "${kind}: resources applied" { cd "$(chart_dir)" local actual=$((helm template \ @@ -139,6 +151,91 @@ load _helpers [ "${actual}" = '{"limits":{"cpu":"500m"},"requests":{"memory":"256Mi"}}' ] } +@test "${kind}: startupProbe rendered from values" { + cd "$(chart_dir)" + local actual=$((helm template \ + --set "kind=${kind}" \ + --namespace default \ + --show-only "${template}" \ + .) | yqj '.spec.template.spec.containers[0].startupProbe') + [ "${actual}" = '{"failureThreshold":60,"httpGet":{"path":"/readyz","port":"http"},"periodSeconds":5}' ] +} + +@test "${kind}: livenessProbe rendered from values" { + cd "$(chart_dir)" + local actual=$((helm template \ + --set "kind=${kind}" \ + --namespace default \ + --show-only "${template}" \ + .) | yqj '.spec.template.spec.containers[0].livenessProbe') + [ "${actual}" = '{"httpGet":{"path":"/healthz","port":"http"}}' ] +} + +@test "${kind}: readinessProbe rendered from values" { + cd "$(chart_dir)" + local actual=$((helm template \ + --set "kind=${kind}" \ + --namespace default \ + --show-only "${template}" \ + .) | yqj '.spec.template.spec.containers[0].readinessProbe') + [ "${actual}" = '{"httpGet":{"path":"/readyz","port":"http"}}' ] +} + +@test "${kind}: startup budget overridable" { + cd "$(chart_dir)" + local actual=$((helm template \ + --set "kind=${kind}" \ + --set 'startupProbe.failureThreshold=180' \ + --namespace default \ + --show-only "${template}" \ + .) | yq -r '.spec.template.spec.containers[0].startupProbe.failureThreshold') + [ "${actual}" = "180" ] +} + +@test "${kind}: probes omitted when set to null" { + cd "$(chart_dir)" + local rendered + rendered=$(helm template \ + --set "kind=${kind}" \ + --set 'startupProbe=null' \ + --set 'livenessProbe=null' \ + --set 'readinessProbe=null' \ + --namespace default \ + --show-only "${template}" \ + .) + local probe + for probe in startupProbe livenessProbe readinessProbe; do + local actual + actual=$(echo "${rendered}" | + yq -r ".spec.template.spec.containers[0].${probe}") + [ "${actual}" = "null" ] + done +} + +# The probes address the listener by name, so the name has to survive a +# multi-listener config. See the port naming in _pod.tpl. +@test "${kind}: probe port is declared by the container" { + cd "$(chart_dir)" + local rendered + rendered=$(helm template \ + --set "kind=${kind}" \ + --set 'orca.varnish.http[0].port=80' \ + --set 'orca.varnish.http[1].port=8080' \ + --namespace default \ + --show-only "${template}" \ + .) + local probe_port + probe_port=$(echo "${rendered}" | + yq -r '.spec.template.spec.containers[0].startupProbe.httpGet.port') + [ "${probe_port}" = "http" ] + + local declared + declared=$(echo "${rendered}" | + yq -r "[.spec.template.spec.containers[0].ports[].name] | + contains([\"${probe_port}\"])") + [ "${declared}" = "true" ] +} + @test "${kind}: securityContext applied" { cd "$(chart_dir)" local actual=$((helm template \ diff --git a/orca-chart/values.yaml b/orca-chart/values.yaml index 90ca796..8049c51 100644 --- a/orca-chart/values.yaml +++ b/orca-chart/values.yaml @@ -91,13 +91,37 @@ resources: {} # cpu: 100m # memory: 128Mi +# Probes. `port: http` refers to the first entry in `orca.varnish.http`, which +# always carries that name. Set any probe to `null` to drop it from the pod. +# +# Liveness asks /healthz, which reports that the process is up. Readiness and +# startup ask /readyz, which reports that it can actually serve traffic. +# +# The startup probe owns the slow part of boot. Kubernetes suspends the +# liveness and readiness probes for as long as a startup probe is still +# failing, so neither begins evaluating until the pod has come up once, and +# the startup budget is what has to cover a cold start. +# +# That budget is deliberately generous, because it scales with things the +# chart cannot see: roughly 4s per entry in `orca.virtual_registry.registries`, +# plus the artifact firewall's initial ruleset sync, which blocks startup. The +# sync takes seconds against a warm cache, but minutes on a cold one, more +# still when several replicas compete for the same CPU. failureThreshold +# multiplied by periodSeconds is the total, 5 minutes here. Raise +# failureThreshold if you add registries or run on a slow or contended node. +startupProbe: + httpGet: + path: /readyz + port: http + periodSeconds: 5 + failureThreshold: 60 livenessProbe: httpGet: path: /healthz port: http readinessProbe: httpGet: - path: /healthz + path: /readyz port: http # Autoscaling is supported but generally a poor fit for caches: scale-up adds