diff --git a/test/framework/.gitignore b/test/framework/.gitignore new file mode 100644 index 000000000..b7325183a --- /dev/null +++ b/test/framework/.gitignore @@ -0,0 +1,6 @@ +.venv/ +__pycache__/ +*.egg-info/ +.ruff_cache/ +runs/ +.mypy_cache/ diff --git a/test/framework/Makefile b/test/framework/Makefile new file mode 100644 index 000000000..c4b5d7d57 --- /dev/null +++ b/test/framework/Makefile @@ -0,0 +1,171 @@ +# sbtest — component/detector framework for large-scale test runs. +# +# Every target that needs the virtualenv depends on it, so there is no separate setup step +# to remember: `make test` on a clean checkout creates .venv, installs, and runs. +# +# make test unit tests (no cluster) +# make analyze RUN= [SUITE=] judge a finished run directory +# make collect OUT= [SUITE=] [DURATION=] collect from a live cluster, then judge +# make run SUITE= [DURATION=] [KEEP=1] drive a full test run, then judge it +# make detectors | make components what is available, with options +# make gate quality gate: ruff + mypy + tests +# make lint | make types | make fmt the gate's parts, individually +# make clean | make distclean +# +# SUITE takes a bundled suite name (see `make suites`) or a path to your own file. + +SHELL := /bin/bash +.DEFAULT_GOAL := help + +VENV := .venv +PY := $(VENV)/bin/python +PIP := $(VENV)/bin/pip +SBTEST := $(VENV)/bin/sbtest +RUFF := $(VENV)/bin/ruff +MYPY := $(VENV)/bin/mypy +STAMP := $(VENV)/.install-stamp +PYTHON ?= python3 + +# ── parameters ──────────────────────────────────────────────────────────────── +# SUITE is optional everywhere: with no suite, detectors all run with their defaults and no +# components do, which is exactly what judging an archive wants. +SUITE ?= +RUN ?= +OUT ?= +DURATION ?= 0 +JSON_NAME ?= findings.json +# KEEP=1 leaves the volumes, pods and CRs behind. Most post-mortems need this, so it is worth +# knowing the flag exists before the run rather than after. +KEEP ?= +EXTRA ?= + +SUITE_ARG := $(if $(SUITE),--suite $(SUITE),) + +# ── venv ────────────────────────────────────────────────────────────────────── + +$(VENV): + @echo "==> creating $(VENV)" + @$(PYTHON) -m venv $(VENV) + +# Re-installs when pyproject changes; the stamp keeps every other target from paying for it. +$(STAMP): pyproject.toml | $(VENV) + @echo "==> installing sbtest (editable) + dev extras" + @$(PIP) install -q --upgrade pip + @$(PIP) install -q -e '.[dev]' + @touch $@ + +.PHONY: venv +venv: $(STAMP) ## Create the virtualenv and install sbtest. + @$(PY) -c "import sbtest; print(f'sbtest {sbtest.__version__} ready in $(VENV)')" + +# ── tests and lint ──────────────────────────────────────────────────────────── + +.PHONY: test +test: $(STAMP) ## Run the unit tests (no cluster needed). + @$(PY) -m unittest discover -s tests -p 'test_*.py' -v + +.PHONY: lint +lint: $(STAMP) ## Lint with ruff. + @$(RUFF) check sbtest tests + +.PHONY: types +types: $(STAMP) ## Type-check with mypy. + @$(MYPY) + +.PHONY: fmt +fmt: $(STAMP) ## Apply ruff's safe fixes. + @$(RUFF) check --fix sbtest tests + +# The gate runs every check even when an earlier one fails, then reports once. Stopping at +# the first failure means three round trips to find out you had three problems; a detector +# framework whose own checks are annoying to run is a framework nobody runs. +.PHONY: gate +gate: $(STAMP) ## Quality gate: ruff + mypy + tests. Non-zero if any part fails. + @rc=0; \ + printf '\n==> ruff\n'; $(RUFF) check sbtest tests || rc=1; \ + printf '\n==> mypy\n'; $(MYPY) || rc=1; \ + printf '\n==> tests\n'; $(PY) -m unittest discover -s tests -p 'test_*.py' || rc=1; \ + printf '\n'; \ + if [ $$rc -eq 0 ]; then echo "GATE: PASS"; else echo "GATE: FAIL"; fi; \ + exit $$rc + +.PHONY: check +check: gate ## Alias for `gate`. + +# ── using it ────────────────────────────────────────────────────────────────── + +.PHONY: detectors +detectors: $(STAMP) ## List the available detectors and their options. + @$(SBTEST) detectors + +.PHONY: components +components: $(STAMP) ## List the available components and their options. + @$(SBTEST) components + +.PHONY: suites +suites: $(STAMP) ## List the bundled suites. + @for f in sbtest/suites/*.yaml; do \ + name=$$(basename $$f .yaml); \ + desc=$$($(PY) -c "import yaml; print(' '.join(yaml.safe_load(open('$$f')).get('description','').split()))"); \ + printf ' %-18s %s\n' "$$name" "$$desc"; \ + done + +.PHONY: analyze +analyze: $(STAMP) ## Judge a finished run directory. RUN= [SUITE=] +ifndef RUN + $(error RUN is required, e.g. make analyze RUN=../../operator/fio-mig-1787171993 SUITE=corruption-hunt) +endif + @$(SBTEST) analyze "$(RUN)" $(SUITE_ARG) --json-name "$(JSON_NAME)" --freeze-table $(EXTRA) + +.PHONY: collect +collect: $(STAMP) ## Collect from a live cluster, then judge. OUT= [SUITE=] [DURATION=] +ifndef OUT + $(error OUT is required, e.g. make collect OUT=./runs/soak SUITE=migration-soak DURATION=600) +endif + @$(SBTEST) collect "$(OUT)" $(SUITE_ARG) --duration "$(DURATION)" --judge \ + --json-name "$(JSON_NAME)" $(EXTRA) + +.PHONY: run +run: $(STAMP) ## Drive a full test run against a cluster, then judge it. SUITE= [DURATION=] [KEEP=1] [OUT=] +ifndef SUITE + $(error SUITE is required, e.g. make run SUITE=migration-full DURATION=7200 KEEP=1) +endif + @$(SBTEST) run $(SUITE_ARG) --duration "$(DURATION)" \ + $(if $(OUT),--outdir "$(OUT)",) $(if $(KEEP),--keep,) \ + --json-name "$(JSON_NAME)" $(EXTRA) + +# Judging every archived run at once: the cheapest way to see whether a new or changed +# detector fires where it should and stays quiet where it should not. +.PHONY: analyze-all +analyze-all: $(STAMP) ## Judge every operator/fio-mig-* run directory. [SUITE=] + @shopt -s nullglob; \ + found=0; \ + for d in ../../operator/fio-mig-*/; do \ + found=1; \ + echo "════════ $$d"; \ + $(SBTEST) analyze "$$d" $(SUITE_ARG) --json-name "$(JSON_NAME)" 2>&1 \ + | grep -E 'RESULT|skipped \(' || true; \ + done; \ + [ $$found -eq 1 ] || echo "no operator/fio-mig-* run directories found" + +# ── housekeeping ────────────────────────────────────────────────────────────── + +.PHONY: clean +clean: ## Remove caches and build leftovers (keeps the venv). + @rm -rf .ruff_cache .mypy_cache **/__pycache__ */__pycache__ */*/__pycache__ *.egg-info src/*.egg-info + @echo "cleaned" + +.PHONY: distclean +distclean: clean ## Also remove the virtualenv. + @rm -rf $(VENV) + @echo "removed $(VENV)" + +.PHONY: help +help: ## Show this help. + @echo "sbtest — component/detector framework for large-scale test runs" + @echo + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}' + @echo + @echo "Parameters: RUN, OUT, SUITE, DURATION, JSON_NAME, EXTRA" + @echo "Example: make analyze RUN=../../operator/fio-mig-1787171993 SUITE=corruption-hunt" \ No newline at end of file diff --git a/test/framework/README.md b/test/framework/README.md new file mode 100644 index 000000000..f0a7cad97 --- /dev/null +++ b/test/framework/README.md @@ -0,0 +1,534 @@ +# sbtest — a component/detector framework for large-scale test runs + +`sbtest` splits a test run into two kinds of pluggable pieces: + +* **Components** do the side-effecting work — drive a workload, sample host paths, follow + logs, snapshot the fabric. Each can be enabled or disabled independently. +* **Detectors** judge the evidence a run produced. Each is a pure function from `Evidence` + to findings: no cluster access, no ordering, no shared state. + +The split is the point. Every check worth having came out of an incident, and adding one +should not mean touching collection code; conversely, turning streaming log collection on or +off should not disturb a single check. + +It drives `kubectl` as a subprocess and depends on nothing but PyYAML, so it runs wherever +the existing harness runs. + +> `operator/test/fio_migration_test.py` is untouched and still the way to run the migration +> soak today. This framework is the reusable substrate underneath it — its detector set is +> already a strict superset of that harness's checks, and it can judge every run that +> harness has ever produced. + +--- + +## Quick start + +Everything goes through `make`, which creates and populates `.venv` on first use — there is +no separate setup step to remember. + +```bash +cd test/framework + +make # help, with the parameters listed +make gate # quality gate: ruff + mypy + 142 tests + +make detectors # what can judge a run, and with which knobs +make components # what can run during a run +make suites # the bundled suites + +# judge any existing run directory — no cluster needed +make analyze RUN=../../operator/fio-mig-1787171993 +make analyze RUN= SUITE=corruption-hunt + +# judge every archived run at once: the cheapest check on a new detector +make analyze-all SUITE=corruption-hunt + +# collect from a live cluster for 10 minutes, then judge +make collect OUT=./runs/my-run SUITE=migration-soak DURATION=600 + +# drive a full migration run — volumes, fio, VolumeMigration CRs — then judge it +make run SUITE=migration-full DURATION=7200 KEEP=1 +``` + +`collect` observes a cluster someone else is loading; `run` creates the load itself. The half +after the components stop is identical in both, which is the point: a live run and an +archived one are judged by the same detectors, so a verdict from either means the same thing. + +`SUITE` takes a bundled name or a path to your own file, and is optional everywhere: with no +suite every detector runs on its defaults and no component does, which is exactly what +judging an archive wants. + +The venv also installs an `sbtest` entry point, if you would rather not go through make: + +```bash +.venv/bin/sbtest analyze --suite corruption-hunt --freeze-table +``` + +### Quality gate + +`make gate` runs ruff, mypy and the unit tests, and — deliberately — **runs all three even +when an earlier one fails**, then reports once and exits non-zero. Finding out you had three +problems should not take three round trips; a framework whose own checks are tedious to run +is one nobody runs. `make lint`, `make types` and `make test` are the parts individually. + +The only runtime dependency is `pyyaml`, for the suites; `ruff` and `mypy` are dev extras. +Everything runs out of the venv that `make` builds, so the dependency costs nothing — and a +suite is a document to be read and argued with, where every threshold wants a sentence saying +why it is that number. JSON cannot carry that sentence. + +`analyze` is the one to reach for first. It runs the whole detector set over an artifact +directory — including every archived `fio-mig-*` run — which means a check can be written or +fixed and tried **immediately against the run that motivated it**, instead of against the +next four-hour run. Not being able to do that was the single biggest gap in the harness this +grew from. + +## It reproduces the findings it was built from + +Run against `operator/fio-mig-1787171993` (20 pods, 46 migrations, 4h08m), the detectors +independently derive the result that took a manual investigation to find: + +``` +ana.freeze-count mig-20 (2 freezes) mig-29 (4) mig-38 (5) mig-42 (5) +fio.checksum mig-20 (5 blocks) mig-29 (4) mig-38 (2) mig-42 (2) + +subjects with more than one finding: + mig-20: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) + mig-29: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) + mig-38: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) + mig-42: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) +``` + +Two unrelated detectors — one reading host ANA samples, one reading fio's own output — land +on the same four subjects out of 46. That correlation is the strongest signal these runs +produce, and surfacing it automatically is why findings carry a `subject`. + +The same detector set on the earlier `fio-mig-1787159565` reports a *different* signature +(8 × EREMOTEIO, 3 overlong pauses, 3481 `does-not-allow-host` matches from the path leak, no +freeze-count findings) and correctly reports `nvme.stale-controllers` as **skipped** rather +than clean, because that run has no fabric snapshot. + +## The path-loss ladder + +dmesg carries something no other source does: when every path to a namespace goes away, the +kernel does not simply fail I/O. It queues, waits, and only fails once a timeout expires — +and each rung is a different amount of damage. + +``` +all paths inaccessible + -> block nvmeXnY: no usable path - requeuing I/O queued; the application waits + -> nvme nvmeN: failfast expired fast_io_fail_tmo elapsed + -> block nvmeXnY: no available path - failing I/O errors reach the application + -> XFS (nvmeXnY): log I/O error -5 + -> XFS (nvmeXnY): Filesystem has been shut down the volume needs repair +``` + +**`fast_io_fail_tmo` is the knob that decides which rung a cutover pause reaches.** A pause +shorter than it is absorbed by queueing; a pause longer becomes application-visible errors and +then filesystem damage. That is why `ana.cutover-pause` and `kernel.path-loss` belong in the +same report: one measures the pause, the other says whether the host survived it. + +Measured across three archived runs, counting only what happened **inside each run's window**: + +| run | requeued | `failfast expired` | failing I/O | filesystems shut down | +|---|---|---|---|---| +| `-1787159565` | 2 | 18 | 0 | 0 | +| `-1787171993` | 163 | 84 | 0 | 0 | +| `-1787205545` | 35 (+186 before) | 20 (+118 before) | 0 (+30 before) | 0 (+20 before) | + +The parenthesised numbers are the reason attribution exists. Read without a window, that last +run looks catastrophic — 30 failed I/Os and 19 filesystems shut down. All of it happened +between 05:46 and 05:48, eleven minutes *before* the run started at 05:59: it is the previous +run's teardown and the cluster reinstall, not this run's doing. (Worth chasing separately — +tearing a cluster down should not kill mounted filesystems — but it is not a migration defect.) + +## Attribution: old damage must not fail a new run + +dmesg is a ring buffer covering hours and a cluster outlives its runs, so evidence routinely +contains the previous runs' mess. Counting it twice over is a trap: a clean run inherits its +predecessor's failure, and — worse — a genuinely broken run hides inside inherited noise. + +Every finding therefore carries an `Attribution`: + +| attribution | meaning | counts against the run? | +|---|---|---| +| `run` | happened inside the run's window | yes | +| `unknown` | no usable timestamp, or no known window | **yes** — "I cannot date this" must not become "not our problem" | +| `pre-existing` | positively dated before the run began | no | + +Severity says *does this matter*; attribution says *whose fault*. The two are orthogonal, so +the same observation is CRITICAL when the run caused it and a hygiene WARNING when it did not. + +### What actually forfeits a run + +Almost nothing, and the distinction is worth stating because the temptation is to fail on any +inherited mess: + +| pre-existing condition | affects this run? | verdict | +|---|---|---| +| Dead-cluster controllers, old reconnect storms, old fabric errors | No — they cannot make a *different* cluster's migration fail | hygiene WARNING | +| A filesystem killed before the run, on a volume the run does not use | No | hygiene WARNING | +| **Live-cluster** controllers already `live`-with-no-namespace at setup (`nvme.dirty-start`) | **Yes** — `VerifyMigrationPaths` will reject migrations that should pass, so the completion rate measures the inherited mess | **INCONCLUSIVE** | + +That last row is the only thing that produces `INCONCLUSIVE`, and it is a third verdict rather +than a softer FAIL because it calls for a different action: clean the fabric and run again, +rather than go looking through the code. It needs the `nvme.snapshot` component's pre-run +snapshot; without one the detector reports itself **skipped** instead of guessing. + +`dmesg --time-format=iso` is collected in preference to `dmesg -T` for exactly this reason: the +ISO form carries a UTC offset, so an event can be placed against the run window soundly rather +than by assuming the host's clock agrees with the harness's. + +## Concepts + +### Evidence + +The only thing a detector may read. Everything is lazy — a run's SPDK logs are tens of MiB +per node and most detectors never open them. + +```python +ev.migrations() -> list[Migration] # the timeline +ev.ana_samples("mig-20") -> list[AnaSample] # host path state, per migration +ev.fio_jobs() -> list[FioJob] # per-pod outcome and errno +ev.fio_timeseries(pod) -> list[IopsSample] # per-second IOPS +ev.fio_log(pod) -> Iterator[str] # where verify failures live +ev.container_logs() -> list[str] # "spdk-4420", "operator", ... +ev.container_log(name) -> Iterator[str] +ev.nvme_controllers() -> list[NvmeController] # fabric snapshot +``` + +Two implementations, kept deliberately close so a check that passes live cannot fail on +replay: `ArchiveEvidence` reads a finished run directory, and `LiveEvidence` is the same +reader pointed at the directory being written, overlaid with in-memory state. + +### Findings, and why "skipped" is not "clean" + +A detector returns `Finding`s carrying `severity`, `subject`, `detail`, structured +`evidence`, and a `note` explaining what it means. Any CRITICAL fails the run. + +A detector whose evidence is **absent** must raise `SkipDetector` rather than return nothing. +Silence makes "could not check" and "checked, all clean" the same output, which is how a +broken check passes a broken run for weeks. Skips are reported separately, by name. + +### Component lifecycle + +Every hook is optional; override only what applies. + +``` +setup() resolve targets, create helpers +start() begin sampling / following / driving +tick() periodic, cheap, must not block +stop() stop doing the thing +collect() gather artifacts into the run directory +teardown() release cluster resources — runs even when the run failed +``` + +`collect` is separate from `stop` for exactly the case that motivated the framework: a +streaming collector has nothing to collect at the end because it has been writing all along, +while a post-run collector does all its work there — and the runner must be able to run +either, or both, without either knowing. + +Set `required = True` on a component that *is* the run (a workload, a migration driver): a +failure in its `setup` aborts, because continuing would produce a green result for a test +that never happened. Collectors leave it False, so losing one evidence stream degrades the +run instead of ending it — and the failure is recorded as a WARNING finding. + +## Configuration + +```yaml +components: + logs.stream: + containers: [spdk-container] + ttl_s: 21600 + logs.collect: true + ana.sample: + interval_s: 2.0 + +detectors: + ana.freeze-count: + # More than one freeze in a migration meant lost writes on 5 of 5 archived runs. + max_freezes: 1 + fio.checksum: + verify_lag_s: 45 + fio.throughput-outlier: false +``` + +Two rules make this predictable: + +* A name absent from the config is **off for components** and **on for detectors**. + Collecting is a cost you opt into; judging is not something you should have to remember to + switch on. +* An unknown name or option is an **error at startup**, with the valid list. A threshold that + looks set but is not is worse than one that is obviously missing. + +Bundled suites live in `sbtest/suites/` (`migration-full`, `migration-soak`, +`corruption-hunt`, `analyze-only`; `make suites` lists them) and are selected with +`--suite `, which also takes a path to your own file. A `.json` suite still loads, for +anything generating them programmatically. CLI `--enable-detector` / `--disable-component` +layer on top, and disable wins over enable. + +## Detector catalogue + +Every one of these came from a real defect. Defaults encode what the runs measured. + +| detector | what it catches | +|---|---| +| `ana.freeze-count` | **A migration that froze the volume more than once.** Exact predictor of silent write loss so far: 4/4 corrupting vs 0/42 clean. A migration takes the cutover pause once; the rest are retries, and each retry replays a non-idempotent transfer against a source that has been serving writes. | +| `ana.cutover-pause` | An all-paths-inaccessible window longer than the design pause (~2s). Complements the count: catches one window that overran, which the count cannot see. | +| `ana.split-brain` | Source and target both `optimized` at the same instant — two writers, silent corruption by construction. | +| `ana.unserved-after-cutover` | A Completed migration whose live target controller serves only some of the subsystem's namespaces — the half-moved case. | +| `ana.path-churn` | More distinct path addresses per host than the topology should produce. Informational: healthy counts are topology-dependent. | +| `fio.checksum` | **fio read back data it never wrote.** Reads succeeded, so nothing else notices. Attributes to a migration through a verify lag (see below). | +| `fio.job-error` | An fio job ended with a non-zero errno, with the errno's meaning — 121/EREMOTEIO points straight at the ANA detectors. | +| `fio.outage` | A pod's I/O stopped for longer than a cutover should cost — reported as a **freeze** when it came back and a **loss** when it never did. Both fail; only one means writes went missing. | +| `fio.throughput-outlier` | A pod far below the run's median IOPS. Weak alone; strong next to an ANA finding on the same subject. | +| `logs.pattern` | **User-definable regex checks over any collected log.** Ships a catalogue: undrained transfer, migration sub-task failure, host-not-allowed reconnect storm, write-to-RO-range, path-validation failure, stuck migration group, kernel reconnect loop. | +| `migration.outcomes` | Completion rate and phase breakdown. | +| `migration.errors` | Distinct migration errors, grouped by shape — 16 identical failures are one defect. | +| `nvme.stale-controllers` | Controllers that are live with no namespace (blocks every later migration of that subsystem) or stuck connecting. | +| `nvme.loss-timeout` | A `ctrl_loss_tmo` long enough that a leaked path outlives the run that made it. | +| `kernel.path-loss` | **How far the kernel got up the path-loss ladder** (see below). Stronger than ANA sampling for the same event: it is what the kernel did, not what a sampler caught, so it cannot miss a window shorter than the interval. | +| `kernel.filesystem-shutdown` | XFS/ext4 shut down or went read-only after failed log I/O — the volume needs unmount and repair. | +| `nvme.foreign-cluster` | **A controller retrying a subsystem whose cluster no longer exists.** No threshold, no topology: an NQN names its cluster. Hygiene only — it cannot affect the live cluster's migrations. | +| `nvme.dirty-start` | The fabric already held blocking debris **for the live cluster** at setup, so the run's results cannot be trusted. The one pre-existing CRITICAL. | +| `nvme.controller-churn` | Controllers created vs removed — "they never disappear", counted — plus controllers retrying without ever succeeding. | +| `kernel.fabric-errors` | Connect/reset/timeout errors grouped by kind. Texture around a failure rather than a verdict. | +| `control.node-flap` | A node marked down and back within seconds — a liveness check that depended on something other than the node. The shape behind a 9.5h outage. | +| `control.volume-health` | A volume or node whose health went false during the run and never returned. | +| `control.task-stuck` | Tasks created and never resolved — "the control plane stopped finishing things". | +| `control.retry-storm` | One operation attempted far more often than it should be; each retry re-does what the last half-did. | +| `control.node-agent` | The node-side agent returning errors, or **gaps in the liveness polling** — the upstream half of a false offline, visible nowhere else. | +| `evidence.log-coverage` | **A collected log that does not span the run**, bounding what every other log-based finding may claim. | +| `evidence.blind-spot` | A migration no log covers, so it cannot be post-mortemed whatever it did. | +| `evidence.inventory` | What evidence the run produced (INFO). | +| `security.secret-exposure` | Credential-shaped strings in collected logs. Reports the location, never the value. | + +Three things are load-bearing and worth knowing: + +* **`fio.checksum` verify lag (45s).** fio detects a lost write when it next *reads* that + block, 3–34s later in practice. Without the lag a migration's own losses are filed under + "no migration was running" — which is how a `Completed`-but-corrupting migration hid. +* **`ana.freeze-count` over `ana.cutover-pause`.** The count is more sensitive (two 3s + freezes look like one healthy pause on any longest-window measure), more specific (one + migration's single 5–6s pause lost nothing), and robust to sampling granularity. Keep both; + they catch different shapes, and `tests/test_detectors.py` pins exactly that. +* **The fio clock is fio's own.** Every offset in `timeseries.csv` is milliseconds since + *that pod's* fio started, so the wall clock hangs off `job_start` from its `result.json`, + never off the run's start — the run begins minutes before any fio does, by a different + amount per pod. The base decides which migration an outage overlaps, so getting it wrong + does not merely shift a chart: it names the wrong migration. `ArchiveEvidence` re-derives + it on replay, which corrects archives written before this was fixed. + +## Component catalogue + +| component | what it does | +|---|---| +| `logs.stream` | Follows chosen container logs for the whole run, surviving kubelet rotation, container restarts, and pod recreation. | +| `logs.collect` | Grabs container logs from each host's `/var/log/pods` at the end. Skips whatever `logs.stream` followed. | +| `host.dmesg` | `dmesg -T` from each storage worker. | +| `cluster.events` | `sbctl cluster get-logs` → `cluster-events.json`. | +| `nvme.snapshot` | Fabric snapshot before *and* after the run — "did the last run leave a mess?" is a real question, because a leaked controller breaks the *next* run. | +| `ana.sample` | Per-namespace ANA state on every consuming node, on an interval, written per migration in the layout `ArchiveEvidence` reads. Needs a driver to tell it which migration is in flight. | +| `workload.fio` | Provisions volumes from two StorageClasses (single-namespace and packed) and drives continuous md5-verified fio against them. `required`. | +| `migration.driver` | Creates `VolumeMigration` CRs in a loop, one at a time, and records what each one did. `required`. | + +### What gets collected + +| artifact | source | why per-what | +|---|---|---| +| `spdk-.txt` | storage-node SPDK container | **must be streamed.** Measured on vm04: a rotation every ~2 min, so the whole 50 MiB budget bought about **10 minutes** of retention. A migration was unrecoverable 6 seconds after it ended | +| `spdk--proxy.txt` | the SPDK JSON-RPC proxy | streamed too, but far lower volume and survives hours — so it is the fallback when the SPDK log is gone. It carries the RPC-level narrative (which calls, in what order, with what arguments) without SPDK's internal errors | +| `snode-api-.txt` | storage-node DaemonSet | per node — it starts and probes SPDK, so it is on the causal path of every node-offline decision | +| `csi-node-.txt` | CSI node plugin | per node — the plugin reconciles per host, so "which node" is the first question about anything it did | +| `.txt` | tasks pod, csi-controller | **per container.** The tasks pod runs seventeen independent runners; merging them gives a 50 MiB file that is not in time order, so its time span is meaningless and a pattern cannot be scoped to one runner | +| `operator.txt`, `webappapi.txt` | control plane | single-container, so per pod is fine | +| `dmesg-.txt` | each storage worker | ISO-timestamped, see the attribution section | +| `cluster-events.json` | `sbctl cluster get-logs` | the control plane's own account | + +### Why both log components exist + +The kubelet keeps only `containerLogMaxSize × containerLogMaxFiles` per container — 10Mi × 5 += **50 MiB** on the clusters this runs against. Measured SPDK write rates were 0.28–1.23 +MiB/min, so 50 MiB buys 41–176 minutes. A post-run grab of a four-hour run therefore returns +its tail and silently drops the rest; that is how several runs' worth of early evidence was +lost. So the high-volume logs are followed live and everything else is grabbed at the end, +where a one-shot grab is still complete. + +Worth knowing which of the two to reach for: when the SPDK log has aged out, the proxy log on +the *same node* usually still has the RPC sequence — on one run it recovered the full +freeze/revert/retry pattern of a corrupting migration whose SPDK log was already gone, and its +count of the retries was more accurate than the ANA sampler's. + +`logs.stream` handles three separate ways a log moves out from under a follower, because any +one of them leaves the stream *silent rather than failed*: rotation (same filename reopened — +`tail -F` handles it), a container restart (`.log` beside the old one — the target is +re-resolved on a timer), and a pod recreation (a whole new UID directory — same re-resolve). + +## Adding a detector + +```python +from sbtest.core import Detector, SkipDetector, critical, detector + +@detector +class MyCheck(Detector): + name = "mine.my-check" # dotted, stable; config keys off it + summary = "one line for `sbtest detectors`" + + def defaults(self) -> dict: + return {"threshold": 3} # also the option allow-list + + def detect(self, ev): + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") # not the same as clean + for m in migs: + if len(m.members) > self.opt("threshold"): + yield critical(self.name, title="subsystem too wide", subject=m.name, + evidence={"members": len(m.members)}) +``` + +Import it from `sbtest/detectors/__init__.py` and it appears in `sbtest detectors`, is +configurable by name, and runs against every archived run. Unit-test it against +`FakeEvidence` in `tests/test_detectors.py` — no cluster. + +## Adding a component + +```python +from sbtest.core import Component, component + +@component +class MyCollector(Component): + name = "mine.collector" + summary = "one line for `sbtest components`" + required = False # True only if this component *is* the run + + def defaults(self) -> dict: + return {"namespace": "default"} + + def setup(self, ctx): ... # resolve targets + def collect(self, ctx): + with open(ctx.path("my-artifact.txt"), "w") as fh: + fh.write("...") + def teardown(self, ctx): ... # always runs +``` + +Write artifacts through `ctx.path(...)` so they land in the run directory in the layout +`ArchiveEvidence` expects — that is what makes them replayable. Record observations with +`ctx.timeline.record("kind", subject=..., **data)` so detectors can read them back without +knowing which component produced them. + +## Layout + +``` +test/framework/ + sbtest/ + core/ evidence, findings, plugin registry, config, context, runner + components/ logs (stream + collect), nvme (sampler + snapshot), events, kube + detectors/ ana, control, fio, kernel, logs, meta, migration, nvme, security + adapters/ archive (finished run dir), live (run in progress) + suites/ migration-full, migration-soak, corruption-hunt, analyze-only (YAML) + cli.py + tests/ detector and core tests — 108, no cluster required + Makefile bootstraps .venv; every target depends on it + pyproject.toml packaging, ruff and mypy configuration +``` + +Note on lint config: `PTH` (pathlib-over-`os.path`) is deliberately **not** selected. +`operator/test/fio_migration_test.py` and everything around it use `os.path`, and one module +in a different idiom is worse than a consistent old one. + +## Driving a run + +`workload.fio` and `migration.driver` are the two components that *are* the run rather than +observers of it, and `migration-full` is the suite that wires them into the configuration the +corruption work used: + +```bash +make run SUITE=migration-full DURATION=7200 KEEP=1 +``` + +Both declare `required = True`, so a failure in their setup aborts the run instead of being +recorded as a warning. That distinction is the whole reason the flag exists: a run whose +workload never came up would otherwise sail through every detector — nothing to find, nothing +found — and report a clean pass for a test that never happened. + +Three things the driver reads from the backend instead of assuming, each because assuming it +produced a wrong answer that *looked* right: + +* **Where the volume is now.** The cluster's own rebalancer moves volumes with no Kubernetes + object changing, so a source cached at setup is stale by the tenth migration — and a stale + source means picking a target the volume already lives on, which the operator rejects and + which reads as a product bug. +* **Which volumes move together.** A migration moves the whole NVMe subsystem, so its blast + radius is the volume's *group*. The control plane decides the packing and a previous + migration can change it, so the group is re-read before each one. +* **What the migration actually did.** `status.sourceNodeUUID` is the operator's own resolved + answer, so it overwrites the driver's guess in the record. + +The driver also feeds `ana.sample`: `begin()` when a migration starts, `set_phase()` on each +phase transition, `end()` when it finishes. Sampling is worthless unattributed — a freeze +window is per migration — and the phase stamp is what makes a transition readable as "all +paths went inaccessible during Cutover" instead of "at 09:31:02". The handshake goes through +`ctx.shared`, so the sampler stays optional: with it disabled the driver still migrates and +the ANA detectors report themselves *skipped* rather than clean. + +Target selection is deliberate rather than random. The `target_policy` controls whether the +target node *also* hosts a pod consuming the subsystem being moved — the materially harder +case, since that host has to join a subsystem on the very node becoming its target. +`alternate` puts `consumer` first, so a run cut short still exercised it. When the policy +cannot be honoured the driver falls back to any other node and records the policy as +`consumer(unmet)`: the migration is still evidence, but the run's own record must not claim a +case it never reached. + +`workload.fio` enables md5 verification only when it can be trusted. fio's verify races +itself when two in-flight I/Os touch the same block; with one job that is fixable +(`--serialize_overlap`), across processes it is not without `io_submit_mode=offload`. So +`numjobs > 1` turns verification off and says so loudly — a run that measures throughput is +not a run that would have noticed data loss, and the report must not imply otherwise. + +## Trying it against a cluster + +`sbtest collect --suite migration-soak --duration 90 --judge` exercises the +observation half of the lifecycle on its own: grabber pods, live following, pre/post fabric +snapshots, control-plane events, dmesg, then judging. Useful for checking that collection +works before committing a long run to it, and for watching a cluster someone else is loading. + +Doing that for the first time found two bugs no amount of archive replay could have: + +* **Two components wanting a grabber on one node collided.** A Pod is immutable, so the second + to apply the same name failed on a field it may not change — and `logs.collect` silently + produced empty files for every node `logs.stream` had already claimed. The pod name now + carries the component. +* **A live run recorded no window**, so nothing could be dated, so every ring-buffer finding + was `unknown` — which counts — and the run failed on twenty-nine filesystem shutdowns from + hours earlier. The runner now writes `run.json`, which `ArchiveEvidence` prefers over + inferring a window from whatever happened to get logged. + +Both have regression tests. The lesson generalises: the detectors were verifiable offline, the +components were not, and only the components had these bugs. + +## Not yet here + +Deliberate gaps, so nobody looks for them: + +* **No snapshots during the run.** `operator/test/fio_migration_test.py` can take a + `VolumeSnapshot` before a migration and re-check afterwards that the backend snapshot still + resolves (`--snapshot-chance`). Not ported: it is a second dimension on top of migration, and + the corruption work ran with it at 0. +* **The driver migrates one volume at a time, in round-robin order.** Deliberate — concurrent + migrations of different subsystems make every host-side observation ambiguous about which one + caused it — but it does mean the concurrent case is untested by this framework. +* **No fault injection.** Killing a node mid-cutover, partitioning the fabric, or filling a + volume are the obvious next scenarios, and nothing here does them yet. +* **No role labelling in `ana.sample` output.** `ana.split-brain` and + `ana.unserved-after-cutover` need source/target roles per listener and skip without them. + Left open on purpose rather than half-done: the driver now knows the source and target node + UUIDs, and `sbctl storage-node list` gives each node's management IP, so labelling by IP is + easy — and wrong. One host serves both sides of a migration on different ports, so an IP + names a *node* while only `ip:port` names a *path*. The existing harness labelled by IP and + read a third-party replica's listener as the source's, which inverted a cutover/revert + conclusion in one review. What is actually missing is the port → lvstore mapping; until that + is available, these two detectors correctly skip instead of guessing. +* **The phase stamp is best-effort.** `set_phase` records the last phase the driver *saw*, + polling every few seconds, so a phase shorter than the poll interval never lands on a sample. + The ANA transitions themselves are sampled independently and are unaffected; only the label + is coarse. \ No newline at end of file diff --git a/test/framework/pyproject.toml b/test/framework/pyproject.toml new file mode 100644 index 000000000..18f9755c1 --- /dev/null +++ b/test/framework/pyproject.toml @@ -0,0 +1,67 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "sbtest" +version = "0.1.0" +description = "Component/detector framework for large-scale simplyblock test runs" +requires-python = ">=3.11" +# Suites are YAML, because a suite is a document to be read and argued with — every threshold +# in one wants a comment saying why it is that number, and JSON cannot carry one. Everything +# runs out of a venv (`make` builds it), so a single dependency costs nothing. +dependencies = ["pyyaml>=6"] + +[project.optional-dependencies] +dev = ["ruff>=0.6", "mypy>=1.11"] + +[project.scripts] +sbtest = "sbtest.cli:main" + +[tool.setuptools.packages.find] +include = ["sbtest*"] + +[tool.setuptools.package-data] +sbtest = ["suites/*.yaml"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4", "RET"] +ignore = [ + "E501", # long lines are the prose in docstrings, not code + "B008", +] + +# PTH (pathlib-over-os.path) is deliberately not selected: operator/test/fio_migration_test.py +# and everything around it use os.path, and one module in a different idiom is worse than a +# consistent old one. + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["E402"] # sys.path juggling before imports is the point in tests + +[tool.mypy] +python_version = "3.11" +files = ["sbtest", "tests"] +# Strict where it buys something. The framework's whole contract is that a detector reads +# Evidence and returns Findings, so that is exactly the boundary worth type-checking. +warn_unused_configs = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = true +no_implicit_optional = true +strict_equality = true +check_untyped_defs = true +disallow_untyped_defs = false # lifecycle hooks and tests read better untyped +disallow_incomplete_defs = true + +[[tool.mypy.overrides]] +module = "yaml" +ignore_missing_imports = true # PyYAML ships no stubs + +[[tool.mypy.overrides]] +# Tests build deliberately-wrong plugins to prove the runner survives them. +module = ["test_core", "test_detectors"] +disallow_incomplete_defs = false diff --git a/test/framework/requirements-dev.txt b/test/framework/requirements-dev.txt new file mode 100644 index 000000000..e74637bdf --- /dev/null +++ b/test/framework/requirements-dev.txt @@ -0,0 +1,3 @@ +# Optional: only needed to author YAML suites and to lint. +pyyaml>=6 +ruff>=0.6 diff --git a/test/framework/sbtest/__init__.py b/test/framework/sbtest/__init__.py new file mode 100644 index 000000000..1f4f7377f --- /dev/null +++ b/test/framework/sbtest/__init__.py @@ -0,0 +1,17 @@ +"""sbtest — a component-and-detector framework for large-scale simplyblock test runs. + +Two extension points: **components** do the side-effecting work of a run (drive a workload, +sample paths, follow logs) and can each be enabled or disabled independently; **detectors** +judge the evidence a run produced and can be added without touching collection. + +Importing this package registers the bundled components and detectors, so +`known_detectors()` and `known_components()` are populated after `import sbtest`. +""" + +from . import components as _components # noqa: F401 (registers via decorators) +from . import detectors as _detectors # noqa: F401 +from .core import * # noqa: F401,F403 +from .core import __all__ as _core_all + +__all__ = list(_core_all) +__version__ = "0.1.0" diff --git a/test/framework/sbtest/__main__.py b/test/framework/sbtest/__main__.py new file mode 100644 index 000000000..dd8a8c90e --- /dev/null +++ b/test/framework/sbtest/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/test/framework/sbtest/adapters/__init__.py b/test/framework/sbtest/adapters/__init__.py new file mode 100644 index 000000000..20ec71a38 --- /dev/null +++ b/test/framework/sbtest/adapters/__init__.py @@ -0,0 +1,11 @@ +"""Evidence adapters — where a detector's input comes from. + +`ArchiveEvidence` reads a finished run directory, which is what lets the detector set be +re-run against any past run. A live run uses `LiveEvidence`, assembled by the components +that collected it. +""" + +from .archive import ArchiveEvidence +from .live import LiveEvidence + +__all__ = ["ArchiveEvidence", "LiveEvidence"] diff --git a/test/framework/sbtest/adapters/archive.py b/test/framework/sbtest/adapters/archive.py new file mode 100644 index 000000000..af49cdcbb --- /dev/null +++ b/test/framework/sbtest/adapters/archive.py @@ -0,0 +1,463 @@ +"""ArchiveEvidence — read a finished run directory as Evidence. + +This is what makes a check testable against the run that motivated it. The layout it reads +is the one `operator/test/fio_migration_test.py` writes, so every archived fio-mig-* run +becomes a fixture for the whole detector set: + + / + state.json run bookkeeping (migrations, pods, pv/nqn maps) + test.log event log; the fallback when state.json is absent + ana/-mig-N.csv host ANA samples per migration + -fio-N/result.json fio's own summary + -fio-N/fio.log the pod's container log (verify failures live here) + -fio-N/timeseries.csv per-second IOPS + spdk-[-proxy].txt host-sourced container logs + operator.txt / webappapi.txt likewise + dmesg-.txt kernel ring buffer per storage worker + nvme-controllers.json fabric snapshot, when the run collected one + +Nothing here is required. A missing file makes the corresponding accessor return nothing, +which makes the detectors that need it report themselves skipped rather than clean — the +distinction the whole findings model is built around. +""" + +from __future__ import annotations + +import csv +import glob +import json +import os +import re +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +from ..core import ( + AnaSample, + ControlEvent, + FioJob, + IopsSample, + LogSpan, + Migration, + NvmeController, +) + + +def _dt(v: object) -> datetime | None: + if not v: + return None + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + try: + d = datetime.fromisoformat(str(v).replace("Z", "+00:00")) + except ValueError: + return None + return d if d.tzinfo else d.replace(tzinfo=UTC) + + +_TS_PATTERNS = ( + # CRI container log: 2026-08-19T22:23:18.994807954Z stderr F + (re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})"), "%Y-%m-%dT%H:%M:%S"), + # dmesg -T: [Thu Aug 20 05:46:57 2026] + (re.compile(r"^\[(\w{3} \w{3}\s+\d+ \d{2}:\d{2}:\d{2} \d{4})\]"), + "%a %b %d %H:%M:%S %Y"), +) + + +def _log_line_ts(raw: str) -> datetime | None: + """The timestamp of a collected log line, whichever of the known formats it uses.""" + for rx, fmt in _TS_PATTERNS: + m = rx.match(raw) + if m: + try: + return datetime.strptime(m.group(1), fmt).replace(tzinfo=UTC) + except ValueError: + return None + return None + + +class ArchiveEvidence: + """Evidence backed by a finished run directory.""" + + def __init__(self, outdir: str) -> None: + self.outdir = os.path.abspath(outdir) + if not os.path.isdir(self.outdir): + raise FileNotFoundError(self.outdir) + self._state = self._load_state() + self.run_id = self._state.get("run_id") or os.path.basename(self.outdir).replace( + "fio-mig-", "fiomig-") + self._ana_cache: dict[str, list[AnaSample]] = {} + self._fio_start_cache: dict[str, datetime | None] = {} + + # ── bookkeeping ──────────────────────────────────────────────────────────────── + + def _load_state(self) -> dict: + p = os.path.join(self.outdir, "state.json") + if os.path.exists(p): + try: + with open(p) as fh: + loaded = json.load(fh) + if isinstance(loaded, dict): + return loaded + except (OSError, json.JSONDecodeError): + pass + return self._state_from_test_log() + + def _state_from_test_log(self) -> dict: + """Rebuild just enough from test.log when state.json is missing. + + Only the migration timeline is recovered, because that is what attribution needs and + it is the one thing the event log records unambiguously. + """ + p = os.path.join(self.outdir, "test.log") + if not os.path.exists(p): + return {} + migs: dict[str, dict] = {} + start_re = re.compile( + r"^(\S+) .*MIGRATION START\s+(\S+)\s+.*?(?:pod=(\S+))?\s*(?:pv=(\S+))?") + stop_re = re.compile(r"^(\S+) .*MIGRATION STOP\s+(\S+)\s+phase=(\S+)") + with open(p, errors="ignore") as fh: + for line in fh: + m = start_re.match(line) + if m and "MIGRATION START" in line: + name = m.group(2) + rec = migs.setdefault(name, {"name": name}) + rec["start"] = m.group(1) + for key, pat in (("pod", r"pod=(\S+)"), ("pv", r"pv=(\S+)"), + ("source", r"source=(\S+)"), ("target", r"target=(\S+)")): + mm = re.search(pat, line) + if mm: + rec[key] = mm.group(1) + mm = re.search(r"moves along: ([^)]*)\)", line) + rec["group_pvs"] = ([x.strip() for x in mm.group(1).split(",")] + [rec.get("pv", "")] + if mm else [rec.get("pv", "")]) + continue + m = stop_re.match(line) + if m: + rec = migs.setdefault(m.group(2), {"name": m.group(2)}) + rec["end"] = m.group(1) + rec["phase"] = m.group(3) + mm = re.search(r"error='([^']*)'", line) + if mm: + rec["error"] = mm.group(1) + return {"migrations": list(migs.values())} + + # ── Evidence protocol ────────────────────────────────────────────────────────── + + def migrations(self) -> list[Migration]: + # The framework's own driver writes migrations.json; the older harness writes them + # inside state.json. Prefer whichever is present, so one reader serves both. + p = os.path.join(self.outdir, "migrations.json") + if os.path.exists(p) and not self._state.get("migrations"): + from ..components.migration import migrations_from_file + try: + return migrations_from_file(p) + except (OSError, json.JSONDecodeError, ValueError): + pass + out = [] + for m in self._state.get("migrations", []): + start = _dt(m.get("start")) + if not start: + continue + cut = {} + for node, ts in (m.get("ana_cutover") or {}).items(): + d = _dt(ts) + if d: + cut[node] = d + out.append(Migration( + name=m.get("name", ""), start=start, end=_dt(m.get("end")), + phase=m.get("phase", ""), source=m.get("source", ""), + target=m.get("target", ""), pv=m.get("pv", ""), pod=m.get("pod", ""), + members=[x for x in (m.get("group_pvs") or []) if x], + error=m.get("error", "") or "", cutover=cut)) + out.sort(key=lambda x: x.start) + return out + + def ana_samples(self, migration: str) -> list[AnaSample]: + if migration in self._ana_cache: + return self._ana_cache[migration] + # state.json records an absolute path from the original host, which may not exist + # here; resolve by name inside this directory instead. + cands = [os.path.join(self.outdir, "ana", f"{migration}.csv")] + cands += glob.glob(os.path.join(self.outdir, "ana", f"*{migration}.csv")) + path = next((c for c in cands if os.path.exists(c)), None) + samples: list[AnaSample] = [] + if path: + grouped: dict[tuple, AnaSample] = {} + try: + with open(path, newline="") as fh: + for r in csv.DictReader(fh): + ts = _dt(r.get("ts")) + if not ts: + continue + key = (ts, r.get("node", ""), r.get("address", "")) + s = grouped.get(key) + if s is None: + s = AnaSample(ts=ts, node=r.get("node", ""), + address=r.get("address", ""), + state=r.get("ctrl_state", ""), + ana={}, phase=r.get("phase", ""), + role=r.get("role", "")) + grouped[key] = s + if r.get("nsid"): + s.ana[int(r["nsid"])] = r.get("ana_state", "") + except OSError: + pass + samples = [grouped[k] for k in sorted(grouped)] + self._ana_cache[migration] = samples + return samples + + def pods(self) -> list[str]: + pods = self._state.get("pods") + if pods: + return list(pods) + return sorted( + d for d in os.listdir(self.outdir) + if os.path.isdir(os.path.join(self.outdir, d)) and "-fio-" in d) + + def _result_json(self, pod: str) -> dict: + p = os.path.join(self.outdir, pod, "result.json") + if not os.path.exists(p): + return {} + try: + with open(p) as fh: + loaded = json.load(fh) + except (OSError, json.JSONDecodeError): + return {} + return loaded if isinstance(loaded, dict) else {} + + def _fio_start(self, pod: str) -> datetime | None: + """The wall clock of second 0 of this pod's fio time series. + + Taken from fio's own `job_start` (epoch milliseconds) rather than from the + wall_clock column, because the column is only as right as whatever wrote it: runs + archived before the base was fixed derived it from a shared run-start stamp taken + minutes before each pod's fio actually began. fio's offsets are milliseconds since + *that job* started, so the job's own start is the only base that is right by + construction, and it is right for old archives too — which is the point, since a + detector that cannot be re-run against the run that motivated it is a detector + nobody trusts. + + Falls back to the CSV's own column when result.json is missing or carries no + job_start. + """ + if pod not in self._fio_start_cache: + jobs = self._result_json(pod).get("jobs") or [] + start_ms = jobs[0].get("job_start") if jobs else None + self._fio_start_cache[pod] = ( + datetime.fromtimestamp(start_ms / 1000.0, tz=UTC) + if isinstance(start_ms, int | float) and start_ms > 0 else None) + return self._fio_start_cache[pod] + + def fio_jobs(self) -> list[FioJob]: + out = [] + for pod in self.pods(): + res = self._result_json(pod) + if not res: + continue + for job in res.get("jobs", []): + rd, wr = job.get("read", {}), job.get("write", {}) + out.append(FioJob( + pod=pod, error=int(job.get("error", 0) or 0), + read_iops=float(rd.get("iops", 0) or 0), + write_iops=float(wr.get("iops", 0) or 0), + total_iops=float(rd.get("iops", 0) or 0) + float(wr.get("iops", 0) or 0))) + return out + + def fio_timeseries(self, pod: str) -> list[IopsSample]: + p = os.path.join(self.outdir, pod, "timeseries.csv") + if not os.path.exists(p): + return [] + out = [] + # The offset column is `second` in the harness's own CSVs and `t`/`offset_s` in + # fio's raw logs. Reading only the latter silently placed every sample at offset 0, + # which does not look like a parse failure — the IOPS column still parsed, so the + # series was the right length with the whole run collapsed onto one instant. Anything + # that locates an outage in time was reading a flatline. + def _first(r: dict[str, str], *keys: str) -> str: + for k in keys: + v = r.get(k) + if v not in (None, ""): + return v + return "" + + base = self._fio_start(pod) + try: + with open(p, newline="") as fh: + for r in csv.DictReader(fh): + try: + off = int(float(_first(r, "second", "t", "offset_s") or 0)) + tot = float(_first(r, "total_iops") or 0) + except (TypeError, ValueError): + continue + wall = (base + timedelta(seconds=off) if base + else _dt(_first(r, "wall", "wall_clock"))) + out.append(IopsSample(offset_s=off, wall=wall, total_iops=tot)) + except OSError: + return [] + out.sort(key=lambda s: s.offset_s) + return out + + def fio_log(self, pod: str) -> Iterator[str]: + p = os.path.join(self.outdir, pod, "fio.log") + if not os.path.exists(p): + return iter(()) + return self._lines(p) + + def container_logs(self) -> list[str]: + names = [] + for p in sorted(glob.glob(os.path.join(self.outdir, "*.txt"))): + base = os.path.basename(p)[:-4] + if base in ("test", "REVIEW"): + continue + names.append(base) + return names + + def container_log(self, name: str) -> Iterator[str]: + p = os.path.join(self.outdir, f"{name}.txt") + if not os.path.exists(p): + return iter(()) + return self._lines(p) + + def run_window(self) -> tuple[datetime | None, datetime | None]: + """From the event log's first and last stamps, falling back to the migrations. + + test.log is preferred because it brackets the whole run including setup and + collection, while the migrations only cover the part that was migrating. + """ + # run.json is authoritative when present: the run wrote it, rather than it being + # inferred from whatever happened to get logged. + p = os.path.join(self.outdir, "run.json") + if os.path.exists(p): + try: + with open(p) as fh: + rec = json.load(fh) + start, end = _dt(rec.get("start")), _dt(rec.get("end")) + if start: + return start, end + except (OSError, json.JSONDecodeError): + pass + + p = os.path.join(self.outdir, "test.log") + if os.path.exists(p): + first = last = None + try: + with open(p, errors="ignore") as fh: + for line in fh: + ts = _dt(line[:20].strip()) + if ts: + first = first or ts + last = ts + except OSError: + pass + if first: + return first, last + migs = self.migrations() + if migs: + ends = [m.end for m in migs if m.end] + return migs[0].start, (max(ends) if ends else None) + return None, None + + def control_events(self) -> list[ControlEvent]: + p = os.path.join(self.outdir, "cluster-events.json") + if not os.path.exists(p): + return [] + try: + with open(p) as fh: + raw = json.load(fh) + except (OSError, json.JSONDecodeError): + return [] + out = [] + for e in raw if isinstance(raw, list) else []: + ts = _dt(str(e.get("Date", "")).replace(" ", "T")) + if not ts: + continue + out.append(ControlEvent( + ts=ts, level=str(e.get("Level", "")), kind=str(e.get("Event", "")), + message=str(e.get("Message", "")), + subject=str(e.get("NodeId") or e.get("Storage_ID") or ""))) + out.sort(key=lambda x: x.ts) + return out + + def log_spans(self) -> list[LogSpan]: + """First and last timestamp of every collected log, however it stamps its lines. + + Three formats appear in one artifact directory — CRI (`2026-...Z stderr F`), dmesg + ctime (`[Thu Aug 20 ...]`) and dmesg ISO — so each is tried per line until one sticks. + + The bounds are the **minimum and maximum** timestamp, not the first and last line's. + That distinction is not pedantry: an artifact holding several containers is written one + container after another, so it is not in time order, and reading the last line gives + whatever the last *container* happened to say. Doing that made a 52 MiB log covering + two hours look like thirteen seconds. + """ + spans = [] + for name in self.container_logs(): + lo = hi = None + lines = 0 + for raw in self.container_log(name): + lines += 1 + ts = _log_line_ts(raw) + if ts is None: + continue + if lo is None or ts < lo: + lo = ts + if hi is None or ts > hi: + hi = ts + spans.append(LogSpan(name=name, first=lo, last=hi, lines=lines)) + return spans + + def cluster_uuid(self) -> str: + """The run's cluster, from the NQNs it recorded, else from the event log.""" + for nqn in (self._state.get("nqn_of") or {}).values(): + m = re.search(r"simplyblock:([0-9a-f-]{36}):", str(nqn)) + if m: + return m.group(1) + p = os.path.join(self.outdir, "test.log") + if os.path.exists(p): + try: + with open(p, errors="ignore") as fh: + for line in fh: + m = re.search(r"live cluster.*=\s*([0-9a-f-]{36})", line) + if m: + return m.group(1) + except OSError: + pass + return "" + + def nvme_controllers_pre(self) -> list[NvmeController]: + """The fabric as it was *before* the run — the state the run inherited. + + Optional, and not part of the Evidence protocol: only nvme.dirty-start wants it, and + a detector that needs it can ask with getattr and skip when it is absent. + """ + return self._controllers_from("nvme-controllers-pre.json") + + def nvme_controllers(self) -> list[NvmeController]: + return self._controllers_from("nvme-controllers-post.json") or \ + self._controllers_from("nvme-controllers.json") + + def _controllers_from(self, filename: str) -> list[NvmeController]: + p = os.path.join(self.outdir, filename) + if not os.path.exists(p): + return [] + try: + with open(p) as fh: + raw = json.load(fh) + except (OSError, json.JSONDecodeError): + return [] + out = [] + for c in raw if isinstance(raw, list) else raw.get("controllers", []): + out.append(NvmeController( + node=c.get("node", ""), name=c.get("name", ""), nqn=c.get("nqn", ""), + address=c.get("address", ""), state=c.get("state", ""), + namespaces={int(k): v for k, v in (c.get("namespaces") or {}).items()}, + ctrl_loss_tmo=c.get("ctrl_loss_tmo"))) + return out + + # ── helpers ──────────────────────────────────────────────────────────────────── + + @staticmethod + def _lines(path: str) -> Iterator[str]: + """Stream a log line by line. These files reach tens of megabytes.""" + with open(path, errors="ignore") as fh: + yield from fh diff --git a/test/framework/sbtest/adapters/live.py b/test/framework/sbtest/adapters/live.py new file mode 100644 index 000000000..1f45a2de8 --- /dev/null +++ b/test/framework/sbtest/adapters/live.py @@ -0,0 +1,37 @@ +"""LiveEvidence — evidence assembled from a run that just happened. + +A thin thing on purpose. Components write their artifacts into the run directory in the +same layout an archive uses, and record what they observed on the timeline; so a live run's +evidence is the archive reader pointed at the directory being written, overlaid with the +timeline for the parts that are not files yet. + +Keeping the two paths this close is deliberate: if live evidence and archived evidence +diverge, a check that passes during a run can fail on replay, and then nobody trusts either. +""" + +from __future__ import annotations + +from ..core import AnaSample, Migration, NvmeController, RunContext +from .archive import ArchiveEvidence + + +class LiveEvidence(ArchiveEvidence): + def __init__(self, ctx: RunContext) -> None: + super().__init__(ctx.outdir) + self.ctx = ctx + self.run_id = ctx.run_id + self._live_ana: dict[str, list[AnaSample]] = ctx.shared.get("ana.samples", {}) + self._live_migs: list[Migration] = ctx.shared.get("migrations", []) + self._live_ctrls: list[NvmeController] = ctx.shared.get("nvme.controllers", []) + + def migrations(self) -> list[Migration]: + return self._live_migs or super().migrations() + + def ana_samples(self, migration: str) -> list[AnaSample]: + return self._live_ana.get(migration) or super().ana_samples(migration) + + def nvme_controllers(self) -> list[NvmeController]: + return self._live_ctrls or super().nvme_controllers() + + def cluster_uuid(self) -> str: + return str(self.ctx.shared.get("cluster.uuid") or super().cluster_uuid()) diff --git a/test/framework/sbtest/cli.py b/test/framework/sbtest/cli.py new file mode 100644 index 000000000..a83a1af1c --- /dev/null +++ b/test/framework/sbtest/cli.py @@ -0,0 +1,241 @@ +"""sbtest command line. + + sbtest detectors what can judge a run, and with which options + sbtest components what can run during a run + sbtest analyze [...] judge a finished run directory + sbtest collect [...] observe a cluster, then collect the evidence + sbtest run --suite [...] drive a full test run, then judge it + +`analyze` is the one to reach for first. It runs the whole detector set over an artifact +directory — including every archived fio-mig-* run — so a check can be written or fixed and +tried immediately against the run that motivated it, instead of against the next four-hour +run. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +from . import __version__ +from .adapters import ArchiveEvidence +from .core import ( + Config, + Logger, + RunContext, + Runner, + apply_cli_toggles, + known_components, + known_detectors, + load, + now_utc, + suite_path, +) +from .detectors.ana import freeze_summary + + +def _cfg(args: argparse.Namespace, known_c: list[str], known_d: list[str]) -> Config: + path = suite_path(args.suite) if args.suite else None + if args.suite and not path: + raise SystemExit(f"suite not found: {args.suite}") + cfg = load(path, known_c, known_d) + apply_cli_toggles(cfg.components, args.enable_component, args.disable_component) + apply_cli_toggles(cfg.detectors, args.enable_detector, args.disable_detector) + return cfg + + +def _first_doc_line(cls: type) -> str: + """The first line of a class docstring, or "" — a class may have none at all.""" + doc = (cls.__doc__ or "").strip() + return doc.splitlines()[0] if doc else "" + + +def cmd_list(args: argparse.Namespace) -> int: + if args.what == "detectors": + for name, dcls in known_detectors().items(): + d = dcls() + print(name) + print(f" {d.summary or _first_doc_line(dcls)}") + for k, v in sorted(d.defaults().items()): + print(f" {k} = {v!r}") + else: + for name, ccls in known_components().items(): + c = ccls() + print(name) + print(f" {c.summary or _first_doc_line(ccls)}") + for k, v in sorted(c.defaults().items()): + print(f" {k} = {v!r}") + return 0 + + +def cmd_analyze(args: argparse.Namespace) -> int: + ev = ArchiveEvidence(args.rundir) + outdir = args.outdir or args.rundir + log = Logger(os.path.join(outdir, "sbtest-analyze.log") if args.write_log else None, + verbose=args.verbose) + ctx = RunContext(run_id=ev.run_id, outdir=outdir, log=log) + + cfg = _cfg(args, list(known_components()), list(known_detectors())) + # analyze never runs components; make that explicit rather than silently ignoring them. + if cfg.components.enabled: + log.info("analyze: components are not run (" + + ", ".join(sorted(cfg.components.enabled)) + ")") + cfg.components.enabled = {} + + log.info(f"analyzing {ev.outdir} (run {ev.run_id})") + runner = Runner(cfg, ctx).build() + runner.judge(ev) + + if args.freeze_table: + rows = freeze_summary(ev) + if rows: + log.info("cutover freezes per migration (>1 has predicted write loss exactly):") + for name, n, worst in sorted(rows, key=lambda r: (-r[1], r[0])): + mark = " <== re-froze" if n > 1 else "" + log.info(f" {name:28} freezes={n:<3} worst={worst:.0f}s{mark}") + + report = runner.emit(json_name=args.json_name) + from .core.runner import findings_by_subject_table + correlated = findings_by_subject_table(report) + if correlated: + log.info("subjects with more than one finding (the strongest signal these runs give):") + for line in correlated: + log.info(f" {line}") + return 1 if report.failed else 0 + + +def cmd_collect(args: argparse.Namespace) -> int: + os.makedirs(args.outdir, exist_ok=True) + run_id = args.run_id or f"sbtest-{int(now_utc().timestamp())}" + log = Logger(os.path.join(args.outdir, "sbtest.log"), verbose=args.verbose) + ctx = RunContext(run_id=run_id, outdir=args.outdir, log=log) + cfg = _cfg(args, list(known_components()), list(known_detectors())) + if not cfg.components.enabled: + raise SystemExit("collect: no components enabled — pass --enable-component or a suite") + + runner = Runner(cfg, ctx).build() + log.info(f"collecting into {args.outdir} for {args.duration}s") + try: + runner.setup() + runner.start() + if args.duration: + runner.run_for(float(args.duration)) + runner.stop() + runner.collect() + finally: + runner.teardown() + + if args.judge: + runner.judge(ArchiveEvidence(args.outdir)) + report = runner.emit(json_name=args.json_name) + return 1 if report.failed else 0 + log.info(f"collected into {args.outdir}; judge it with: sbtest analyze {args.outdir}") + return 0 + + +def cmd_run(args: argparse.Namespace) -> int: + """A driven run: components create the load, then the same detectors judge it. + + The difference from `collect` is only what the components do — one of them drives + migrations instead of watching them. Everything after that is identical, which is the + point: a live run and an archived one are judged by the same code, so a verdict from + either means the same thing. + """ + cfg = _cfg(args, list(known_components()), list(known_detectors())) + run_id = args.run_id or f"{cfg.run_id_prefix}-{int(now_utc().timestamp())}" + outdir = args.outdir or os.path.join(cfg.outdir, run_id) + os.makedirs(outdir, exist_ok=True) + log = Logger(os.path.join(outdir, "test.log"), verbose=args.verbose) + ctx = RunContext(run_id=run_id, outdir=outdir, log=log) + # Read by the components that create cluster objects. A kept run leaves the volumes and + # the CRs behind for inspection — the reason most post-mortems are possible at all. + ctx.shared["keep"] = bool(args.keep) + + if not cfg.components.enabled: + raise SystemExit("run: no components enabled — pass --enable-component or a suite") + drivers = [n for n in cfg.components.enabled if n.endswith(".driver")] + if not drivers: + log.warn("no driver component is enabled, so nothing will create load — this run " + "will only observe. Use `sbtest collect` if that is what you meant") + + runner = Runner(cfg, ctx).build() + log.info(f"run {run_id} -> {outdir} (duration {args.duration or 0:.0f}s, " + f"keep={bool(args.keep)})") + try: + runner.setup() + runner.start() + if args.duration: + runner.run_for(float(args.duration)) + runner.stop() + runner.collect() + finally: + runner.teardown() + + # Judged from the artifact directory rather than from memory, so the verdict comes from + # exactly the evidence someone else would re-analyse later. A run that reports PASS from + # in-memory state it never wrote down is not reproducible. + runner.judge(ArchiveEvidence(outdir)) + report = runner.emit(json_name=args.json_name) + return 1 if report.failed else 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="sbtest", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--version", action="version", version=f"sbtest {__version__}") + sub = p.add_subparsers(dest="cmd", required=True) + + def common(sp: argparse.ArgumentParser) -> None: + sp.add_argument("--suite", help="suite file, or a bare name under sbtest/suites/") + sp.add_argument("--enable-detector", action="append", default=[], metavar="NAME") + sp.add_argument("--disable-detector", action="append", default=[], metavar="NAME") + sp.add_argument("--enable-component", action="append", default=[], metavar="NAME") + sp.add_argument("--disable-component", action="append", default=[], metavar="NAME") + sp.add_argument("--json-name", default="findings.json", + help="where to write findings inside the run directory") + sp.add_argument("-v", "--verbose", action="store_true") + + for what in ("detectors", "components"): + sp = sub.add_parser(what, help=f"list the available {what}") + sp.set_defaults(func=cmd_list, what=what) + + sp = sub.add_parser("analyze", help="judge a finished run directory") + sp.add_argument("rundir") + sp.add_argument("--outdir", help="where to write findings (default: the run directory)") + sp.add_argument("--write-log", action="store_true", + help="also write sbtest-analyze.log into the output directory") + sp.add_argument("--freeze-table", action="store_true", + help="print freezes per migration alongside the findings") + common(sp) + sp.set_defaults(func=cmd_analyze) + + sp = sub.add_parser("collect", help="run the collection components against a cluster") + sp.add_argument("outdir") + sp.add_argument("--duration", type=float, default=0.0, + help="seconds to keep components running before collecting") + sp.add_argument("--run-id") + sp.add_argument("--judge", action="store_true", help="judge the result when done") + common(sp) + sp.set_defaults(func=cmd_collect) + + sp = sub.add_parser("run", help="drive a full test run against a cluster, then judge it") + sp.add_argument("--duration", type=float, default=0.0, + help="seconds to run before stopping the drivers and collecting") + sp.add_argument("--outdir", help="where to write artifacts " + "(default: /)") + sp.add_argument("--run-id") + sp.add_argument("--keep", action="store_true", + help="leave the volumes, pods and CRs behind for inspection") + common(sp) + sp.set_defaults(func=cmd_run) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return int(args.func(args)) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/test/framework/sbtest/components/__init__.py b/test/framework/sbtest/components/__init__.py new file mode 100644 index 000000000..b472f9819 --- /dev/null +++ b/test/framework/sbtest/components/__init__.py @@ -0,0 +1,9 @@ +"""Bundled components. Importing this module registers them all. + +Grouped by what they touch: `logs` collects container logs (live or post-run), `nvme` +observes the host fabric, `events` pulls the control plane's own event log. +""" + +from . import events, logs, migration, nvme, workload # noqa: F401 + +__all__ = ["events", "logs", "migration", "nvme", "workload"] diff --git a/test/framework/sbtest/components/events.py b/test/framework/sbtest/components/events.py new file mode 100644 index 000000000..2e4600033 --- /dev/null +++ b/test/framework/sbtest/components/events.py @@ -0,0 +1,60 @@ +"""Control-plane event log collection. + +Small, cheap, and disproportionately useful: it is the only source that says what the +control plane *thought* it was doing, which is what turns a host-side symptom into a +diagnosis. +""" + +from __future__ import annotations + +import json +from typing import Any + +from ..core import Component, RunContext, component +from . import kube + + +@component +class ClusterEvents(Component): + """Dump the simplyblock cluster event log via sbctl inside a webappapi pod.""" + + name = "cluster.events" + summary = "sbctl cluster get-logs -> cluster-events.json" + + def defaults(self) -> dict[str, Any]: + return {"namespace": "simplyblock", "pod_prefix": "simplyblock-webappapi", + "limit": 50000, "cluster_uuid": None} + + def collect(self, ctx: RunContext) -> None: + pods = kube.list_pods(self.opt("namespace"), [self.opt("pod_prefix")]) + if not pods: + ctx.log.warn(f"{self.name}: no {self.opt('pod_prefix')} pod; skipping") + return + pod = pods[0].name + cluster = self.opt("cluster_uuid") or ctx.shared.get("cluster.uuid") + if not cluster: + out = kube.exec_sh(self.opt("namespace"), pod, + "sbctl cluster list --json 2>/dev/null || true", timeout=60) + try: + items = json.loads(out) + cluster = (items[0].get("UUID") or items[0].get("uuid")) if items else None + except (json.JSONDecodeError, AttributeError, IndexError): + cluster = None + if not cluster: + ctx.log.warn(f"{self.name}: cannot resolve the cluster uuid; skipping") + return + + out = kube.exec_sh( + self.opt("namespace"), pod, + f"sbctl cluster get-logs {cluster} --json --limit={int(self.opt('limit'))}", + timeout=180) + if not out.strip(): + ctx.log.warn(f"{self.name}: sbctl returned nothing") + return + path = ctx.path("cluster-events.json") + with open(path, "w") as fh: + fh.write(out) + try: + ctx.log.info(f"{self.name}: {len(json.loads(out))} entries -> {path}") + except json.JSONDecodeError: + ctx.log.info(f"{self.name}: -> {path} (not valid JSON)") diff --git a/test/framework/sbtest/components/kube.py b/test/framework/sbtest/components/kube.py new file mode 100644 index 000000000..3c3e83166 --- /dev/null +++ b/test/framework/sbtest/components/kube.py @@ -0,0 +1,68 @@ +"""Minimal kubectl plumbing shared by the cluster-touching components. + +Deliberately subprocess-over-kubectl rather than a client library: it keeps the framework +stdlib-only, it works with whatever kubeconfig and context the operator already uses, and +every call it makes is one a human can paste into a terminal when a run misbehaves — which +is most of the debugging value. +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass + + +class KubectlError(RuntimeError): + pass + + +def run(args: list[str], timeout: int = 60, check: bool = True, + stdin: str | None = None) -> subprocess.CompletedProcess[str]: + cp = subprocess.run(["kubectl", *args], input=stdin, capture_output=True, + text=True, timeout=timeout, check=False) + if check and cp.returncode != 0: + raise KubectlError(f"kubectl {' '.join(args)}: {cp.stderr.strip() or cp.returncode}") + return cp + + +def run_bytes(args: list[str], timeout: int = 300) -> bytes: + cp = subprocess.run(["kubectl", *args], capture_output=True, timeout=timeout, check=False) + return cp.stdout + + +@dataclass(frozen=True) +class Pod: + name: str + namespace: str + node: str + containers: tuple[str, ...] + phase: str = "" + + +def list_pods(namespace: str, name_contains: list[str] | None = None) -> list[Pod]: + cp = run(["-n", namespace, "get", "pods", "-o", "json"]) + out = [] + for it in json.loads(cp.stdout).get("items", []): + name = it["metadata"]["name"] + if name_contains and not any(s in name for s in name_contains): + continue + out.append(Pod( + name=name, namespace=namespace, + node=it.get("spec", {}).get("nodeName", "") or "", + containers=tuple(c["name"] for c in it.get("spec", {}).get("containers", [])), + phase=it.get("status", {}).get("phase", "") or "")) + return out + + +def exec_sh(namespace: str, pod: str, script: str, container: str | None = None, + timeout: int = 300) -> str: + args = ["-n", namespace, "exec", pod] + if container: + args += ["-c", container] + args += ["--", "sh", "-c", script] + return run(args, timeout=timeout, check=False).stdout + + +def short(node: str) -> str: + return node.split(".")[0] diff --git a/test/framework/sbtest/components/logs.py b/test/framework/sbtest/components/logs.py new file mode 100644 index 000000000..b29a26acf --- /dev/null +++ b/test/framework/sbtest/components/logs.py @@ -0,0 +1,486 @@ +"""Container-log collection: a live follower and a post-run grab. + +Two components rather than one flag, because they are genuinely different mechanisms with +different failure modes, and the reason to have both is worth stating. + +The kubelet keeps only `containerLogMaxSize x containerLogMaxFiles` per container — 10Mi x 5 +on the clusters this runs against — and a busy SPDK container writes that in well under an +hour. A post-run grab of a four-hour run therefore returns its last forty minutes and +silently drops the rest, which is how several runs' worth of early evidence was lost. + +So: `logs.stream` follows the high-volume logs for the whole run, and `logs.collect` grabs +everything else at the end. Enabling both is the normal configuration; the stream declares +which containers it owns so the grab skips them. +""" + +from __future__ import annotations + +import contextlib +import os +import subprocess +from typing import Any + +from ..core import Component, RunContext, component +from . import kube + +#: Where the host keeps CRI container logs. Mounted read-only into the grabber. +HOST_POD_LOGS = "/var/log/pods" + +#: A busybox-ish image with sh, tail and gzip. The fio image already satisfies this and is +#: guaranteed to be pullable wherever these tests run. +DEFAULT_GRABBER_IMAGE = "quay.io/simplyblock-io/fio:latest" + + +def _log_dir(namespace: str, pod: str, container: str, if_missing: str) -> str: + """Resolve the container's CRI log directory into $d. + + One directory per pod *UID*, so a pod recreated under the same name leaves two behind. + Take the most recently written rather than letting the glob expand to several words — + that would make every quoted use of $d a path that does not exist, which silently + empties the grab for exactly the pods that have been restarted. + """ + return (f'd=$(ls -1dt {HOST_POD_LOGS}/{namespace}_{pod}_*/{container}/ 2>/dev/null | head -1); ' + f'[ -n "$d" ] || {{ {if_missing}; }}; ') + + +#: The live file is ".log"; rotation renames it to ".log.[.gz]" and +#: opens a new one under the same name. Pick it by name, not mtime: a .gz written by the +#: rotation that just happened is briefly the newest file in the directory. +_CURRENT = 'cur=$(ls -1 "$d" 2>/dev/null | grep -E "^[0-9]+\\.log$" | sort -n | tail -1); ' + +#: `gzip -cd`, not `zcat`: the latter is a .Z-only alias on some hosts, which would silently +#: drop every gzipped — i.e. every older — segment. +_CAT_ONE = 'case "$f" in *.gz) gzip -cd "$d$f" 2>/dev/null;; *) cat "$d$f" 2>/dev/null;; esac; ' + + +def dump_script(namespace: str, pod: str, container: str) -> str: + """Everything the host still retains for one container, oldest segment first.""" + return (_log_dir(namespace, pod, container, "exit 0") + + 'for f in $(ls -1tr "$d" 2>/dev/null); do ' + _CAT_ONE + 'done') + + +def stream_script(namespace: str, pod: str, container: str, poll_s: int = 5) -> str: + """Retained segments, then follow the live one for as long as this runs. + + The dump and the follow are one command so there is no gap between them: rotated + segments are catted oldest-first and the live file is handed to `tail -F` from its first + line instead of being catted. + + Three things move the log out from under a follower and all three are handled, because + any one of them leaves the stream silent rather than failing: + + * **rotation** renames the live file and reopens the same name — `tail -F` follows by + name and reopens it itself. + * **a container restart** opens `.log` beside the old one. `tail -F` would wait + forever on a file that still exists and never grows, so the target is re-resolved on + a timer and the old follower killed. + * **a pod recreation** makes a whole new UID directory, which the same re-resolve picks + up. + + Only the first target emits the rotated history; on a later switch the history is the + file that was already being followed. A new target is read from its first line, so a + restart costs latency, not data. + """ + return ( + 'prev=""; tpid=""; ' + 'while :; do ' + + _log_dir(namespace, pod, container, f"sleep {poll_s}; continue") + + _CURRENT + + f'[ -n "$cur" ] || {{ sleep {poll_s}; continue; }}; ' + 'if [ "$d$cur" != "$prev" ]; then ' + ' [ -n "$tpid" ] && kill "$tpid" 2>/dev/null; ' + ' if [ -z "$prev" ]; then ' + ' for f in $(ls -1tr "$d" 2>/dev/null); do ' + ' [ "$f" = "$cur" ] && continue; ' + _CAT_ONE + + ' done; ' + ' fi; ' + ' tail -F -n +1 "$d$cur" 2>/dev/null & tpid=$!; ' + ' prev="$d$cur"; ' + 'fi; ' + f'sleep {poll_s}; ' + 'done') + + +class _GrabberBase(Component): + """Shared management of the privileged pod that reads the host's /var/log/pods. + + A Component rather than a bare mixin: it uses `opt` and `name`, so inheriting states + that dependency instead of leaving it to whatever it happens to be mixed into. + """ + + def _grabber_manifest(self, ctx: RunContext, node: str, name: str, ttl_s: int) -> str: + import json as _json + return _json.dumps({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": name, "namespace": self.opt("namespace"), + "labels": {"sbtest-run": ctx.run_id, "sbtest": "loggrab"}}, + "spec": { + "nodeName": node, "restartPolicy": "Never", + "tolerations": [{"operator": "Exists"}], + "containers": [{ + "name": "grab", "image": self.opt("image"), + "imagePullPolicy": "IfNotPresent", + "command": ["sh", "-c", f"sleep {ttl_s}"], + # privileged + runAsUser 0 is required to read the host's + # /var/log/pods under SELinux: without it the container runs as + # container_t and gets EACCES even as root, and every grab + # silently produces empty files. + "securityContext": {"privileged": True, "runAsUser": 0}, + "volumeMounts": [{"name": "pods", "mountPath": HOST_POD_LOGS, + "readOnly": True}], + }], + "volumes": [{"name": "pods", "hostPath": {"path": HOST_POD_LOGS}}], + }, + }) + + def _start_grabbers(self, ctx: RunContext, nodes: list[str], ttl_s: int) -> dict[str, str]: + # The component name is in the pod name deliberately. Two components can both want a + # grabber on the same node — the follower needs one for the whole run, the post-run + # grab needs one at the end — and a Pod is immutable, so sharing a name means the + # second one to apply fails on a field it is not allowed to change. Which it did: + # logs.collect silently produced empty files for every node logs.stream had claimed. + out: dict[str, str] = {} + slug = self.name.replace(".", "-") + for node in sorted(nodes): + name = f"sbtest-{slug}-{kube.short(node)}-{ctx.run_id}"[:63] + try: + kube.run(["apply", "-f", "-"], + stdin=self._grabber_manifest(ctx, node, name, ttl_s)) + except kube.KubectlError as e: + ctx.log.warn(f"{self.name}: cannot start grabber on {node}: {e}") + continue + out[node] = name + if out: + kube.run(["-n", self.opt("namespace"), "wait", "--for=condition=Ready", + *[f"pod/{n}" for n in out.values()], "--timeout=120s"], + timeout=140, check=False) + ready = {} + for node, name in out.items(): + cp = kube.run(["-n", self.opt("namespace"), "get", "pod", name, + "-o", "jsonpath={.status.phase}"], check=False, timeout=30) + if cp.stdout.strip() == "Running": + ready[node] = name + else: + ctx.log.warn(f"{self.name}: grabber on {node} is not Running; " + "logs from that node will be missing") + return ready + + def _delete_grabbers(self, ctx: RunContext, names: list[str]) -> None: + if not names: + return + kube.run(["-n", self.opt("namespace"), "delete", "pod", *names, + "--ignore-not-found", "--wait=false"], check=False) + + +@component +class LogStream(_GrabberBase): + """Follow the high-volume container logs for the whole run. + + Enable this for anything that outruns kubelet rotation — in practice the SPDK containers + and their proxies. Everything it follows is registered in + `ctx.shared["logs.streamed"]` so `logs.collect` does not overwrite a full-run stream + with the tail the kubelet happens to still have. + """ + + name = "logs.stream" + summary = "follow chosen container logs live, surviving kubelet rotation and restarts" + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + "image": DEFAULT_GRABBER_IMAGE, + # Pods to follow, matched by substring, and the containers within them. + "pods_matching": ["snode-spdk"], + "containers": ["spdk-container", "spdk-proxy-container"], + # Artifact name: "spdk-" and "spdk--proxy" for the SPDK pods. + "name_from": "snode-port", + "ttl_s": 6 * 3600, + "poll_s": 5, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._grabbers: dict[str, str] = {} + self._streams: list[dict] = [] + self._parts: dict[tuple, int] = {} + + # -- naming --------------------------------------------------------------------- + @staticmethod + def _snode_port(pod: str) -> str: + parts = pod.split("-") # snode-spdk-pod-- + return parts[3] if len(parts) > 3 and parts[3].isdigit() else pod + + def _artifact_name(self, pod: str, container: str) -> str: + if self.opt("name_from") == "snode-port": + suffix = "-proxy" if "proxy" in container else "" + return f"spdk-{self._snode_port(pod)}{suffix}" + return f"{pod}-{container}" + + # -- lifecycle ------------------------------------------------------------------ + def setup(self, ctx: RunContext) -> None: + self._pods = kube.list_pods(self.opt("namespace"), self.opt("pods_matching")) + if not self._pods: + ctx.log.warn(f"{self.name}: no pods matching {self.opt('pods_matching')}; " + "nothing to follow") + return + nodes = {p.node for p in self._pods if p.node} + self._grabbers = self._start_grabbers(ctx, sorted(nodes), int(self.opt("ttl_s"))) + # Published so logs.collect can reuse these instead of starting a second privileged + # pod per node. Its TTL already covers the whole run, and by the time collection runs + # the followers have stopped, so the pod is idle and free to be exec'd into again. + ctx.shared.setdefault("logs.grabbers", {}).update(self._grabbers) + + def start(self, ctx: RunContext) -> None: + streamed = ctx.shared.setdefault("logs.streamed", set()) + for p in getattr(self, "_pods", []): + grab = self._grabbers.get(p.node) + if not grab: + continue + for container in self.opt("containers"): + if container not in p.containers: + continue + self._attach(ctx, grab, p, container) + streamed.add((p.name, container)) + if self._streams: + ctx.log.info(f"{self.name}: following {len(self._streams)} container log(s) " + "for the whole run") + + def _attach(self, ctx: RunContext, grabber: str, pod: kube.Pod, container: str) -> None: + base = self._artifact_name(pod.name, container) + key = (pod.name, container) + part = self._parts.get(key, 0) + # A re-attach goes to a new file rather than appending: it re-reads whatever is + # still retained, which would duplicate a stretch of the previous part in the + # middle of the file. Separate parts stay internally ordered, which is what makes + # them readable. + name = f"{base}.txt" if part == 0 else f"{base}.part{part}.txt" + path = ctx.path(name) + try: + fh = open(path, "ab") # noqa: SIM115 + proc = subprocess.Popen( # noqa: S603 + ["kubectl", "-n", self.opt("namespace"), "exec", grabber, "--", "sh", "-c", + stream_script(pod.namespace, pod.name, container, int(self.opt("poll_s")))], + stdout=fh, stderr=subprocess.DEVNULL) + except Exception as e: # noqa: BLE001 + ctx.log.warn(f"{self.name}: cannot follow {pod.name}/{container}: {e}") + return + self._parts[key] = part + 1 + self._streams.append({"proc": proc, "fh": fh, "path": path, "grabber": grabber, + "pod": pod, "container": container}) + + def tick(self, ctx: RunContext) -> None: + """Re-attach followers that died. Cheap enough to run on every tick.""" + for st in list(self._streams): + if st["proc"].poll() is None: + continue + self._streams.remove(st) + with contextlib.suppress(Exception): + st["fh"].close() + size = os.path.getsize(st["path"]) if os.path.exists(st["path"]) else 0 + ctx.log.warn(f"{self.name}: follower for {st['pod'].name}/{st['container']} " + f"ended early (rc={st['proc'].returncode}, {size / 1048576:.1f} MiB); " + "re-attaching into a new part") + ctx.timeline.record("logs.stream.reattach", subject=st["pod"].name, + container=st["container"]) + self._attach(ctx, st["grabber"], st["pod"], st["container"]) + + def stop(self, ctx: RunContext) -> None: + total = 0 + for st in self._streams: + proc = st["proc"] + try: + proc.terminate() + proc.wait(timeout=15) + except Exception: # noqa: BLE001 + with contextlib.suppress(Exception): + proc.kill() + with contextlib.suppress(Exception): + st["fh"].flush() + st["fh"].close() + if os.path.exists(st["path"]): + total += os.path.getsize(st["path"]) + if self._streams: + ctx.log.info(f"{self.name}: stopped {len(self._streams)} follower(s); " + f"{total / 1048576:.1f} MiB captured live") + self._streams = [] + + def teardown(self, ctx: RunContext) -> None: + self.stop(ctx) # idempotent; covers a run that failed before stop + for node in self._grabbers: + ctx.shared.get("logs.grabbers", {}).pop(node, None) + self._delete_grabbers(ctx, sorted(self._grabbers.values())) + self._grabbers = {} + + +@component +class LogCollect(_GrabberBase): + """Grab container logs from the hosts at the end of the run. + + Correct for anything that fits inside what the kubelet retains, which is everything + except the SPDK containers on a long run. Skips whatever `logs.stream` followed. + """ + + name = "logs.collect" + summary = "grab container logs from each host's /var/log/pods after the run" + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + "control_plane_namespace": "simplyblock", + "image": DEFAULT_GRABBER_IMAGE, + #: [{pods: [substr], containers: [name]|"all", name_from: ..., namespace: ...}] + "targets": [ + {"pods": ["snode-spdk"], "containers": ["spdk-container", "spdk-proxy-container"], + "name_from": "snode-port"}, + {"pods": ["operator", "webappapi"], "containers": "all", + "namespace": "simplyblock", "name_from": "pod-key"}, + # One artifact per container, not per pod. The tasks pod runs seventeen + # independent runners, so merging them produces a 50 MiB file that is not in + # time order — which makes its time span meaningless and stops a pattern being + # scoped to the one runner you care about. The container names are already + # unique and descriptive, so they are the artifact names. + {"pods": ["tasks"], "containers": "all", + "namespace": "simplyblock", "name_from": "container"}, + # The CSI driver is where the connects, the path reconcilers and + # NodeStage/NodePublish actually happen, so it is the log that says what the + # *host side* did and why. Kept per node rather than merged: the node plugin + # reconciles per host, so "which node" is the first question about anything + # it did, and a merged file loses it. + {"pods": ["simplyblock-csi-node"], "containers": ["csi-node"], + "namespace": "simplyblock", "name_from": "pod-node", "name": "csi-node"}, + # The node-side agent the control plane talks to on each host. It is what + # starts and probes the SPDK process, which puts it on the causal path of + # every "the node went offline" event — including the liveness check that + # concluded SPDK was dead because a Kubernetes API call blipped. + {"pods": ["simplyblock-storage-node-ds"], "containers": "all", + "namespace": "default", "name_from": "pod-node", "name": "snode-api"}, + {"pods": ["simplyblock-csi-controller"], "containers": "all", + "namespace": "simplyblock", "name_from": "container"}, + ], + "ttl_s": 1800, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._grabbers: dict[str, str] = {} + #: Only the ones this component created — the rest belong to whoever published them + #: and are that component's to remove. + self._own: dict[str, str] = {} + + def collect(self, ctx: RunContext) -> None: + streamed = ctx.shared.get("logs.streamed", set()) + plan: list[tuple[kube.Pod, str, str]] = [] + for target in self.opt("targets"): + ns = target.get("namespace", self.opt("namespace")) + for p in kube.list_pods(ns, target["pods"]): + wanted = (list(p.containers) if target.get("containers") == "all" + else [c for c in target["containers"] if c in p.containers]) + for c in wanted: + if (p.name, c) in streamed: + continue + plan.append((p, c, self._name_for(target, p, c))) + if not plan: + return + + nodes = {p.node for p, _c, _n in plan if p.node} + # logs.stream publishes its grabbers for exactly this. Its TTL already covers the + # whole run, and by collection time its followers have stopped, so the pod is idle and + # free to be exec'd into — a second privileged pod per node doing the same job is what + # the distinct names were only ever a backstop against. + borrowed = {n: g for n, g in ctx.shared.get("logs.grabbers", {}).items() if n in nodes} + missing = sorted(nodes - set(borrowed)) + # Recorded separately from `_grabbers`, because teardown may only delete these: a + # borrowed pod belongs to the component that published it. + self._own = self._start_grabbers(ctx, missing, int(self.opt("ttl_s"))) if missing else {} + self._grabbers = {**borrowed, **self._own} + + grouped: dict[str, list[tuple[kube.Pod, str]]] = {} + for p, c, artifact in plan: + grouped.setdefault(artifact, []).append((p, c)) + + for artifact, items in sorted(grouped.items()): + path = ctx.path(f"{artifact}.txt") + with open(path, "wb") as fh: + for p, c in items: + grab = self._grabbers.get(p.node) + if not grab: + continue + if len(items) > 1: # several containers share one artifact; header them + fh.write(f"==================== {p.name} / {c} " + f"({kube.short(p.node)}) ====================\n".encode()) + data = kube.run_bytes( + ["-n", self.opt("namespace"), "exec", grab, "--", "sh", "-c", + dump_script(p.namespace, p.name, c)]) + fh.write(data) + if not data: + # The dump script swallows read errors, so an empty grab is + # otherwise indistinguishable from "this container logged nothing". + ctx.log.warn(f"{self.name}: empty grab for {p.name}/{c}") + ctx.log.info(f"{self.name}: {artifact}.txt " + f"({os.path.getsize(path) / 1048576:.1f} MiB)") + + @staticmethod + def _name_for(target: dict, pod: kube.Pod, container: str) -> str: + mode = str(target.get("name_from", "pod-container")) + if mode == "snode-port": + parts = pod.name.split("-") + port = parts[3] if len(parts) > 3 and parts[3].isdigit() else pod.name + return f"spdk-{port}{'-proxy' if 'proxy' in container else ''}" + if mode == "pod-key": + for key in target["pods"]: + if key in pod.name: + return str(key) + if mode == "container": + return container + if mode == "pod-node": + # "-": one artifact per node, which is how a DaemonSet's behaviour is + # actually reasoned about. `name` overrides the matched substring, because the + # generated pod names are long and carry nothing a reader wants. + key = target.get("name") or next( + (k for k in target["pods"] if k in pod.name), pod.name) + return f"{key}-{kube.short(pod.node)}" if pod.node else str(key) + return f"{pod.name}-{container}" + + def teardown(self, ctx: RunContext) -> None: + # Only what this component created. Deleting a grabber it merely borrowed would pull + # the pod out from under the component that owns it. + self._delete_grabbers(ctx, sorted(self._own.values())) + self._grabbers = {} + self._own = {} + + +@component +class Dmesg(Component): + """Kernel ring buffer from each storage worker. + + Rotation-limited in a different way from a container log: the ring buffer is a fixed + size, so a node that is logging heavily keeps only minutes of it — while a quiet one + keeps hours, which is the real hazard. A buffer spanning ten hours contains the previous + runs' damage, and counting that against this run is how a green build inherits its + predecessor's mess. + + That is why the timestamp format matters here more than it looks. `dmesg -T` renders + *local* time with no offset, so an event cannot be placed against a run window recorded + in UTC without assuming the two agree. `--time-format=iso` emits an offset, which makes + the comparison sound; it is preferred, with `-T` kept as a fallback for older util-linux. + """ + + name = "host.dmesg" + summary = "dmesg from each storage worker, ISO-timestamped so events can be placed in time" + + def defaults(self) -> dict[str, Any]: + return {"namespace": "default", "pods_matching": ["snode-spdk"], + "container": "spdk-container"} + + def collect(self, ctx: RunContext) -> None: + for p in kube.list_pods(self.opt("namespace"), self.opt("pods_matching")): + if not p.node: + continue + data = kube.run_bytes( + ["-n", p.namespace, "exec", p.name, "-c", self.opt("container"), "--", + "sh", "-c", "dmesg --time-format=iso 2>/dev/null || dmesg -T"], + timeout=120) + if not data: + ctx.log.warn(f"{self.name}: empty dmesg from {kube.short(p.node)}") + with open(ctx.path(f"dmesg-{kube.short(p.node)}.txt"), "wb") as fh: + fh.write(data) diff --git a/test/framework/sbtest/components/migration.py b/test/framework/sbtest/components/migration.py new file mode 100644 index 000000000..540a590eb --- /dev/null +++ b/test/framework/sbtest/components/migration.py @@ -0,0 +1,445 @@ +"""The volume-migration driver: the component that makes migrations happen. + +This is the first component that *is* the run rather than an observer of it, and that changes +two things about it. + +It is `required`, so a failure in its setup aborts instead of being recorded — a run whose +driver never started would otherwise report a clean pass for a test that never happened. + +And it drives the ANA sampler. Sampling only means something when the samples are attributed +to a migration, so the driver tells the sampler when each one begins, what phase it is in, and +when it ends. That is the one genuine dependency between components here, and it goes through +`ctx.shared` so the sampler stays optional: with sampling disabled the driver still migrates, +and the ANA detectors report themselves skipped rather than clean. + +Three things it reads from the backend rather than assuming: + +**Where the volume is now.** The cluster's own rebalancer moves volumes without any Kubernetes +object changing, so a source cached at setup is wrong by the tenth migration — and a wrong +source means picking a target the volume already lives on, which the operator rejects and +which reads as a product bug. + +**Which volumes move together.** A migration moves the whole NVMe subsystem, so what it +affects is the volume's *group*, not the volume. The control plane decides the packing; the +driver re-reads it before every migration, because a previous migration may have changed it. + +**What the migration actually did.** `status.sourceNodeUUID` is the operator's own resolved +answer and beats the driver's guess, so it is written back over it. + +What it deliberately does not do is judge anything. Whether a migration was acceptable is the +detectors' business; this records what happened and moves on. That separation is why the same +run can be re-judged later against a check that did not exist when it ran. +""" + +from __future__ import annotations + +import contextlib +import json +import random +import threading +import time +from datetime import datetime +from typing import Any + +from ..core import Component, Migration, RunContext, component, iso, now_utc +from . import kube +from .sbctl import Sbctl, SbctlError + +#: Phases the operator will not move on from. +TERMINAL = ("Completed", "Failed", "Aborted") + + +@component +class MigrationDriver(Component): + """Drive VolumeMigration CRs in a loop, one at a time, recording each outcome. + + One at a time on purpose. Concurrent migrations of the same subsystem are rejected by the + control plane, and concurrent migrations of *different* subsystems make every host-side + observation ambiguous about which one caused it. A test that cannot attribute what it saw + produces anecdotes. + """ + + name = "migration.driver" + summary = "create VolumeMigration CRs in a loop and record what each one did" + required = True + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + "api_group": "storage.simplyblock.io/v1alpha1", + # alternate | consumer | no-consumer | random. The policy is about whether the + # *target* also hosts a pod consuming this subsystem — the materially harder case, + # so which one a migration exercised is chosen rather than left to chance. + "target_policy": "alternate", + "gap_s": 30.0, # settle time between migrations + "timeout_s": 600.0, # per migration, before it is called a TIMEOUT + "poll_s": 3.0, + "max_migrations": 0, # 0 = for as long as the run lasts + # Volumes to migrate. Normally whatever the workload published; set explicitly to + # drive migrations against volumes this framework did not create. + "pvs": None, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._thread: threading.Thread | None = None + self._stop = threading.Event() + self._records: list[Migration] = [] + self._sb: Sbctl | None = None + self._nodes: list[str] = [] + self._node_host: dict[str, str] = {} + self._placement: dict[str, str] = {} # pv -> storage node uuid + self._volume_of: dict[str, str] = {} # pv -> lvol uuid + self._nqn_of: dict[str, str] = {} # pv -> subsystem nqn + self._groups: dict[str, list[str]] = {} + self._pvs: list[str] = [] + self._idx = 0 + + # ── setup ────────────────────────────────────────────────────────────────────── + + def setup(self, ctx: RunContext) -> None: + self._nodes, self._node_host = self._storage_nodes() + if len(self._nodes) < 2: + raise RuntimeError("need at least two online storage nodes to migrate between, " + f"found {len(self._nodes)}") + + self._pvs = list(self.opt("pvs") or ctx.shared.get("workload.pvs") or []) + if not self._pvs: + raise RuntimeError( + "no volumes to migrate: enable a workload component, or set this component's " + "`pvs` option to migrate volumes the run did not create") + + # Reuse the workload's sbctl handle when there is one, so the webappapi pod is + # resolved once per run rather than once per component. + self._sb = ctx.shared.get("workload.sbctl") or Sbctl() + self._volume_of = dict(ctx.shared.get("workload.volume_of", {})) + self._nqn_of = dict(ctx.shared.get("workload.nqn_of", {})) + self._groups = self._regroup() + self._placement = dict(ctx.shared.get("workload.placement", {})) + ctx.shared["migrations"] = self._records + ctx.log.info(f"{self.name}: {len(self._pvs)} volume(s) across {len(self._nodes)} " + f"storage node(s), policy={self.opt('target_policy')}") + + def _storage_nodes(self) -> tuple[list[str], dict[str, str]]: + """Online storage nodes as (uuids, uuid -> k8s hostname). + + Only online ones: migrating *to* an offline node is a rejected request, not a test. + """ + cp = kube.run(["get", "storagenodes", "-A", "-o", "json"], check=False) + try: + items = json.loads(cp.stdout).get("items", []) if cp.stdout else [] + except json.JSONDecodeError: + items = [] + uuids, hosts = [], {} + for it in items: + st = it.get("status", {}) + uuid = st.get("uuid") or "" + if not uuid or st.get("status") != "online": + continue + uuids.append(uuid) + hosts[uuid] = it.get("spec", {}).get("workerNode", "") or st.get("workerNode", "") + return uuids, hosts + + # ── the loop ─────────────────────────────────────────────────────────────────── + + def start(self, ctx: RunContext) -> None: + def loop() -> None: + gap = float(self.opt("gap_s")) + cap = int(self.opt("max_migrations")) + while not self._stop.is_set() and not ctx.stopping.is_set(): + if cap and self._idx >= cap: + ctx.log.info(f"{self.name}: reached max_migrations={cap}") + return + self._idx += 1 + try: + self._one(ctx, self._idx) + except Exception as e: # noqa: BLE001 + # A migration that blows up is a data point, not the end of the run: the + # next one may behave differently, and the detectors want the whole set. + ctx.log.error(f"{self.name}: migration {self._idx} errored: {e}") + if self._stop.wait(gap): + return + + self._thread = threading.Thread(target=loop, name="migration-driver", daemon=True) + self._thread.start() + + def _one(self, ctx: RunContext, idx: int) -> None: + pv = self._pvs[(idx - 1) % len(self._pvs)] + self._reread_subsystem(ctx, pv) + group = self._group_of(pv) + self._refresh_placement(group) + source = self._placement.get(pv, "") + target, policy, consumers = self._pick_target(ctx, group, idx, source) + + name = f"{ctx.run_id}-mig-{idx}" + rec = Migration(name=name, start=now_utc(), source=source, target=target, pv=pv, + pod=ctx.shared.get("workload.pod_of", {}).get(pv, ""), + members=list(group)) + self._records.append(rec) + + sampler = ctx.shared.get("ana.sampler") + if sampler is not None: + sampler.begin(name) + + ctx.timeline.record("migration.start", subject=name, pv=pv, source=source, + target=target, policy=policy, members=len(group), + nqn=self._nqn_of.get(pv, "")) + ctx.log.event( + f"MIGRATION START {name} pv={pv} members={len(group)} " + f"source={source[:8] or '?'}({self._node_host.get(source, '?')}) " + f"target={target[:8]}({self._node_host.get(target, '?')}) policy={policy} " + + (f"target hosts consumer(s): {', '.join(consumers)}" if consumers + else "target hosts no consumer")) + # A group whose members are not all on the source is inconsistent *before* the + # migration starts, and every later placement check would be judging that instead. + off = [p for p in group if self._placement.get(p, source) != source] + if off: + ctx.log.warn(f"{self.name}: {name}: subsystem members are not all on the source " + f"node: {', '.join(sorted(off))}") + + try: + kube.run(["apply", "-f", "-"], stdin=self._manifest(name, pv, target)) + except kube.KubectlError as e: + rec.phase, rec.error, rec.end = "Failed", f"create: {e}", now_utc() + ctx.log.error(f"{self.name}: cannot create {name}: {e}") + if sampler is not None: + sampler.end(ctx, name) + return + + self._await_terminal(ctx, rec, name, sampler) + + if sampler is not None: + sampler.end(ctx, name) + # Where the group ended up — for the next target pick, and for the placement checks. + self._refresh_placement(group) + ctx.timeline.record("migration.stop", subject=name, phase=rec.phase, error=rec.error, + landed=self._placement.get(pv, "")) + secs = f"{(rec.end - rec.start).total_seconds():.0f}s" if rec.end else "?" + ctx.log.event(f"MIGRATION STOP {name} phase={rec.phase} duration={secs}" + + (f" error={rec.error!r}" if rec.error else "")) + + def _await_terminal(self, ctx: RunContext, rec: Migration, name: str, + sampler: Any) -> None: + deadline = time.time() + float(self.opt("timeout_s")) + poll = float(self.opt("poll_s")) + seen = "" + while time.time() < deadline and not self._stop.is_set(): + cp = kube.run(["-n", self.opt("namespace"), "get", "volumemigration", name, + "-o", "json"], check=False, timeout=30) + if cp.returncode == 0 and cp.stdout: + try: + st = json.loads(cp.stdout).get("status", {}) + except json.JSONDecodeError: + st = {} + # The operator resolved the source from the control plane; ours came from a + # listing that may already be stale, so its answer wins. + if st.get("sourceNodeUUID"): + rec.source = str(st["sourceNodeUUID"]) + self._placement[rec.pv] = rec.source + phase = str(st.get("phase", "")) + if phase and phase != seen: + seen = phase + # The sampler stamps this on every sample it takes, which is what makes an + # ANA transition readable as "during cutover" rather than "at 09:31:02". + if sampler is not None and hasattr(sampler, "set_phase"): + sampler.set_phase(phase) + ctx.timeline.record("migration.phase", subject=name, phase=phase) + ctx.log.info(f"{self.name}: {name} -> {phase}") + if phase in TERMINAL: + rec.phase = phase + rec.error = str(st.get("errorMessage", "") or "") + rec.end = now_utc() + return + self._stop.wait(poll) + # Distinct from Failed on purpose: a migration the control plane rejected and one that + # never finished are different defects, and only one of them has an error to read. + rec.phase = "TIMEOUT" + rec.end = now_utc() + + # ── target selection ─────────────────────────────────────────────────────────── + + def _pick_target(self, ctx: RunContext, group: list[str], idx: int, + source: str) -> tuple[str, str, list[str]]: + """Pick the node to migrate to, honouring the policy. + + `alternate` starts with `consumer` on the first migration, so if a run is cut short the + harder case is the one that got exercised. + + When the policy cannot be honoured — no candidate matches — it falls back to any other + node and records that it did, rather than skipping the migration. A migration that ran + under the other condition is still evidence; one that did not run is not. + """ + candidates = [n for n in self._nodes if n != source] or list(self._nodes) + policy = str(self.opt("target_policy")) + if policy == "alternate": + policy = "consumer" if idx % 2 == 1 else "no-consumer" + + consuming = self._consumer_nodes(ctx, group) # k8s host -> consuming pods + want = None if policy == "random" else (policy == "consumer") + + pool = candidates + if want is not None: + matching = [n for n in candidates + if (self._node_host.get(n, "") in consuming) == want] + if matching: + pool = matching + else: + ctx.log.warn( + f"{self.name}: target policy {policy!r} cannot be honoured: no candidate " + f"node {'hosts' if want else 'is free of'} a consumer of this subsystem " + f"(consumers on {', '.join(sorted(consuming)) or 'none'}); picking any " + "other node") + policy = f"{policy}(unmet)" + target = random.choice(pool) # noqa: S311 (test placement, not cryptography) + return target, policy, sorted(consuming.get(self._node_host.get(target, ""), [])) + + def _consumer_nodes(self, ctx: RunContext, pvs: list[str]) -> dict[str, list[str]]: + """k8s node -> pods on it consuming any volume in this subsystem. + + The whole group, not just the volume being named: every pod holding *any* namespace of + the subsystem has its paths moved, so each one is a consumer for this purpose. + """ + pod_of = ctx.shared.get("workload.pod_of", {}) + node_of = ctx.shared.get("workload.node_of", {}) + out: dict[str, list[str]] = {} + for pv in pvs: + pod = pod_of.get(pv) + node = node_of.get(pod) if pod else None + if pod and node: + out.setdefault(node, []).append(pod) + return out + + # ── backend state ────────────────────────────────────────────────────────────── + + def _refresh_placement(self, pvs: list[str]) -> None: + """Re-read where these volumes are, from one `sbctl volume list`. + + One listing for the whole group: its members are compared against each other, and + per-volume lookups taken seconds apart could straddle a move and make a consistent + subsystem look split. + """ + if not self._sb: + return + lvols = {pv: self._volume_of[pv] for pv in pvs if pv in self._volume_of} + if not lvols: + return + # A failure here leaves the last known placement in place: a stale source beats no + # source, because "unknown" would make every target pick arbitrary. + with contextlib.suppress(SbctlError): + self._placement.update(self._sb.nodes_of(lvols)) + + def _reread_subsystem(self, ctx: RunContext, pv: str) -> None: + """Re-read this volume's subsystem before migrating it. + + Not cached from setup, because a previous migration can have changed the packing. The + group is what a migration moves, so a stale group means sampling the wrong nodes and + verifying the wrong volumes. + """ + if not self._sb: + return + lvol = self._volume_of.get(pv, "") + if not lvol: + return + nqn, _ = self._sb.subsystem_of(lvol) + if nqn and nqn != self._nqn_of.get(pv): + ctx.log.info(f"{self.name}: {pv} changed subsystem: " + f"{self._nqn_of.get(pv, '?')} -> {nqn}; regrouping") + self._nqn_of[pv] = nqn + self._groups = self._regroup() + + def _regroup(self) -> dict[str, list[str]]: + groups: dict[str, list[str]] = {} + for pv, nqn in self._nqn_of.items(): + groups.setdefault(nqn, []).append(pv) + return {n: sorted(m) for n, m in groups.items()} + + def _group_of(self, pv: str) -> list[str]: + """Everything a migration of `pv` moves: the volumes sharing its subsystem.""" + nqn = self._nqn_of.get(pv, "") + return list(self._groups.get(nqn, [pv])) if nqn else [pv] + + def _manifest(self, name: str, pv: str, target: str) -> str: + return json.dumps({ + "apiVersion": self.opt("api_group"), + "kind": "VolumeMigration", + "metadata": {"name": name, "namespace": self.opt("namespace"), + "labels": {"sbtest-run": "true", "sbtest": name}}, + "spec": {"pvName": pv, "targetNodeUUID": target}, + }) + + # ── lifecycle tail ───────────────────────────────────────────────────────────── + + def stop(self, ctx: RunContext) -> None: + self._stop.set() + if self._thread: + # Long enough for a migration already in flight to reach its own timeout: cutting + # it short would record a TIMEOUT the product never caused. + self._thread.join(timeout=float(self.opt("timeout_s")) + 30) + self._thread = None + done = [r for r in self._records if r.end] + ctx.log.info(f"{self.name}: {len(done)}/{len(self._records)} migration(s) reached a " + "terminal state") + + def collect(self, ctx: RunContext) -> None: + """Write the migration timeline where the analyser can read it back.""" + ctx.save_json("migrations.json", [{ + "name": r.name, + "start": iso(r.start), + "end": iso(r.end) if r.end else None, + "phase": r.phase, + "source": r.source, + "target": r.target, + "pv": r.pv, + "pod": r.pod, + "group_pvs": r.members, + "nqn": self._nqn_of.get(r.pv, ""), + "landed": self._placement.get(r.pv, ""), + "error": r.error, + } for r in self._records]) + by_phase: dict[str, int] = {} + for r in self._records: + by_phase[r.phase or "?"] = by_phase.get(r.phase or "?", 0) + 1 + ctx.log.info(f"{self.name}: " + + ", ".join(f"{k}={v}" for k, v in sorted(by_phase.items(), + key=lambda kv: -kv[1]))) + + def teardown(self, ctx: RunContext) -> None: + """Remove the CRs this run created, unless asked to keep them. + + By label rather than by name, so a CR whose creation succeeded but whose name never + made it into the records still goes. + """ + if ctx.shared.get("keep"): + ctx.log.info(f"{self.name}: keep set; leaving {len(self._records)} CR(s) in place") + return + kube.run(["-n", self.opt("namespace"), "delete", "volumemigration", + "-l", "sbtest-run=true", "--ignore-not-found", "--wait=false"], + check=False) + + +def migrations_from_file(path: str) -> list[Migration]: + """Read back what the driver wrote. Used by the archive adapter.""" + with open(path) as fh: + raw = json.load(fh) + + def _dt(v: object) -> datetime | None: + if not v: + return None + try: + return datetime.fromisoformat(str(v).replace("Z", "+00:00")) + except ValueError: + return None + + out = [] + for m in raw: + start = _dt(m.get("start")) + if not start: + continue + out.append(Migration( + name=m.get("name", ""), start=start, end=_dt(m.get("end")), + phase=m.get("phase", ""), source=m.get("source", ""), + target=m.get("target", ""), pv=m.get("pv", ""), pod=m.get("pod", ""), + members=[x for x in (m.get("group_pvs") or []) if x], + error=m.get("error", "") or "")) + out.sort(key=lambda x: x.start) + return out diff --git a/test/framework/sbtest/components/nvme.py b/test/framework/sbtest/components/nvme.py new file mode 100644 index 000000000..b41cc1ce9 --- /dev/null +++ b/test/framework/sbtest/components/nvme.py @@ -0,0 +1,247 @@ +"""Host NVMe observation: periodic ANA sampling and an end-of-run fabric snapshot. + +Both read the host's sysfs through a pod that already has it — the CSI node plugin — rather +than starting anything, because they need to run on every consuming node and the plugin is +already there on all of them. +""" + +from __future__ import annotations + +import threading +from typing import Any + +from ..core import AnaSample, Component, NvmeController, RunContext, component, now_utc +from . import kube + +#: Read every lvol controller's state, its address and its per-namespace ANA states. +#: One line per controller: name|state|address|ctrl_loss_tmo|nsid=ana,nsid=ana,... +_SNAPSHOT_SH = r''' +for c in /sys/class/nvme/nvme*; do + [ -f "$c/subsysnqn" ] || continue + nqn=$(cat "$c/subsysnqn" 2>/dev/null) + case "$nqn" in *lvol:*) ;; *) continue;; esac + ns="" + for n in "$c"/nvme*n*; do + [ -d "$n" ] || continue + id=$(cat "$n/nsid" 2>/dev/null) || continue + a=$(cat "$n/ana_state" 2>/dev/null) + ns="$ns$id=$a," + done + printf '%s|%s|%s|%s|%s|%s\n' \ + "$(basename "$c")" "$(cat "$c/state" 2>/dev/null)" \ + "$(cat "$c/address" 2>/dev/null | tr -d '\n')" \ + "$(cat "$c/ctrl_loss_tmo" 2>/dev/null)" "$nqn" "$ns" +done +''' + + +def _parse_snapshot(node: str, out: str) -> list[NvmeController]: + ctrls = [] + for line in out.splitlines(): + parts = line.split("|") + if len(parts) < 6: + continue + name, state, address, clt, nqn, ns = parts[:6] + traddr = trsvcid = "" + for kv in address.split(","): + if kv.startswith("traddr="): + traddr = kv[7:] + elif kv.startswith("trsvcid="): + trsvcid = kv[8:] + namespaces = {} + for item in ns.split(","): + if "=" in item: + k, _, v = item.partition("=") + try: + namespaces[int(k)] = v + except ValueError: + continue + try: + loss = int(clt) if clt.strip() else None + except ValueError: + loss = None + ctrls.append(NvmeController( + node=node, name=name, nqn=nqn, address=f"{traddr}:{trsvcid}", + state=state, namespaces=namespaces, ctrl_loss_tmo=loss)) + return ctrls + + +class _CsiNodeBase(Component): + """Shared lookup of the CSI node plugin, the window onto each host's sysfs. + + A Component for the same reason as _GrabberBase: it uses `opt` and `name`. + """ + + def _csi_node_pods(self, ctx: RunContext) -> dict[str, str]: + """node -> CSI node-plugin pod, the window onto each host's sysfs.""" + out = {} + for p in kube.list_pods(self.opt("csi_namespace"), [self.opt("csi_pod_prefix")]): + if p.node: + out[p.node] = p.name + if not out: + ctx.log.warn(f"{self.name}: no CSI node pods matching " + f"{self.opt('csi_pod_prefix')!r} in {self.opt('csi_namespace')}; " + "host NVMe state cannot be read") + return out + + +@component +class FabricSnapshot(_CsiNodeBase): + """Record every lvol NVMe controller on every node, at setup and at the end. + + Two snapshots on purpose. The one at setup answers "did the last run leave a mess?" — + which is a real question, because a leaked controller breaks the *next* run rather than + the one that created it. The one at the end answers "did this run leave a mess?". + """ + + name = "nvme.snapshot" + summary = "snapshot host NVMe controllers before and after the run (leak detection)" + + def defaults(self) -> dict[str, Any]: + return {"csi_namespace": "simplyblock", "csi_pod_prefix": "simplyblock-csi-node", + "container": "csi-node", "at_setup": True, "at_collect": True} + + def _snapshot(self, ctx: RunContext, label: str) -> list[NvmeController]: + ctrls: list[NvmeController] = [] + for node, pod in sorted(self._csi_node_pods(ctx).items()): + out = kube.exec_sh(self.opt("csi_namespace"), pod, _SNAPSHOT_SH, + container=self.opt("container"), timeout=120) + ctrls.extend(_parse_snapshot(kube.short(node), out)) + ctx.save_json(f"nvme-controllers-{label}.json", + [{"node": c.node, "name": c.name, "nqn": c.nqn, "address": c.address, + "state": c.state, "namespaces": {str(k): v for k, v in c.namespaces.items()}, + "ctrl_loss_tmo": c.ctrl_loss_tmo} for c in ctrls]) + stuck = [c for c in ctrls if c.state == "connecting"] + empty = [c for c in ctrls if c.state == "live" and c.serves_nothing] + ctx.log.info(f"{self.name} ({label}): {len(ctrls)} lvol controller(s), " + f"{len(stuck)} connecting, {len(empty)} live-with-no-namespace") + return ctrls + + def setup(self, ctx: RunContext) -> None: + if not self.opt("at_setup"): + return + pre = self._snapshot(ctx, "pre") + ctx.shared["nvme.controllers.pre"] = pre + + def collect(self, ctx: RunContext) -> None: + if not self.opt("at_collect"): + return + post = self._snapshot(ctx, "post") + ctx.shared["nvme.controllers"] = post + # The detectors read the canonical name; keep it pointing at the post-run state. + ctx.save_json("nvme-controllers.json", + [{"node": c.node, "name": c.name, "nqn": c.nqn, "address": c.address, + "state": c.state, "namespaces": {str(k): v for k, v in c.namespaces.items()}, + "ctrl_loss_tmo": c.ctrl_loss_tmo} for c in post]) + + +@component +class AnaSampler(_CsiNodeBase): + """Sample per-namespace ANA state on every consuming node, on an interval. + + The evidence behind every ANA detector, and the reason the sampling interval is a + first-class option: the freeze *count* is robust to it, but a freeze shorter than one + interval can be missed entirely, so a suite hunting short pauses should lower it. + + Samples are written per migration into ana/.csv in the same layout the + archive reader expects, so a run's ANA evidence is replayable. + """ + + name = "ana.sample" + summary = "sample host ANA state per namespace on an interval, per migration" + + def defaults(self) -> dict[str, Any]: + return {"csi_namespace": "simplyblock", "csi_pod_prefix": "simplyblock-csi-node", + "container": "csi-node", "interval_s": 2.0, "nodes": None} + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._thread: threading.Thread | None = None + self._stop = threading.Event() + self._current: str | None = None + self._phase = "" + self._samples: dict[str, list[AnaSample]] = {} + self._pods: dict[str, str] = {} + + def setup(self, ctx: RunContext) -> None: + self._pods = self._csi_node_pods(ctx) + ctx.shared["ana.samples"] = self._samples + # Published so a driver can attribute samples to the migration in flight. Samples + # that belong to no migration are worth nothing — freeze_windows is per migration — + # so without a driver calling begin/end this component collects nothing, and the ANA + # detectors correctly report themselves skipped rather than clean. + ctx.shared["ana.sampler"] = self + + # -- the scenario drives these two ---------------------------------------------- + def begin(self, migration: str) -> None: + """Start attributing samples to `migration`. Called by whatever drives migrations.""" + self._current = migration + self._phase = "" + self._samples.setdefault(migration, []) + + def set_phase(self, phase: str) -> None: + """Stamp subsequent samples with the migration's current phase. + + This is what turns a wall-clock ANA transition into a readable one: "all paths went + inaccessible during Cutover" is a finding, "all paths went inaccessible at 09:31:02" + is a timestamp. Optional — the driver calls it if it has a phase to report, and a + sampler that is never told stamps the empty string. + """ + self._phase = phase + + def end(self, ctx: RunContext, migration: str) -> str | None: + """Stop attributing and write the CSV for `migration`.""" + self._current = None + self._phase = "" + samples = self._samples.get(migration) or [] + if not samples: + return None + path = ctx.path(f"ana/{migration}.csv") + with open(path, "w") as fh: + fh.write("ts,node,phase,address,role,ctrl_state,nsid,ana_state\n") + for s in sorted(samples, key=lambda x: (x.ts, x.node, x.address)): + stamp = s.ts.strftime("%Y-%m-%dT%H:%M:%SZ") + if not s.ana: + # A controller with no namespace at all is itself the finding, so it + # gets a row rather than being skipped. + fh.write(f"{stamp},{s.node},{s.phase},{s.address},{s.role},{s.state},,\n") + for nsid, ana in sorted(s.ana.items()): + fh.write(f"{stamp},{s.node},{s.phase},{s.address},{s.role}," + f"{s.state},{nsid},{ana}\n") + return path + + def start(self, ctx: RunContext) -> None: + if not self._pods: + return + interval = float(self.opt("interval_s")) + if interval <= 0: + ctx.log.info(f"{self.name}: disabled (interval_s <= 0)") + return + + def loop() -> None: + while not self._stop.is_set() and not ctx.stopping.is_set(): + mig = self._current + if mig: + for node, pod in self._pods.items(): + try: + out = kube.exec_sh(self.opt("csi_namespace"), pod, _SNAPSHOT_SH, + container=self.opt("container"), timeout=30) + except Exception: # noqa: BLE001 + continue + ts = now_utc() + for c in _parse_snapshot(kube.short(node), out): + self._samples.setdefault(mig, []).append(AnaSample( + ts=ts, node=c.node, address=c.address, state=c.state, + ana=dict(c.namespaces), phase=self._phase)) + self._stop.wait(interval) + + self._thread = threading.Thread(target=loop, name="ana-sampler", daemon=True) + self._thread.start() + ctx.log.info(f"{self.name}: sampling every {interval}s on " + f"{len(self._pods)} node(s)") + + def stop(self, ctx: RunContext) -> None: + self._stop.set() + if self._thread: + self._thread.join(timeout=15) + self._thread = None diff --git a/test/framework/sbtest/components/sbctl.py b/test/framework/sbtest/components/sbctl.py new file mode 100644 index 000000000..835ac6d88 --- /dev/null +++ b/test/framework/sbtest/components/sbctl.py @@ -0,0 +1,194 @@ +"""A thin `sbctl` client — the control plane's own view of the cluster. + +Kubernetes objects say what was *asked for*; sbctl says what the storage cluster actually +did. The two disagree often enough that mixing them up is its own class of bug: a +StorageClass can carry a `cluster_id` from an installation that no longer exists, and a +volume's placement drifts under the cluster's own rebalancer without any Kubernetes object +changing. So placement, subsystem grouping and the live cluster id all come from here. + +Not a Component: it holds no lifecycle and produces no artifacts. Components construct one +and ask it questions. +""" + +from __future__ import annotations + +import json +from typing import Any + +from . import kube + + +class SbctlError(RuntimeError): + pass + + +class Sbctl: + """Runs `sbctl ... --json` inside a webappapi pod. + + The pod is resolved once and cached. If it is replaced mid-run the next call re-resolves + rather than failing the run: a restarted webappapi is normal, and losing placement + resolution would turn every later verification into "skipped" for no good reason. + """ + + def __init__(self, namespace: str = "simplyblock", pod_match: str = "webappapi") -> None: + self.namespace = namespace + self.pod_match = pod_match + self._pod = "" + self._cluster_uuid = "" + + # ── plumbing ─────────────────────────────────────────────────────────────────── + + def pod(self, refresh: bool = False) -> str: + if self._pod and not refresh: + return self._pod + for p in kube.list_pods(self.namespace, [self.pod_match]): + if p.phase == "Running": + self._pod = p.name + return p.name + raise SbctlError(f"no running '*{self.pod_match}*' pod in namespace {self.namespace}") + + def _json(self, *argv: str, timeout: int = 60) -> Any: + for attempt in (0, 1): + pod = self.pod(refresh=bool(attempt)) + cp = kube.run(["-n", self.namespace, "exec", pod, "--", "sbctl", *argv, "--json"], + check=False, timeout=timeout) + if cp.returncode == 0: + break + if attempt: + raise SbctlError(f"sbctl {' '.join(argv)} failed: " + f"{(cp.stderr or cp.stdout).strip()}") + try: + return json.loads(cp.stdout) + except json.JSONDecodeError as e: + raise SbctlError(f"sbctl {' '.join(argv)} returned unparseable output: {e}") from e + + # ── queries ──────────────────────────────────────────────────────────────────── + + def cluster_uuid(self) -> str: + """The live cluster's UUID. + + Authoritative after a reinstall, which is the whole reason it is read at all: a + StorageClass cloned from the pool's own SC can still carry a dead cluster id, and + every volume provisioned from it would target a cluster that no longer exists. + """ + if self._cluster_uuid: + return self._cluster_uuid + clusters = self._json("cluster", "list") + if not clusters: + raise SbctlError("sbctl cluster list returned no clusters") + active = [c for c in clusters if str(c.get("Status", "")).upper() == "ACTIVE"] + chosen = active or clusters + if len(chosen) != 1: + desc = ", ".join(f"{c.get('Name')}={c.get('UUID')}({c.get('Status')})" + for c in chosen) + raise SbctlError(f"expected exactly one active cluster, found " + f"{len(chosen)}: {desc}") + uuid = str(chosen[0].get("UUID") or "") + if not uuid: + raise SbctlError("the active cluster has no UUID in sbctl output") + self._cluster_uuid = uuid + return uuid + + def volume_list(self) -> list[dict]: + out = self._json("volume", "list") + return out if isinstance(out, list) else [] + + def volume_get(self, lvol: str) -> dict: + out = self._json("volume", "get", lvol) + return out if isinstance(out, dict) else {} + + def storage_node_list(self) -> list[dict]: + out = self._json("storage-node", "list") + return out if isinstance(out, list) else [] + + def snapshot_list(self) -> list[dict]: + out = self._json("snapshot", "list") + return out if isinstance(out, list) else [] + + # ── derived views ────────────────────────────────────────────────────────────── + + def host_map(self) -> tuple[dict[str, str], dict[str, str]]: + """(sbctl hostname -> node uuid, node uuid -> management IP). + + The translation exists because a volume's `Hostname` is the storage node's own name + (short host + RPC port, e.g. `vm04_4424`) while everything on the Kubernetes side + speaks node UUIDs. The management IP is the transport address the node's subsystems + listen on, which is the only way a sampled path can be attributed to a role. + """ + hosts: dict[str, str] = {} + ips: dict[str, str] = {} + for n in self.storage_node_list(): + host, uuid = str(n.get("Hostname") or ""), str(n.get("UUID") or "") + if host and uuid: + hosts[host] = uuid + if uuid and n.get("Management IP"): + ips[uuid] = str(n["Management IP"]) + if not hosts: + raise SbctlError("sbctl storage-node list reported no Hostname/UUID pairs") + return hosts, ips + + def subsystem_of(self, lvol: str) -> tuple[str, int]: + """(NQN, ns_id) of one volume. ('', 0) when unresolvable. + + Read per volume rather than derived from the StorageClass, because the control plane + decides how it packs namespaced volumes into subsystems. What a migration has to move + follows from the real grouping, not the requested one. + """ + if not lvol: + return "", 0 + try: + data = self.volume_get(lvol) + except SbctlError: + return "", 0 + nqn = str(data.get("nqn") or data.get("NQN") or "") + try: + ns_id = int(data.get("ns_id") or data.get("NS ID") or 0) + except (TypeError, ValueError): + ns_id = 0 + return nqn, ns_id + + def nodes_of(self, lvols: dict[str, str]) -> dict[str, str]: + """{key: node uuid} for `{key: lvol uuid}`, from a *single* volume listing. + + One listing for the whole set on purpose: the members of a shared subsystem are + compared against each other, and per-volume listings taken seconds apart could + straddle a move and make a consistent subsystem look split. + """ + try: + vols = self.volume_list() + hosts, _ = self.host_map() + except SbctlError: + return {} + by_lvol: dict[str, dict] = {} + for v in vols: + for k in (v.get("Id"), v.get("LVolUUID")): + if k: + by_lvol[str(k)] = v + + out: dict[str, str] = {} + missed: set[str] = set() + for key, lvol in lvols.items(): + vol = by_lvol.get(lvol) + if not vol: + continue + node = hosts.get(str(vol.get("Hostname") or "")) + if node: + out[key] = node + else: + missed.add(str(vol.get("Hostname") or "")) + # An unknown hostname is not proof a volume is unplaceable — a node may have joined + # since the map was built. Rebuild once before giving up, so a stale map cannot + # quietly turn every placement check into "skipped". + if missed: + try: + hosts, _ = self.host_map() + except SbctlError: + return out + for key, lvol in lvols.items(): + if key in out: + continue + vol = by_lvol.get(lvol) + node = hosts.get(str(vol.get("Hostname") or "")) if vol else "" + if node: + out[key] = node + return out diff --git a/test/framework/sbtest/components/workload.py b/test/framework/sbtest/components/workload.py new file mode 100644 index 000000000..81928fd0c --- /dev/null +++ b/test/framework/sbtest/components/workload.py @@ -0,0 +1,562 @@ +"""The fio workload: volumes with continuous, verified I/O on them. + +A migration test needs I/O in flight to be a test at all — a volume nobody is writing to +migrates cleanly whatever the code does, because nothing is there to lose. So this component +provisions volumes, drives fio against them for the length of the run, and collects what fio +saw. + +Two things here are less obvious than they look. + +**Verification is only enabled when it can be trusted.** fio's md5 verify races itself when +two in-flight I/Os touch the same block, and reports corruption that never happened. With one +job that is solvable (`--serialize_overlap`); across processes it is not, without +`io_submit_mode=offload`. So `numjobs > 1` turns verification *off* and says so loudly, +because a run that measures throughput is not a run that would have noticed data loss — and +the report must not imply otherwise. + +**Two StorageClasses, not one.** Batch migration only exists when several volumes share an +NVMe subsystem, and whether they do is the control plane's decision, not the test's. So the +namespaced volumes get their own class and the real grouping is *read back* from the backend +afterwards. If the volumes each ended up in their own subsystem, the batch half of the run is +vacuous — which is worth knowing at minute one rather than during the analysis. +""" + +from __future__ import annotations + +import json +import time +from datetime import UTC, datetime, timedelta +from typing import Any + +from ..core import Component, RunContext, component +from . import kube +from .sbctl import Sbctl, SbctlError + +FIO_IMAGE = "alpine:3.20" +PARAM_MAX_NS = "max_namespace_per_subsys" + + +@component +class FioWorkload(Component): + """Provision volumes, run verified fio on them, collect what fio saw. + + `required`, because a run whose workload never came up is not a passing run — it is no run + at all, and the detectors have nothing to judge. + """ + + name = "workload.fio" + summary = "provision volumes and drive continuous verified fio I/O against them" + required = True + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + # The operator's own pool SC, named simplyblock---. Cloned + # rather than hand-written so the run inherits whatever the pool really uses. + "source_storageclass": "simplyblock-default-simplyblock-cluster-pool1", + "pods": 0, # single-namespace volumes + "ns_pods": 6, # namespaced volumes (share subsystems -> batch migration) + "ns_per_subsys": 6, + "volume_size_gb": 10, + "file_size_gb": 1, + "fstype": "xfs", + "runtime_s": 3600, + "iodepth": 8, + "numjobs": 1, # >1 disables verification; see the module docstring + "verify_fatal": False, + "bs": "4k", + "rwmixread": 70, + "ioengine": "libaio", + "ready_timeout_s": 420, + "image": FIO_IMAGE, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._sb = Sbctl() + self._pods: list[str] = [] + self._pvcs: list[str] = [] + self._pvc_of: dict[str, str] = {} + self._kind_of: dict[str, str] = {} + self._pv_of: dict[str, str] = {} # pod -> pv + self._pod_of: dict[str, str] = {} # pv -> pod + self._volume_of: dict[str, str] = {} # pv -> lvol uuid + self._nqn_of: dict[str, str] = {} # pv -> subsystem nqn + self._nsid_of: dict[str, int] = {} + self._groups: dict[str, list[str]] = {} # nqn -> pvs, ns_id ordered + self._node_host: dict[str, str] = {} # storage node uuid -> k8s host + self._sc_single = "" + self._sc_ns = "" + + # ── setup: classes, volumes, pods, then read back what really happened ───────── + + def setup(self, ctx: RunContext) -> None: + self._node_host = self._storage_node_hosts() + self._ensure_storageclasses(ctx) + self._create(ctx) + self._wait_running(ctx) + self._resolve_pvs(ctx) + self._resolve_subsystems(ctx) + self._publish(ctx) + + def _storage_node_hosts(self) -> dict[str, str]: + cp = kube.run(["get", "storagenodes", "-A", "-o", "json"], check=False) + try: + items = json.loads(cp.stdout).get("items", []) if cp.stdout else [] + except json.JSONDecodeError: + return {} + out = {} + for it in items: + st = it.get("status", {}) + uuid = st.get("uuid") or "" + if uuid: + out[uuid] = it.get("spec", {}).get("workerNode", "") or st.get("workerNode", "") + return out + + def _ensure_storageclasses(self, ctx: RunContext) -> None: + """(Re)create the run's StorageClasses from the live pool SC. + + Always delete-then-create, so a class left behind by a previous run can never be + silently reused with different parameters. And `cluster_id` is forced to the live + cluster rather than copied: the source SC can still carry a dead one after a + reinstall, and every volume from it would target a cluster that no longer exists. + """ + src_name = self.opt("source_storageclass") + cp = kube.run(["get", "sc", src_name, "-o", "json"], check=False) + if cp.returncode != 0 or not cp.stdout: + # The name embeds the namespace, cluster and pool, so it differs per install. + # Naming the candidates turns a config error into a one-line fix instead of a + # trip to kubectl. + raise RuntimeError( + f"source StorageClass {src_name} not found. Either the name is wrong for this " + f"install — candidates: {', '.join(self._candidate_sources()) or 'none'} — or " + "the operator has not created it yet, which usually means the Pool is not " + "Active (its create is stuck retrying on the backend). Check: kubectl -n " + f"{self.opt('namespace')} get pool -o jsonpath='{{.items[*].status}}'") + src = json.loads(cp.stdout) + + cluster = self._sb.cluster_uuid() + base = dict(src.get("parameters", {})) + stale = base.get("cluster_id") + if stale and stale != cluster: + ctx.log.warn(f"{self.name}: source SC {src_name} carries stale cluster_id " + f"{stale}; overriding with the live cluster {cluster}") + base["cluster_id"] = cluster + base["csi.storage.k8s.io/fstype"] = str(self.opt("fstype")) + # A single-namespace class must say so explicitly rather than inherit whatever the + # source SC or the CSI default carries — otherwise the "single" half of the run could + # quietly be namespaced too, and the comparison would be with itself. + base.pop(PARAM_MAX_NS, None) + + self._sc_single = f"sbtest-{ctx.run_id}-single" + self._apply_sc(src, self._sc_single, dict(base, **{PARAM_MAX_NS: "1"})) + ctx.log.info(f"{self.name}: StorageClass {self._sc_single} " + f"(fstype={self.opt('fstype')}, {PARAM_MAX_NS}=1, cluster_id={cluster})") + if int(self.opt("ns_pods")) > 0: + self._sc_ns = f"sbtest-{ctx.run_id}-ns" + self._apply_sc(src, self._sc_ns, + dict(base, **{PARAM_MAX_NS: str(self.opt("ns_per_subsys"))})) + ctx.log.info(f"{self.name}: StorageClass {self._sc_ns} " + f"({PARAM_MAX_NS}={self.opt('ns_per_subsys')})") + + @staticmethod + def _candidate_sources() -> list[str]: + """simplyblock StorageClasses that are not themselves a test artifact. + + Anything carrying `sbtest-` is one of ours from an earlier run and would be a + confusing suggestion — cloning a clone inherits its overrides. + """ + cp = kube.run(["get", "sc", "-o", "json"], check=False) + if cp.returncode != 0 or not cp.stdout: + return [] + try: + items = json.loads(cp.stdout).get("items", []) + except json.JSONDecodeError: + return [] + return sorted( + it["metadata"]["name"] for it in items + if it.get("provisioner") == "csi.simplyblock.io" + and "sbtest-" not in it["metadata"]["name"]) + + @staticmethod + def _apply_sc(src: dict, name: str, params: dict) -> None: + kube.run(["delete", "sc", name, "--ignore-not-found"], check=False) + kube.run(["apply", "-f", "-"], stdin=json.dumps({ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": name, "labels": {"sbtest-run": "true"}}, + "provisioner": src.get("provisioner", "csi.simplyblock.io"), + "parameters": params, + "reclaimPolicy": src.get("reclaimPolicy", "Delete"), + "volumeBindingMode": src.get("volumeBindingMode", "WaitForFirstConsumer"), + "allowVolumeExpansion": src.get("allowVolumeExpansion", True), + })) + + def _create(self, ctx: RunContext) -> None: + script = self._fio_script(ctx) + affinity = self._affinity(ctx) + docs: list[dict] = [] + # One continuous index across both kinds, so a single name filter matches every pod + # this component created. + plan = ([("single", self._sc_single)] * int(self.opt("pods")) + + [("namespaced", self._sc_ns)] * int(self.opt("ns_pods"))) + if not plan: + raise RuntimeError("workload.fio: both `pods` and `ns_pods` are 0, so the run " + "would have no I/O and could not detect anything") + for i, (kind, sc) in enumerate(plan): + pvc, pod = f"{ctx.run_id}-pvc-{i}", f"{ctx.run_id}-fio-{i}" + self._pvcs.append(pvc) + self._pods.append(pod) + self._pvc_of[pod] = pvc + self._kind_of[pod] = kind + labels = {"sbtest": ctx.run_id, "sbtest-run": "true", "volume-kind": kind} + docs.append({ + "apiVersion": "v1", "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc, "labels": labels}, + "spec": {"accessModes": ["ReadWriteOnce"], "storageClassName": sc, + "resources": {"requests": { + "storage": f"{self.opt('volume_size_gb')}Gi"}}}, + }) + docs.append({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": pod, "labels": dict(labels, app="fio")}, + "spec": { + "restartPolicy": "Never", + "terminationGracePeriodSeconds": 5, + "affinity": affinity, + "containers": [{ + "name": "fio", "image": str(self.opt("image")), + "imagePullPolicy": "IfNotPresent", + "command": ["sh", "-c", script], + "volumeMounts": [{"name": "data", "mountPath": "/data"}, + {"name": "logs", "mountPath": "/logs"}], + "resources": {"requests": {"cpu": "250m", "memory": "256Mi"}}, + }], + "volumes": [ + {"name": "data", "persistentVolumeClaim": {"claimName": pvc}}, + # fio's own logs live on an emptyDir, never on the volume under test: + # collecting the evidence must not depend on the health of the thing + # the evidence is about. + {"name": "logs", "emptyDir": {}}, + ], + }, + }) + kube.run(["-n", self.opt("namespace"), "apply", "-f", "-"], + stdin="\n---\n".join(json.dumps(d) for d in docs)) + ctx.log.info(f"{self.name}: created {len(plan)} PVC(s) + fio pod(s): " + f"{self.opt('pods')} single-namespace + {self.opt('ns_pods')} namespaced") + + def _affinity(self, ctx: RunContext) -> dict: + """Pin fio pods to the storage worker nodes and off the control plane. + + Off the control plane in particular: a consuming pod there would exercise a path the + product does not otherwise have, and a fio pod competing with the API server produces + latency findings about the test rather than the system. + """ + workers = sorted({h for h in self._node_host.values() if h}) + exprs: list[dict] = [{"key": "node-role.kubernetes.io/control-plane", + "operator": "DoesNotExist"}] + if workers: + exprs.insert(0, {"key": "kubernetes.io/hostname", "operator": "In", + "values": workers}) + ctx.log.info(f"{self.name}: pods restricted to {', '.join(workers)}") + else: + ctx.log.warn(f"{self.name}: no storage worker hostnames known; only excluding " + "control-plane nodes") + return {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{"matchExpressions": exprs}]}}} + + def _fio_script(self, ctx: RunContext) -> str: + # fio always lays out a file-backed target before random I/O and cannot be told to + # skip it for a filesystem file. A large working set buys nothing here — the point is + # continuous I/O, not capacity — so keep the file small and the layout takes seconds. + file_gb = max(1, min(int(self.opt("file_size_gb")), + int(self.opt("volume_size_gb")) - 2)) + numjobs = int(self.opt("numjobs")) + args = [ + "fio", "--name=fiotest", "--filename=/data/fiotest", f"--size={file_gb}G", + f"--ioengine={self.opt('ioengine')}", "--direct=1", "--rw=randrw", + f"--rwmixread={self.opt('rwmixread')}", f"--bs={self.opt('bs')}", + f"--iodepth={self.opt('iodepth')}", f"--numjobs={numjobs}", + "--group_reporting", "--time_based", f"--runtime={self.opt('runtime_s')}", + "--continue_on_error=all", # record EIO, do not die on it + "--percentile_list=50:95:99:99.9", + "--write_iops_log=/logs/iops", "--write_lat_log=/logs/lat", + "--write_bw_log=/logs/bw", "--log_avg_msec=1000", + "--eta=always", "--eta-newline=30", + "--output=/logs/result.json", "--output-format=json", + ] + if numjobs == 1: + args += [ + # An md5 header per block, re-verified continuously during the run rather + # than only at the end — so a lost write surfaces while the volume is still + # migrating, close enough in time to attribute to a migration. + "--verify=md5", "--verify_backlog=4096", "--verify_backlog_batch=4096", + # Dump the mismatching block so its content can be read afterwards: a block + # holding its pre-write content is a lost write, which is a different defect + # from a block holding garbage. + "--verify_dump=1", + # Not fatal by default, so EVERY corrupted block gets reported instead of + # only the first. That turns the count into a measurement — with a known + # write rate it bounds how wide the window of lost writes was. Fatal stops at + # the first, making the count a lower bound of 1 that says nothing about size. + f"--verify_fatal={1 if self.opt('verify_fatal') else 0}", + ] + if int(self.opt("iodepth")) > 1: + args.append("--serialize_overlap=1") + else: + ctx.log.warn( + f"{self.name}: data-integrity verification DISABLED: numjobs={numjobs} (>1 " + "cannot serialize overlapping writes without io_submit_mode=offload, so " + "verify would report corruption that never happened). This run measures I/O " + "only, not integrity — use iodepth for concurrency instead") + return ( + "set -u\n" + 'echo "[pod] $(date -u +%FT%TZ) installing fio"\n' + "apk add --no-cache fio >/dev/null 2>&1 || " + '{ echo "[pod] apk add fio FAILED"; exit 90; }\n' + "mkdir -p /logs\n" + 'echo "[pod] $(date -u +%FT%TZ) starting fio"\n' + + " ".join(args) + "\n" + "rc=$?\n" + 'echo "$rc" > /logs/fio.rc\n' + 'echo "[pod] $(date -u +%FT%TZ) fio exited rc=$rc"\n' + # Stay alive after fio exits so the logs on the emptyDir can still be collected. + "sleep 100000\n" + ) + + def _wait_running(self, ctx: RunContext) -> None: + deadline = time.time() + float(self.opt("ready_timeout_s")) + ns = self.opt("namespace") + phases: dict[str, str] = {} + while time.time() < deadline and not ctx.stopping.is_set(): + cp = kube.run(["-n", ns, "get", "pods", "-l", f"sbtest={ctx.run_id}", + "-o", "json"], check=False) + phases = {} + if cp.returncode == 0 and cp.stdout: + for it in json.loads(cp.stdout).get("items", []): + phases[it["metadata"]["name"]] = it.get("status", {}).get("phase", "?") + bad = [f"{p}={ph}" for p, ph in phases.items() if ph in ("Failed", "Unknown")] + if bad: + raise RuntimeError(f"fio pod(s) failed during startup: {', '.join(bad)}") + running = [p for p, ph in phases.items() if ph == "Running"] + if len(running) == len(self._pods): + ctx.log.info(f"{self.name}: all {len(running)} fio pod(s) Running") + return + time.sleep(5) + # Name the pods that did not make it and what they are stuck at: "6 of 6 did not + # start" sends someone to kubectl for the one fact the message could have carried. + stuck = ", ".join(f"{p}={phases.get(p, 'absent')}" for p in self._pods + if phases.get(p) != "Running") + raise RuntimeError( + f"only {sum(1 for p in self._pods if phases.get(p) == 'Running')} of " + f"{len(self._pods)} fio pod(s) reached Running within " + f"{self.opt('ready_timeout_s')}s: {stuck}") + + def _resolve_pvs(self, ctx: RunContext) -> None: + ns = self.opt("namespace") + for pod in self._pods: + pvc = self._pvc_of[pod] + cp = kube.run(["-n", ns, "get", "pvc", pvc, "-o", "json"]) + pv = json.loads(cp.stdout).get("spec", {}).get("volumeName", "") + if not pv: + raise RuntimeError(f"PVC {pvc} has no bound PV") + cp = kube.run(["get", "pv", pv, "-o", "json"]) + handle = json.loads(cp.stdout).get("spec", {}).get("csi", {}).get( + "volumeHandle", "") + # "::" + parts = handle.split(":") + if len(parts) != 3 or not parts[2]: + raise RuntimeError(f"PV {pv} has an unexpected CSI volume handle {handle!r}") + self._pv_of[pod] = pv + self._pod_of[pv] = pod + self._volume_of[pv] = parts[2] + ctx.log.info(f"{self.name}: resolved PVC -> PV -> lvol:") + for pod in self._pods: + pv = self._pv_of[pod] + ctx.log.info(f" {pod} {self._pvc_of[pod]} -> {pv} " + f"(lvol {self._volume_of[pv]}, {self._kind_of[pod]})") + + def _resolve_subsystems(self, ctx: RunContext) -> None: + """Read each volume's real NVMe subsystem from the backend and group by it. + + Discovered, never assumed: the control plane decides the packing, and everything + downstream — which volumes a migration moves, which nodes must be sampled — follows + from the real grouping. + """ + for pv, lvol in self._volume_of.items(): + nqn, nsid = self._sb.subsystem_of(lvol) + if not nqn: + ctx.log.warn(f"{self.name}: cannot resolve the subsystem of {pv} " + f"(lvol {lvol}); it will migrate as a group of one") + continue + self._nqn_of[pv] = nqn + self._nsid_of[pv] = nsid + groups: dict[str, list[str]] = {} + for pv, nqn in self._nqn_of.items(): + groups.setdefault(nqn, []).append(pv) + self._groups = {n: sorted(m, key=lambda p: self._nsid_of.get(p, 0)) + for n, m in groups.items()} + + shared = {n: m for n, m in self._groups.items() if len(m) > 1} + ctx.log.info(f"{self.name}: {len(self._groups)} subsystem(s), {len(shared)} shared by " + "more than one volume") + for nqn, pvs in sorted(self._groups.items(), key=lambda kv: -len(kv[1])): + members = ", ".join(f"{self._pod_of.get(p, p)}(ns{self._nsid_of.get(p, '?')})" + for p in pvs) + ctx.log.info(f" {nqn} [{len(pvs)}] {members}") + + # Say this at minute one rather than leaving it for the analysis: if the namespaced + # volumes each got their own subsystem, no batch migration can be exercised at all, + # and the run is only testing the single-namespace path whatever its verdict says. + if int(self.opt("ns_pods")) >= 2 and not shared: + ctx.log.warn( + f"{self.name}: the {self.opt('ns_pods')} namespaced volume(s) each got their " + f"OWN subsystem ({PARAM_MAX_NS}={self.opt('ns_per_subsys')}), so no batch " + "migration can be exercised. Check that the CSI driver honours " + f"{PARAM_MAX_NS} and that the volumes landed on the same storage node — a " + "subsystem is shared per node") + + def _publish(self, ctx: RunContext) -> None: + """Hand the driver everything it needs to choose targets and attribute observations.""" + pvs = [self._pv_of[p] for p in self._pods if p in self._pv_of] + node_of: dict[str, str] = {} + cp = kube.run(["-n", self.opt("namespace"), "get", "pods", "-l", + f"sbtest={ctx.run_id}", "-o", "json"], check=False) + if cp.returncode == 0 and cp.stdout: + for it in json.loads(cp.stdout).get("items", []): + node_of[it["metadata"]["name"]] = it.get("spec", {}).get("nodeName", "") or "" + + ctx.shared.update({ + "workload.pvs": pvs, + "workload.pods": list(self._pods), + "workload.pod_of": dict(self._pod_of), + "workload.pv_of": dict(self._pv_of), + "workload.volume_of": dict(self._volume_of), + "workload.node_of": node_of, + "workload.nqn_of": dict(self._nqn_of), + "workload.subsystem_of": {pv: list(self._groups.get(self._nqn_of.get(pv, ""), + [pv])) + for pv in pvs}, + "workload.placement": self._placement(), + "workload.sbctl": self._sb, + }) + + def _placement(self) -> dict[str, str]: + try: + return self._sb.nodes_of(dict(self._volume_of)) + except SbctlError: + return {} + + # ── collection ───────────────────────────────────────────────────────────────── + + def collect(self, ctx: RunContext) -> None: + """Pull fio's own account out of each pod, into the layout the analyser reads. + + Read with `exec cat` rather than `kubectl cp`, which truncates large files without + reporting an error — a silently short result.json reads as a clean run. + """ + ns = self.opt("namespace") + migs = ctx.shared.get("migrations") or [] + for pod in self._pods: + d = ctx.dir(pod) + for remote, local in (("/logs/result.json", "result.json"), + ("/logs/fio.rc", "fio.rc")): + out = kube.exec_sh(ns, pod, f"cat {remote} 2>/dev/null", timeout=120) + if out: + with open(f"{d}/{local}", "w") as fh: + fh.write(out) + log = kube.run(["-n", ns, "logs", pod, "--tail=-1"], check=False, timeout=300) + if log.stdout: + with open(f"{d}/fio.log", "w") as fh: + fh.write(log.stdout) + self._write_timeseries(ctx, ns, pod, d, migs) + ctx.log.info(f"{self.name}: collected fio output for {len(self._pods)} pod(s)") + + def _write_timeseries(self, ctx: RunContext, ns: str, pod: str, d: str, + migs: list) -> None: + """Per-second IOPS from fio's iops log, with the migration in flight that second. + + The migration column is the point: correlating a throughput dip with the migration + that caused it is otherwise a manual join across two files, and the detectors need it + to attribute an outage to a specific migration rather than to the run. + + Column names match the older harness's CSVs (`second`, `wall_clock`) so a single + reader serves both. + """ + raw = kube.exec_sh(ns, pod, "cat /logs/iops.*log 2>/dev/null", timeout=180) + if not raw.strip(): + return + # fio: , , , , ... + per_sec: dict[int, dict[str, float]] = {} + for line in raw.splitlines(): + f = [x.strip() for x in line.split(",")] + if len(f) < 3: + continue + try: + sec = int(int(f[0]) / 1000) + val = float(f[1]) + rw = int(f[2]) + except ValueError: + continue + row = per_sec.setdefault(sec, {"read": 0.0, "write": 0.0}) + row["read" if rw == 0 else "write"] += val + if not per_sec: + return + + start = self._fio_time_base(ctx, pod, d) + with open(f"{d}/timeseries.csv", "w") as fh: + fh.write("second,wall_clock,total_iops,read_iops,write_iops,active_migration\n") + for sec in sorted(per_sec): + row = per_sec[sec] + total = row["read"] + row["write"] + wall = "" + active = "" + if start: + ts = start + timedelta(seconds=sec) + wall = ts.strftime("%Y-%m-%dT%H:%M:%SZ") + active = next((m.name for m in migs if m.covers(ts)), "") + fh.write(f"{sec},{wall},{total},{row['read']},{row['write']},{active}\n") + + def _fio_time_base(self, ctx: RunContext, pod: str, d: str) -> datetime | None: + """The wall clock of second 0 of one pod's fio time series. + + fio's log timestamps are milliseconds since *that job* started, so the only correct + base is the job's own start — `job_start` in its result.json, which fio records in + epoch milliseconds. The run's own window start is not that base: it is stamped when + the run began, before the PVCs, the pods and their fio processes existed, so it sits + well ahead of every pod's fio start (162-211s in fiomig-1787685649, and every pod by a + different amount). Using it shifts every wall clock in timeseries.csv by that much + and, because the same base decides which migration an outage overlaps, names the + wrong migration for gaps that fall within a few minutes of a real one. + + Falls back to the run's start when a pod's result.json is unreadable — a wrong base + is still better than an empty wall_clock column, and the fallback is reported. + """ + try: + with open(f"{d}/result.json") as fh: + jobs = json.load(fh).get("jobs") or [] + start_ms = jobs[0].get("job_start") if jobs else None + if isinstance(start_ms, int | float) and start_ms > 0: + return datetime.fromtimestamp(start_ms / 1000.0, tz=UTC) + except (OSError, json.JSONDecodeError, AttributeError, IndexError): + pass + run_start, _ = ctx.window() + ctx.log.warn(f"{self.name}: {pod}: fio result.json carries no job_start; falling back " + "to the run's start, which leads each pod's real fio start") + return run_start + + def teardown(self, ctx: RunContext) -> None: + if ctx.shared.get("keep"): + ctx.log.info(f"{self.name}: keep set; leaving {len(self._pods)} pod(s), " + f"{len(self._pvcs)} PVC(s) and the StorageClasses in place") + return + ns = self.opt("namespace") + kube.run(["-n", ns, "delete", "pod", "-l", "sbtest-run=true", + "--ignore-not-found", "--grace-period=5"], check=False, timeout=300) + kube.run(["-n", ns, "delete", "pvc", "-l", "sbtest-run=true", + "--ignore-not-found"], check=False, timeout=300) + for sc in (self._sc_single, self._sc_ns): + if sc: + kube.run(["delete", "sc", sc, "--ignore-not-found"], check=False) + ctx.log.info(f"{self.name}: removed pods, PVCs and StorageClasses") diff --git a/test/framework/sbtest/core/__init__.py b/test/framework/sbtest/core/__init__.py new file mode 100644 index 000000000..eb4caf883 --- /dev/null +++ b/test/framework/sbtest/core/__init__.py @@ -0,0 +1,41 @@ +"""Framework core: evidence, findings, plugins, config, context, runner.""" + +from .config import Config, Selection, apply_cli_toggles, load, suite_path +from .context import Event, Logger, RunContext, Timeline, iso, now_utc +from .evidence import ( + AnaSample, + ControlEvent, + Evidence, + FioJob, + IopsSample, + LogSpan, + Migration, + NvmeController, + attribute, + attribute_window, + freeze_windows, +) +from .findings import Attribution, Finding, Report, Severity, critical, info, warning +from .plugin import ( + Component, + Detector, + SkipDetector, + build_component, + build_detector, + component, + detector, + known_components, + known_detectors, +) +from .runner import Runner, findings_by_subject_table + +__all__ = [ + "AnaSample", "Attribution", "Component", "Config", "Detector", "Event", "Evidence", + "ControlEvent", "Finding", "LogSpan", + "FioJob", "IopsSample", "Logger", "Migration", "NvmeController", "Report", + "RunContext", "Runner", "Selection", "Severity", "SkipDetector", "Timeline", + "apply_cli_toggles", "attribute", "attribute_window", "build_component", + "build_detector", "component", + "critical", "detector", "findings_by_subject_table", "freeze_windows", "info", "iso", + "known_components", "known_detectors", "load", "now_utc", "suite_path", "warning", +] diff --git a/test/framework/sbtest/core/config.py b/test/framework/sbtest/core/config.py new file mode 100644 index 000000000..93d91b8cd --- /dev/null +++ b/test/framework/sbtest/core/config.py @@ -0,0 +1,162 @@ +"""Suite configuration — which components run, which detectors judge, and with what. + +The shape is deliberately flat and explicit: + + run: + id_prefix: fiomig + outdir: ./runs + components: + logs.stream: {enabled: true, containers: [spdk-container, spdk-proxy-container]} + logs.collect: {enabled: true} + ana.sample: {enabled: true, interval_s: 2.0} + detectors: + ana.freeze-count: {enabled: true, max_freezes: 1} + fio.checksum: {enabled: true, verify_lag_s: 45} + +Two rules make this predictable. A name absent from the config is **off** for components +and **on** for detectors — collecting is a cost you opt into, judging is not something you +should have to remember to switch on. And an unknown option is an error rather than a +silently ignored key, because a threshold that looks set but is not is worse than one that +is obviously missing. + +Suites are YAML. A suite is a document to be read and argued with — every threshold in one +wants a sentence saying why it is that number, and the numbers here were each paid for by a +run that went wrong. JSON cannot carry that sentence, so the format was the wrong one. A +suite written as `.json` still loads, for anything generating them programmatically. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from typing import Any + +import yaml + +DEFAULT_DETECTORS_ON = True +DEFAULT_COMPONENTS_ON = False + + +@dataclass +class Selection: + """A resolved name -> options mapping for one extension point.""" + + enabled: dict[str, dict[str, Any]] = field(default_factory=dict) + disabled: list[str] = field(default_factory=list) + + def __bool__(self) -> bool: + return bool(self.enabled) + + +@dataclass +class Config: + run: dict[str, Any] = field(default_factory=dict) + components: Selection = field(default_factory=Selection) + detectors: Selection = field(default_factory=Selection) + raw: dict[str, Any] = field(default_factory=dict) + + @property + def run_id_prefix(self) -> str: + return str(self.run.get("id_prefix", "sbtest")) + + @property + def outdir(self) -> str: + return str(self.run.get("outdir", ".")) + + +def _resolve(section: dict[str, Any], known: list[str], default_on: bool) -> Selection: + sel = Selection() + section = section or {} + + unknown = set(section) - set(known) + if unknown: + raise ValueError(f"unknown name(s) in config: {sorted(unknown)}; known: {sorted(known)}") + + for name in known: + spec = section.get(name) + if spec is None: + spec = default_on + if isinstance(spec, bool): + if spec: + sel.enabled.setdefault(name, {}) + else: + sel.disabled.append(name) + continue + if not isinstance(spec, dict): + raise ValueError(f"config for {name!r} must be a bool or a mapping, got {type(spec).__name__}") + opts = dict(spec) + if not opts.pop("enabled", True): + sel.disabled.append(name) + continue + sel.enabled[name] = opts + return sel + + +def load(path: str | None, known_components: list[str], known_detectors: list[str], + overrides: dict[str, Any] | None = None) -> Config: + """Read a suite file (or take defaults) and resolve it against what is registered. + + Resolving against the registry here rather than at use time means a typo in a name + fails at startup, with the list of valid names, instead of quietly running a suite that + is missing the check it was written for. + """ + raw: dict[str, Any] = {} + if path: + with open(path) as fh: + text = fh.read() + try: + # JSON keeps its own parser rather than riding YAML's superset handling, which + # differs on duplicate keys and tabs — a generated suite should fail on those + # rather than be quietly reinterpreted. + if path.endswith(".json"): + raw = json.loads(text) if text.strip() else {} + else: + raw = yaml.safe_load(text) or {} + except (yaml.YAMLError, json.JSONDecodeError) as e: + raise ValueError(f"cannot parse suite {path}: {e}") from e + if not isinstance(raw, dict): + raise ValueError(f"suite {path} must be a mapping at the top level, got " + f"{type(raw).__name__}") + for k, v in (overrides or {}).items(): + raw.setdefault(k, v) if not isinstance(v, dict) else raw.setdefault(k, {}).update(v) + + unknown_top = set(raw) - {"run", "components", "detectors", "description"} + if unknown_top: + raise ValueError(f"unknown top-level key(s) in {path}: {sorted(unknown_top)}") + + return Config( + run=raw.get("run", {}) or {}, + components=_resolve(raw.get("components", {}), known_components, DEFAULT_COMPONENTS_ON), + detectors=_resolve(raw.get("detectors", {}), known_detectors, DEFAULT_DETECTORS_ON), + raw=raw, + ) + + +def apply_cli_toggles(sel: Selection, enable: list[str], disable: list[str]) -> Selection: + """Apply `--enable x --disable y` on top of a resolved selection. + + Disable wins over enable when both name the same thing: a run someone explicitly narrowed + should not be widened by a default in a suite file they did not write. + """ + for name in enable: + if name in sel.disabled: + sel.disabled.remove(name) + sel.enabled.setdefault(name, {}) + for name in disable: + sel.enabled.pop(name, None) + if name not in sel.disabled: + sel.disabled.append(name) + return sel + + +def suite_path(name: str) -> str | None: + """Resolve a bare suite name against the bundled suites directory.""" + if os.path.sep in name or name.endswith((".yaml", ".yml", ".json")): + return name if os.path.exists(name) else None + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + for ext in (".yaml", ".yml", ".json"): + p = os.path.join(here, "suites", name + ext) + if os.path.exists(p): + return p + return None diff --git a/test/framework/sbtest/core/context.py b/test/framework/sbtest/core/context.py new file mode 100644 index 000000000..a8dd0bd8d --- /dev/null +++ b/test/framework/sbtest/core/context.py @@ -0,0 +1,169 @@ +"""RunContext — the shared state a component is handed, and the run's event timeline. + +Components get exactly one object, so adding a component never changes a signature +elsewhere. The timeline is the part worth explaining: components record events on it, and +detectors read those events back as evidence. That is how a sampler's observation reaches a +check without the two knowing about each other. +""" + +from __future__ import annotations + +import json +import os +import sys +import threading +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any + + +def now_utc() -> datetime: + return datetime.now(UTC) + + +def iso(ts: datetime) -> str: + return ts.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class Logger: + """Timestamped console log, mirrored to a file in the artifact directory. + + A file mirror rather than console-only because the whole framework exists to make runs + re-readable afterwards, and a run whose log lives only in someone's terminal scrollback + is a run that cannot be reviewed. + """ + + LEVELS = ("DEBUG", "INFO", "EVENT", "WARN", "ERROR", "CRITICAL") + + def __init__(self, path: str | None = None, verbose: bool = False) -> None: + self._fh = open(path, "a", buffering=1) if path else None # noqa: SIM115 + self._verbose = verbose + self._lock = threading.Lock() + + def _emit(self, level: str, msg: str) -> None: + if level == "DEBUG" and not self._verbose: + return + line = f"{iso(now_utc())} [{level:8}] {msg}" + with self._lock: + stream = sys.stderr if level in ("ERROR", "CRITICAL") else sys.stdout + print(line, file=stream, flush=True) + if self._fh: + self._fh.write(line + "\n") + + def debug(self, m: str) -> None: self._emit("DEBUG", m) + def info(self, m: str) -> None: self._emit("INFO", m) + def event(self, m: str) -> None: self._emit("EVENT", m) + def warn(self, m: str) -> None: self._emit("WARN", m) + def error(self, m: str) -> None: self._emit("ERROR", m) + def crit(self, m: str) -> None: self._emit("CRITICAL", m) + + def close(self) -> None: + if self._fh: + self._fh.close() + self._fh = None + + +@dataclass +class Event: + """Something a component observed, timestamped and typed. + + `kind` is what detectors filter on and should be a stable dotted name + ("migration.start", "node.offline"), because a detector keying off a message string + breaks the first time someone improves the wording. + """ + + ts: datetime + kind: str + subject: str = "" + data: dict[str, Any] = field(default_factory=dict) + + +class Timeline: + """Thread-safe ordered record of what happened during a run.""" + + def __init__(self) -> None: + self._events: list[Event] = [] + self._lock = threading.Lock() + + def record(self, kind: str, subject: str = "", **data: Any) -> Event: + ev = Event(ts=now_utc(), kind=kind, subject=subject, data=data) + with self._lock: + self._events.append(ev) + return ev + + def of_kind(self, *kinds: str) -> list[Event]: + with self._lock: + return [e for e in self._events if e.kind in kinds] + + def all(self) -> list[Event]: + with self._lock: + return sorted(self._events, key=lambda e: e.ts) + + def to_list(self) -> list[dict]: + return [{"ts": iso(e.ts), "kind": e.kind, "subject": e.subject, "data": e.data} + for e in self.all()] + + +@dataclass +class RunContext: + """Everything a component may need, and the only thing it is given.""" + + run_id: str + outdir: str + log: Logger + timeline: Timeline = field(default_factory=Timeline) + #: Free-form scratch space shared between components, keyed by component name. Used for + #: the handful of genuine dependencies between them (a workload publishing the pods it + #: created, so a sampler knows which nodes to watch) — always read defensively, since + #: the producing component may be disabled. + shared: dict[str, Any] = field(default_factory=dict) + #: Set when the run is being torn down, so a component's background loop can exit. + stopping: threading.Event = field(default_factory=threading.Event) + + def path(self, *parts: str) -> str: + """A path inside the artifact directory, with parent directories created.""" + p = os.path.join(self.outdir, *parts) + os.makedirs(os.path.dirname(p) or self.outdir, exist_ok=True) + return p + + def dir(self, *parts: str) -> str: + """A *directory* inside the artifact directory, created. Unlike `path`, which creates + the parent of the thing you name, this creates the thing you name.""" + p = os.path.join(self.outdir, *parts) + os.makedirs(p, exist_ok=True) + return p + + def window(self) -> tuple[datetime | None, datetime | None]: + """The run's start and end as currently known — end is None while it is still running. + + Components need the start to place a relative offset (fio counts seconds from its own + launch) on the wall clock, which is the only way an observation from one component can + be lined up against another's. + """ + return getattr(self, "_window_start", None), getattr(self, "_window_end", None) + + def mark_window(self, start: datetime | None = None, end: datetime | None = None) -> None: + """Record when the run began and ended, into run.json. + + Without this nothing in the artifact directory says when the run was, so every + detector that reads a ring buffer has to mark what it finds Attribution.UNKNOWN — and + UNKNOWN counts against the run. A live collect therefore failed on twenty-nine + filesystem shutdowns that had happened hours before it started. + """ + self._window_start = getattr(self, "_window_start", None) or start or now_utc() + # The *stored* end, not the argument. A later mark without one — a second collect + # phase, a component closing its own books — must not rewrite the record with no end + # at all: the window is what bounds every ring-buffer detector to this run, and an + # open window makes evidence from after it count as the run's. + self._window_end = end or getattr(self, "_window_end", None) + self.save_json("run.json", { + "run_id": self.run_id, + "start": iso(self._window_start), + "end": iso(self._window_end) if self._window_end else None, + }) + + def save_json(self, name: str, obj: Any) -> str: + p = self.path(name) + with open(p, "w") as fh: + json.dump(obj, fh, indent=2, default=str) + return p diff --git a/test/framework/sbtest/core/evidence.py b/test/framework/sbtest/core/evidence.py new file mode 100644 index 000000000..5b2b5745e --- /dev/null +++ b/test/framework/sbtest/core/evidence.py @@ -0,0 +1,305 @@ +"""Evidence — the only thing a detector is allowed to read. + +Detectors do not touch the cluster and do not know how anything was collected. They are +pure functions from Evidence to findings, and that is the whole point: the same detector +runs against a live run's in-memory state and against a four-hour archive on disk, so a +check can be fixed and re-tried against the run that motivated it instead of only against +the next one. Reproducing a verdict offline was the single biggest gap in the harness this +framework grew out of. + +Everything here is lazy. A run's SPDK logs are tens of megabytes per node and most +detectors never open them, so `container_log` yields lines and `ana_samples` is fetched per +migration rather than eagerly for all of them. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import Protocol, runtime_checkable + +# ── the value types detectors reason about ────────────────────────────────────────── + + +@dataclass(frozen=True) +class AnaSample: + """One controller's state on one consuming host at one instant.""" + + ts: datetime + node: str + address: str # ":" — an ip alone names a node, only ip:port names a path + state: str # controller state: live / connecting / resetting / ... + ana: dict[int, str] = field(default_factory=dict) # nsid -> ANA state + phase: str = "" # the migration's phase at that instant, when known + role: str = "" # source / target / other, when the collector knew it + + ACCESSIBLE = ("optimized", "non-optimized", "nonoptimized", "non_optimized") + + def accessible_nsids(self) -> set[int]: + return {n for n, a in self.ana.items() if a in self.ACCESSIBLE} + + @property + def ip(self) -> str: + return self.address.rsplit(":", 1)[0] + + @property + def port(self) -> str: + _, _, p = self.address.rpartition(":") + return p + + +@dataclass +class Migration: + """One migration attempt, as the timeline saw it.""" + + name: str + start: datetime + end: datetime | None = None + phase: str = "" # Completed / Failed / TIMEOUT / ... + source: str = "" # storage-node uuid + target: str = "" + pv: str = "" + pod: str = "" + members: list[str] = field(default_factory=list) # PVs moving together + error: str = "" + # Host-observed cutover instants per node, when the collector derived them. + cutover: dict[str, datetime] = field(default_factory=dict) + + @property + def batch(self) -> bool: + return len(self.members) > 1 + + def covers(self, ts: datetime, lag: timedelta = timedelta(0)) -> bool: + """Whether ts falls in this migration's window, optionally extended by `lag`. + + The lag exists for symptoms that are *detected* later than they happen — an fio + verify failure surfaces when fio next reads the block, seconds to tens of seconds + after the write was lost. Without it a migration's own losses get filed under "no + migration was running", which is exactly how a completed-but-corrupting migration + stayed invisible. + """ + if ts < self.start: + return False + end = self.end or self.start + return ts <= end + lag + + +@dataclass +class FioJob: + """One fio job's outcome for one pod.""" + + pod: str + error: int = 0 # errno fio ended with, 0 = clean + total_iops: float = 0.0 + read_iops: float = 0.0 + write_iops: float = 0.0 + + +@dataclass(frozen=True) +class IopsSample: + """One second of one pod's I/O, from the fio time series.""" + + offset_s: int # seconds since fio started + wall: datetime | None + total_iops: float + + +@dataclass(frozen=True) +class ControlEvent: + """One control-plane event — a status change, an object creation, a task update. + + The control plane's own account of what it thought was happening, which is the other half + of every host-side symptom: "the path went away" and "the control plane decided the node + was down" are the same incident told from two ends. + """ + + ts: datetime + level: str # Info / Warning / Error + kind: str # STATUS_CHANGE / OBJ_CREATED / ... + message: str + subject: str = "" # node / volume / task id, when the event names one + + +@dataclass(frozen=True) +class LogSpan: + """What time range a collected log actually covers. + + Exists because the answer is routinely "less than the run". A log-based verdict over a log + that only covers the last forty minutes of a four-hour run is not a verdict, and nothing + else in the evidence makes that visible. + """ + + name: str + first: datetime | None + last: datetime | None + lines: int = 0 + + +@dataclass(frozen=True) +class NvmeController: + """One NVMe controller on one host, as sysfs reports it. + + The shape that matters for leak detection: a controller can be `live` and serve no + namespace at all, which looks connected from every angle a connect checks. + """ + + node: str + name: str # "nvme7" + nqn: str + address: str # ":" + state: str # live / connecting / ... + namespaces: dict[int, str] = field(default_factory=dict) # nsid -> ana state + ctrl_loss_tmo: int | None = None + + @property + def serves_nothing(self) -> bool: + return not self.namespaces + + +# ── the contract ──────────────────────────────────────────────────────────────────── + + +@runtime_checkable +class Evidence(Protocol): + """What a detector may ask for. Every accessor may legitimately return nothing. + + A detector that needs evidence a run does not have must *say so* (see + `Detector.detect` and `Report.skip`) rather than return no findings — silence is how a + check that cannot run gets mistaken for a check that passed. + """ + + run_id: str + outdir: str + + def migrations(self) -> list[Migration]: ... + + def ana_samples(self, migration: str) -> list[AnaSample]: ... + + def fio_jobs(self) -> list[FioJob]: ... + + def fio_timeseries(self, pod: str) -> list[IopsSample]: ... + + def fio_log(self, pod: str) -> Iterator[str]: ... + + def container_logs(self) -> list[str]: + """Names of the container logs available, e.g. "spdk-4420", "operator".""" + ... + + def container_log(self, name: str) -> Iterator[str]: ... + + def nvme_controllers(self) -> list[NvmeController]: ... + + def pods(self) -> list[str]: ... + + def run_window(self) -> tuple[datetime | None, datetime | None]: + """When the run started and ended, or (None, None) if not known. + + Needed by any detector reading evidence that outlives the run — dmesg is a ring + buffer covering hours, so without a window a detector counts the previous runs' + damage as this one's. A detector that gets (None, None) must mark what it finds + Attribution.UNKNOWN rather than assume. + """ + ... + + def control_events(self) -> list[ControlEvent]: ... + + def log_spans(self) -> list[LogSpan]: + """The time range each collected log covers. Empty when not determinable.""" + ... + + def cluster_uuid(self) -> str: + """The cluster this run ran against, or "" when unknown. + + Present for one reason: an NQN names its cluster, so a controller whose NQN names a + *different* cluster is leaked beyond any doubt — no threshold, no topology, no + "might be a transient reconnect". See detectors/kernel.py::ForeignCluster. + """ + ... + + +# ── helpers shared by detectors ───────────────────────────────────────────────────── + + +def freeze_windows(samples: Iterable[AnaSample], + expected_nsids: set[int] | None = None) -> list[tuple[datetime, float]]: + """The windows in which some namespace had no accessible path on a node. + + Returns (start, seconds) per window, taken from the node that saw the most of them. + Per node rather than merged across nodes: every consuming host sees the same freeze, so + the count is how many times the volume froze, not how many hosts noticed. + + Zero-length windows are dropped. A window one sample wide began and ended between two + samples, so counting it would make the result depend on the sampling interval rather + than on what the volume did. + + This is the primitive behind the freeze-count detector, which is the sharpest predictor + of silent write loss found so far — see detectors/ana.py. + """ + by_node: dict[str, dict[datetime, set[int]]] = {} + for s in samples: + if not s.ana: + continue + per_ts = by_node.setdefault(s.node, {}) + per_ts.setdefault(s.ts, set()).update(s.accessible_nsids()) + + best: list[tuple[datetime, float]] = [] + for per_ts in by_node.values(): + times = sorted(per_ts) + if not times: + continue + want = expected_nsids or {n for acc in per_ts.values() for n in acc} + if not want: + continue + windows: list[tuple[datetime, float]] = [] + start: datetime | None = None + for t in times: + if want - per_ts[t]: + if start is None: + start = t + elif start is not None: + windows.append((start, (t - start).total_seconds())) + start = None + if start is not None: + windows.append((start, (times[-1] - start).total_seconds())) + windows = [w for w in windows if w[1] > 0] + if len(windows) > len(best): + best = windows + return best + + +def attribute(migrations: list[Migration], ts: datetime, + lag: timedelta = timedelta(0)) -> Migration | None: + """The migration a symptom at `ts` belongs to, allowing for detection lag.""" + for m in migrations: + if m.covers(ts, lag): + return m + return None + + +def attribute_window(migrations: list[Migration], start: datetime, end: datetime, + lag: timedelta = timedelta(0)) -> Migration | None: + """The migration a symptom that *lasted* belongs to: the one it shares most seconds with. + + Not the same question as `attribute`, and answering it with `attribute(start)` is what + made outages disappear. A gap does not have to begin inside a migration's window to + belong to it — the host goes dry a few seconds before the operator records the migration + as started — so testing only the first second files those gaps under "no migration was + running", which reads as "the cluster is unwell" rather than "the cutover cost this". + + Overlap is measured, not merely tested, because a long window can touch two migrations; + the one holding most of it is the one worth naming. Zero counts as an overlap: a + zero-length window inside a migration, and a window that only touches one, are both + inside rather than outside. + """ + t0, t1 = start.timestamp(), end.timestamp() + best: Migration | None = None + best_overlap: float | None = None + for m in migrations: + m_end = (m.end or m.start) + lag + overlap = min(t1, m_end.timestamp()) - max(t0, m.start.timestamp()) + if overlap < 0: + continue + if best_overlap is None or overlap > best_overlap: + best, best_overlap = m, overlap + return best diff --git a/test/framework/sbtest/core/findings.py b/test/framework/sbtest/core/findings.py new file mode 100644 index 000000000..7d6d78f3a --- /dev/null +++ b/test/framework/sbtest/core/findings.py @@ -0,0 +1,223 @@ +"""Findings — what a detector produces, and how a run is judged from them. + +A finding is deliberately more than a log line. The thing that made the migration +post-mortems expensive was never noticing that something was wrong; it was reconstructing, +hours later, *which* migration a symptom belonged to and *what evidence* said so. So a +finding carries its subject, the evidence that supports it, and where to look next — and +the report is assembled from findings rather than printed as it goes. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from enum import Enum, IntEnum + + +class Attribution(Enum): + """Whether a finding is something *this run* did. + + dmesg is a ring buffer and a cluster outlives its runs, so evidence routinely contains + damage and debris from before the run began. Counting that against the run is how a + green build gets blamed for its predecessor's mess — and, worse, how a genuinely broken + run hides inside inherited noise. + + The distinction changes the verdict rather than merely annotating it. A CRITICAL the run + caused is a FAIL. A CRITICAL it *inherited* means the environment was already damaged, so + the run cannot be trusted either way: that is INCONCLUSIVE, which is a different + conversation (fix the environment, run again) from a failure (fix the code). + """ + + RUN = "run" # happened inside the run's window + PRE_EXISTING = "pre-existing" # happened before the run started + UNKNOWN = "unknown" # no usable timestamp, or no known run window + + def __str__(self) -> str: # noqa: D105 + return self.value + + +class Severity(IntEnum): + """How much a finding matters. Ordered, so `max()` over findings is the verdict. + + The line that matters is CRITICAL vs the rest: CRITICAL fails the run. WARNING is for + something a human should read but that does not condemn the build — a threshold + approached, a cleanup that did not complete. INFO carries context that is only + interesting next to another finding. + """ + + INFO = 10 + WARNING = 20 + CRITICAL = 30 + + def __str__(self) -> str: # noqa: D105 + return self.name + + +@dataclass +class Finding: + """One defect, with everything needed to act on it without re-deriving it. + + `subject` is what the finding is *about* — a migration name, a pod, a node. It is what + makes findings joinable across detectors, which is where most of the diagnostic value + turned out to be: "the four migrations that re-froze" and "the four migrations that lost + writes" are only obviously the same set if both name their subjects the same way. + """ + + detector: str + severity: Severity + title: str + subject: str = "" + detail: str = "" + #: Whether the run caused this. Defaults to RUN: most detectors read evidence that only + #: exists because the run produced it, so anything they find is the run's. Detectors + #: reading a ring buffer or a long-lived cluster's state must say otherwise. + attribution: Attribution = Attribution.RUN + # Structured backing for the claim: counts, timestamps, thresholds. Goes to JSON + # verbatim, so keep it to primitives. + evidence: dict = field(default_factory=dict) + # Where a human should look — artifact paths, log offsets. + artifacts: list[str] = field(default_factory=list) + # Optional remediation or interpretation note. Used for findings whose meaning is not + # obvious from the title, which is most of the interesting ones. + note: str = "" + + def to_dict(self) -> dict: + d = asdict(self) + d["severity"] = str(self.severity) + d["attribution"] = str(self.attribution) + return d + + def one_line(self) -> str: + head = f"[{self.severity}] {self.detector}" + if self.subject: + head += f" {self.subject}" + if self.attribution is not Attribution.RUN: + head += f" ({self.attribution})" + return f"{head}: {self.title}" + + @property + def counts_against_the_run(self) -> bool: + """Whether this finding may fail the run. + + UNKNOWN counts: a detector that cannot place an event in time has found something + real, and treating "I am not sure when" as "not this run" is how a defect gets + excused. Only evidence positively dated before the run is excluded. + """ + return self.attribution is not Attribution.PRE_EXISTING + + +def _make(detector: str, severity: Severity, title: str, subject: str, detail: str, + evidence: dict | None, artifacts: list[str] | None, note: str, + attribution: Attribution) -> Finding: + return Finding(detector=detector, severity=severity, title=title, subject=subject, + detail=detail, evidence=evidence or {}, artifacts=artifacts or [], + note=note, attribution=attribution) + + +def critical(detector: str, title: str, subject: str = "", detail: str = "", + evidence: dict | None = None, artifacts: list[str] | None = None, + note: str = "", + attribution: Attribution = Attribution.RUN) -> Finding: + """A defect that fails the run.""" + return _make(detector, Severity.CRITICAL, title, subject, detail, evidence, artifacts, note, + attribution) + + +def warning(detector: str, title: str, subject: str = "", detail: str = "", + evidence: dict | None = None, artifacts: list[str] | None = None, + note: str = "", + attribution: Attribution = Attribution.RUN) -> Finding: + """Something a human should read that does not condemn the run.""" + return _make(detector, Severity.WARNING, title, subject, detail, evidence, artifacts, note, + attribution) + + +def info(detector: str, title: str, subject: str = "", detail: str = "", + evidence: dict | None = None, artifacts: list[str] | None = None, + note: str = "", + attribution: Attribution = Attribution.RUN) -> Finding: + """Context that is interesting next to another finding.""" + return _make(detector, Severity.INFO, title, subject, detail, evidence, artifacts, note, + attribution) + + +@dataclass +class Report: + """Every finding of a run, plus the verdict they add up to.""" + + run_id: str = "" + findings: list[Finding] = field(default_factory=list) + # Detectors that ran but could not decide, and why. A detector that silently returns + # nothing because its evidence was missing is indistinguishable from one that returned + # nothing because the run was clean — which is the failure mode that lets a broken + # check pass a broken run for weeks. + skipped: dict[str, str] = field(default_factory=dict) + + def add(self, *findings: Finding) -> None: + self.findings.extend(findings) + + def skip(self, detector: str, reason: str) -> None: + self.skipped[detector] = reason + + @property + def verdict(self) -> str: + """PASS, FAIL, or INCONCLUSIVE. + + INCONCLUSIVE is not a softer FAIL, it is a different problem. It means the run began + in a state that was already broken — a filesystem shut down before it started, a host + retrying controllers for a cluster that no longer exists — so neither a pass nor a + failure can be believed. The action is to clean the environment and run again, not to + go looking through the code. + """ + if self.failed: + return "FAIL" + if self.inherited_damage: + return "INCONCLUSIVE" + return "PASS" + + @property + def failed(self) -> bool: + """A CRITICAL this run is answerable for.""" + return any(f.severity >= Severity.CRITICAL and f.counts_against_the_run + for f in self.findings) + + @property + def inherited_damage(self) -> bool: + """A CRITICAL that pre-dates the run — the environment was already broken.""" + return any(f.severity >= Severity.CRITICAL and not f.counts_against_the_run + for f in self.findings) + + def attributed(self, attribution: Attribution) -> list[Finding]: + return [f for f in self.findings if f.attribution is attribution] + + def of_severity(self, sev: Severity, counting_only: bool = False) -> list[Finding]: + return [f for f in self.findings if f.severity == sev + and (f.counts_against_the_run or not counting_only)] + + def by_subject(self) -> dict[str, list[Finding]]: + """Findings grouped by what they are about, worst first within each subject. + + This is the join that turns separate detectors into a diagnosis: a subject carrying + both a freeze-count finding and a checksum finding is a much stronger statement than + either alone. + """ + out: dict[str, list[Finding]] = {} + for f in self.findings: + out.setdefault(f.subject or "-", []).append(f) + for v in out.values(): + v.sort(key=lambda f: -int(f.severity)) + return out + + def to_dict(self) -> dict: + return { + "run_id": self.run_id, + "verdict": self.verdict, + "counts": {str(s): len(self.of_severity(s)) for s in Severity}, + "counts_by_attribution": { + str(a): len(self.attributed(a)) for a in Attribution}, + "findings": [f.to_dict() for f in self.findings], + "skipped": self.skipped, + } + + def to_json(self, indent: int = 2) -> str: + return json.dumps(self.to_dict(), indent=indent, sort_keys=False) diff --git a/test/framework/sbtest/core/plugin.py b/test/framework/sbtest/core/plugin.py new file mode 100644 index 000000000..a49feff70 --- /dev/null +++ b/test/framework/sbtest/core/plugin.py @@ -0,0 +1,182 @@ +"""Detectors, components, and the registry that makes both selectable by name. + +Two extension points, deliberately different shapes: + +* A **Detector** is a pure judgement: Evidence in, findings out. No cluster access, no + ordering constraints, no state between runs. That is what makes them cheap to write, to + unit-test against a handful of synthetic samples, and to re-run against an archive. + +* A **Component** does the side-effecting work — starting pods, sampling, following logs, + driving a workload. It has a lifecycle, it can fail, and it may be enabled or disabled + independently of everything else. + +The split is the point. Every check worth having came from an incident, and none of them +should require touching collection code to add. Conversely, turning streaming log +collection on or off should not disturb a single check. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any + +from .findings import Finding + +if TYPE_CHECKING: # pragma: no cover + from .context import RunContext + from .evidence import Evidence + + +class SkipDetector(Exception): + """Raised by a detector whose evidence is absent, so the report can say so. + + The alternative — returning no findings — makes "could not check" and "checked, all + clean" the same output, which is how a broken check passes a broken run. + """ + + +class Detector: + """Base class for a defect detector. + + Subclasses set `name` and implement `detect`. Options arrive through `configure` from + the config file, so thresholds are never hard-coded at the call site — a detector's + default should encode what is known (see the ANA detectors, whose defaults come from + measured runs) while staying overridable per suite. + + Deliberately not a dataclass: a generated __init__ would assign the base class's empty + default over every subclass's `name`, so every finding and every skip would be reported + against an anonymous detector. + """ + + #: Dotted, stable — findings and config both key off it. + name: str = "" + #: One line, shown by `sbtest detectors`. Say what it catches, not how. + summary: str = "" + + def __init__(self) -> None: + self.options: dict[str, Any] = dict(self.defaults()) + + def configure(self, **options: Any) -> Detector: + unknown = set(options) - set(self.defaults()) + if unknown: + raise ValueError( + f"detector {self.name}: unknown option(s) {sorted(unknown)}; " + f"known: {sorted(self.defaults())}") + self.options = {**self.defaults(), **options} + return self + + def defaults(self) -> dict[str, Any]: + """Option names and their defaults. Also the allow-list for `configure`.""" + return {} + + def opt(self, key: str) -> Any: + return self.options.get(key, self.defaults().get(key)) + + def detect(self, ev: Evidence) -> Iterable[Finding]: # pragma: no cover + raise NotImplementedError + + +class Component: + """Base class for a lifecycle unit: collection, sampling, workload, driver. + + Every hook is optional; override only what applies. The runner calls them in this + order, and guarantees `stop`/`teardown` run for any component whose `setup` was + entered, so a component that starts pods is responsible for removing them and will be + given the chance to. + + setup() once, before anything runs. Resolve targets, create helpers. + start() begin doing the thing (sampling, following, driving). + tick() called periodically by the runner; cheap, must not block long. + stop() stop doing the thing. Data already written stays written. + collect() gather artifacts into the run directory. After stop, before detect. + teardown() release cluster resources. Runs even when the run failed. + + `collect` is separate from `stop` because the two differ for exactly the case that + motivated this framework: a streaming collector has nothing to collect at the end (it + has been writing all along) while a post-run collector does all its work there, and the + runner has to be able to run one, the other, or both without either knowing. + """ + + name: str = "" + #: One line, shown by `sbtest components`. + summary: str = "" + #: Whether a failure in `setup` should abort the run. + #: + #: False for collectors: a run that loses one evidence stream is still a run, and the + #: detectors that needed it will report themselves skipped. True for a component that + #: *is* the run — a workload or a migration driver — because continuing without it + #: produces a green result for a test that never happened, which is the worst outcome + #: available. + required: bool = False + + def __init__(self, **options: Any) -> None: + self.options = {**self.defaults(), **options} + unknown = set(options) - set(self.defaults()) + if unknown: + raise ValueError( + f"component {self.name}: unknown option(s) {sorted(unknown)}; " + f"known: {sorted(self.defaults())}") + + def defaults(self) -> dict[str, Any]: + return {} + + def opt(self, key: str) -> Any: + return self.options.get(key) + + # -- lifecycle, all optional ---------------------------------------------------- + def setup(self, ctx: RunContext) -> None: ... + def start(self, ctx: RunContext) -> None: ... + def tick(self, ctx: RunContext) -> None: ... + def stop(self, ctx: RunContext) -> None: ... + def collect(self, ctx: RunContext) -> None: ... + def teardown(self, ctx: RunContext) -> None: ... + + +# ── registry ──────────────────────────────────────────────────────────────────────── + +_DETECTORS: dict[str, Callable[[], Detector]] = {} +_COMPONENTS: dict[str, Callable[..., Component]] = {} + + +def detector(cls: type[Detector]) -> type[Detector]: + """Register a detector class under its `name`.""" + if not cls.name: + raise ValueError(f"{cls.__qualname__} must set `name`") + if cls.name in _DETECTORS: + raise ValueError(f"duplicate detector name {cls.name!r}") + _DETECTORS[cls.name] = cls + return cls + + +def component(cls: type[Component]) -> type[Component]: + """Register a component class under its `name`.""" + if not cls.name: + raise ValueError(f"{cls.__qualname__} must set `name`") + if cls.name in _COMPONENTS: + raise ValueError(f"duplicate component name {cls.name!r}") + _COMPONENTS[cls.name] = cls + return cls + + +def known_detectors() -> dict[str, type[Detector]]: + return dict(sorted(_DETECTORS.items())) # type: ignore[arg-type] + + +def known_components() -> dict[str, type[Component]]: + return dict(sorted(_COMPONENTS.items())) # type: ignore[arg-type] + + +def build_detector(name: str, **options: Any) -> Detector: + try: + cls = _DETECTORS[name] + except KeyError: + raise KeyError(f"unknown detector {name!r}; known: {sorted(_DETECTORS)}") from None + return cls().configure(**options) + + +def build_component(name: str, **options: Any) -> Component: + try: + cls = _COMPONENTS[name] + except KeyError: + raise KeyError(f"unknown component {name!r}; known: {sorted(_COMPONENTS)}") from None + return cls(**options) diff --git a/test/framework/sbtest/core/runner.py b/test/framework/sbtest/core/runner.py new file mode 100644 index 000000000..afb9a3c47 --- /dev/null +++ b/test/framework/sbtest/core/runner.py @@ -0,0 +1,234 @@ +"""The runner — drives component lifecycles, then runs detectors over the evidence. + +Two entry points, and the important thing is that they share the second half: + + Runner.execute() full run: components through their lifecycle, then judge. + Runner.judge(ev) judge alone, over evidence from anywhere — including an archive. + +Sharing the judging half is what makes a check fixable against the run that motivated it. +""" + +from __future__ import annotations + +import time +import traceback +from typing import Any + +from .config import Config +from .context import RunContext, now_utc +from .evidence import Evidence +from .findings import Report, Severity, warning +from .plugin import Component, Detector, SkipDetector, build_component, build_detector + + +class Runner: + def __init__(self, cfg: Config, ctx: RunContext) -> None: + self.cfg = cfg + self.ctx = ctx + self.components: list[Component] = [] + self.detectors: list[Detector] = [] + self.report = Report(run_id=ctx.run_id) + #: Components whose setup was entered, so teardown is owed to them. + self._entered: list[Component] = [] + + # ── wiring ───────────────────────────────────────────────────────────────────── + + def build(self) -> Runner: + for name, opts in self.cfg.components.enabled.items(): + try: + self.components.append(build_component(name, **opts)) + except Exception as e: # noqa: BLE001 + raise RuntimeError(f"cannot build component {name!r}: {e}") from e + for name, opts in self.cfg.detectors.enabled.items(): + try: + self.detectors.append(build_detector(name, **opts)) + except Exception as e: # noqa: BLE001 + raise RuntimeError(f"cannot build detector {name!r}: {e}") from e + + if self.cfg.components.disabled: + self.ctx.log.info("components disabled: " + ", ".join(sorted(self.cfg.components.disabled))) + if self.cfg.detectors.disabled: + self.ctx.log.info("detectors disabled: " + ", ".join(sorted(self.cfg.detectors.disabled))) + self.ctx.log.info(f"components enabled ({len(self.components)}): " + + (", ".join(c.name for c in self.components) or "-")) + self.ctx.log.info(f"detectors enabled ({len(self.detectors)}): " + + (", ".join(d.name for d in self.detectors) or "-")) + return self + + # ── lifecycle ────────────────────────────────────────────────────────────────── + + def _phase(self, hook: str, comps: list[Component], fatal: bool = False) -> None: + """Call one hook on each component, recording rather than raising on failure. + + A collector that fails must not take the run with it — the run's purpose is the + workload and the judgement, and partial evidence still judges; the failure becomes a + WARNING finding so the gap is on the record rather than invisible. + + `fatal` is passed for the setup of a component that declares itself `required`, + which is how a workload or a migration driver says that continuing without it would + produce a green result for a test that never ran. + """ + for c in comps: + try: + getattr(c, hook)(self.ctx) + except Exception as e: # noqa: BLE001 + self.ctx.log.error(f"component {c.name}.{hook} failed: {e}") + self.ctx.log.debug(traceback.format_exc()) + self.report.add(warning( + detector=f"component/{c.name}", + title=f"{hook} failed: {e}", + subject=c.name, + note="Evidence this component would have produced may be missing; any " + "detector that depends on it will report itself skipped.", + )) + if fatal: + raise + + def setup(self) -> None: + # Before any component runs: the window has to include setup, because a component + # that breaks something breaks it during setup as much as during the run. + self.ctx.mark_window() + for c in self.components: + # Appended before the hook runs, so a component that failed half-way through + # allocating still gets the teardown it is owed. + self._entered.append(c) + self._phase("setup", [c], fatal=c.required) + + def start(self) -> None: + self._phase("start", self.components) + + def tick(self) -> None: + self._phase("tick", self.components) + + def run_for(self, seconds: float, tick_every: float = 5.0) -> None: + """Tick components for `seconds`. The scenario's own work happens in a component.""" + deadline = time.time() + seconds + while time.time() < deadline and not self.ctx.stopping.is_set(): + self.tick() + time.sleep(min(tick_every, max(0.0, deadline - time.time()))) + + def stop(self) -> None: + self.ctx.stopping.set() + self._phase("stop", list(reversed(self.components))) + + def collect(self) -> None: + self._phase("collect", self.components) + # Closed after collection, so the window covers everything the artifacts contain. + self.ctx.mark_window(end=now_utc()) + + def teardown(self) -> None: + self._phase("teardown", list(reversed(self._entered))) + + # ── judging ──────────────────────────────────────────────────────────────────── + + def judge(self, ev: Evidence) -> Report: + """Run every enabled detector over `ev` and fold the results into the report.""" + for d in self.detectors: + try: + found = list(d.detect(ev)) + except SkipDetector as e: + self.report.skip(d.name, str(e) or "evidence not available") + continue + except Exception as e: # noqa: BLE001 + # A detector that throws is a bug in the detector, not a clean run. Say so + # loudly rather than letting it look like "nothing found". + self.ctx.log.error(f"detector {d.name} raised: {e}") + self.ctx.log.debug(traceback.format_exc()) + self.report.skip(d.name, f"raised {type(e).__name__}: {e}") + self.report.add(warning( + detector=d.name, title=f"detector raised {type(e).__name__}: {e}", + note="This is a detector bug: the run was not judged on this dimension.")) + continue + self.report.add(*found) + return self.report + + def execute(self, evidence_for: Any, duration_s: float = 0.0) -> Report: + """Full run: lifecycle, then judge. `evidence_for` builds Evidence from the ctx.""" + try: + self.setup() + self.start() + if duration_s: + self.run_for(duration_s) + self.stop() + self.collect() + finally: + self.teardown() + return self.judge(evidence_for(self.ctx)) + + # ── output ───────────────────────────────────────────────────────────────────── + + def emit(self, json_name: str = "findings.json") -> Report: + r = self.report + log = self.ctx.log + line = "=" * 78 + log.info(line) + log.info("FINDINGS") + log.info(line) + + if not r.findings: + log.info("no findings") + for sev in (Severity.CRITICAL, Severity.WARNING, Severity.INFO): + group = [f for f in r.of_severity(sev) if f.counts_against_the_run] + if not group: + continue + log.info(f"{sev} ({len(group)}):") + for f in group: + emit = log.crit if sev is Severity.CRITICAL else ( + log.warn if sev is Severity.WARNING else log.info) + emit(f" {f.subject or '-'}: {f.title}") + if f.detail: + for chunk in f.detail.splitlines(): + emit(f" {chunk}") + if f.note: + emit(f" note: {f.note}") + for a in f.artifacts: + emit(f" evidence: {a}") + + inherited = [f for f in r.findings if not f.counts_against_the_run] + if inherited: + log.info(line) + log.info(f"PRE-EXISTING ({len(inherited)}) — already true before this run began, " + "so not counted against it:") + for f in inherited: + emit = log.crit if f.severity is Severity.CRITICAL else log.warn + emit(f" [{f.severity}] {f.subject or '-'}: {f.title}") + if f.detail: + for chunk in f.detail.splitlines(): + emit(f" {chunk}") + if f.note: + emit(f" note: {f.note}") + + if r.skipped: + log.info(f"skipped ({len(r.skipped)}) — not judged on these dimensions:") + for name, why in sorted(r.skipped.items()): + log.warn(f" {name}: {why}") + + path = self.ctx.path(json_name) + with open(path, "w") as fh: + fh.write(r.to_json()) + log.info(line) + summary = (f"{len(r.of_severity(Severity.CRITICAL, counting_only=True))} critical, " + f"{len(r.of_severity(Severity.WARNING, counting_only=True))} warning") + if inherited: + summary += f", {len(inherited)} pre-existing" + log.info(f"RESULT: {r.verdict} ({summary}) -> {path}") + if r.verdict == "INCONCLUSIVE": + log.crit("The environment was already damaged before this run started, so neither " + "a pass nor a failure can be believed. Clean it up and run again.") + log.info(line) + return r + + +def findings_by_subject_table(report: Report) -> list[str]: + """Findings grouped by subject — the join that makes correlated defects visible. + + A subject carrying two independent findings is the strongest signal these runs produce: + "re-froze the volume" plus "lost writes" on the same migration is what turned a + correlation into a root cause. + """ + lines = [] + for subject, group in sorted(report.by_subject().items()): + if len(group) < 2: + continue + lines.append(f"{subject}: " + "; ".join(f"{f.detector}({f.severity})" for f in group)) + return lines diff --git a/test/framework/sbtest/detectors/__init__.py b/test/framework/sbtest/detectors/__init__.py new file mode 100644 index 000000000..8a44ddba9 --- /dev/null +++ b/test/framework/sbtest/detectors/__init__.py @@ -0,0 +1,23 @@ +"""Bundled detectors. Importing this module registers them all. + +Each module groups checks by the evidence they read, not by the subsystem they blame: +`ana` reads host path samples, `fio` reads the workload's own output, `nvme` reads a fabric +snapshot, `logs` reads collected container logs, `migration` reads the timeline, `kernel` reads +dmesg — the only source for what the *host* did about a fabric event — `control` reads the +control plane's own event log, `security` scans whatever was collected, and `meta` judges the +evidence itself rather than the system. +""" + +from . import ( # noqa: F401 + ana, + control, + fio, + kernel, + logs, + meta, + migration, + nvme, + security, +) + +__all__ = ["ana", "control", "fio", "kernel", "logs", "meta", "migration", "nvme", "security"] diff --git a/test/framework/sbtest/detectors/ana.py b/test/framework/sbtest/detectors/ana.py new file mode 100644 index 000000000..0a0ed519f --- /dev/null +++ b/test/framework/sbtest/detectors/ana.py @@ -0,0 +1,311 @@ +"""ANA / host-path detectors. + +These are the checks that came out of the migration corruption work, and the first one is +the reason this framework has a detector abstraction at all: it is a one-number check that +predicted silent data loss exactly, and it was not expressible in the harness without +editing the harness. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import ( + AnaSample, + Detector, + Evidence, + Finding, + SkipDetector, + critical, + detector, + freeze_windows, + info, + warning, +) + +#: How long the control plane deliberately holds every path inaccessible while it moves a +#: volume. Measured, not guessed: it is a `time.sleep(2)` in the batch migration barrier. +CUTOVER_PAUSE_DESIGN_S = 2.0 + + +@detector +class FreezeCount(Detector): + """More than one cutover freeze in a migration. + + On the run this came from (fiomig-1787171993, 46 migrations) this was **exact**: the + four migrations that froze the volume more than once are precisely the four that + silently lost writes, and none of the other 42 lost anything. + + The mechanism is why it works. A migration takes the pause once; a second window means + the control plane released the source back into the read/write path and retried the + transfer, and the retry replays a non-idempotent step against a source that has been + serving writes in between. The freeze count is the retry count. + + Prefer this over the pause *duration*, which the same run showed to be both less + sensitive and less specific: two 3s freezes look like one healthy pause on any + longest-window measure, and one migration's single 5-6s pause lost nothing. + """ + + name = "ana.freeze-count" + summary = ("a migration that froze the volume more than once — retried cutover, and an " + "exact predictor of silent write loss so far") + + def defaults(self) -> dict: + return { + "max_freezes": 1, + # A freeze shorter than this is treated as sampling noise rather than a window. + # 0 keeps every non-zero window, which is what the measured result used. + "min_window_s": 0.0, + } + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples: + continue + judged += 1 + windows = [w for w in freeze_windows(samples) + if w[1] >= float(self.opt("min_window_s"))] + if len(windows) <= int(self.opt("max_freezes")): + continue + spans = ", ".join(f"{s.strftime('%H:%M:%S')}+{d:.0f}s" for s, d in windows) + yield critical( + self.name, + title=f"volume froze {len(windows)} times during one migration", + subject=m.name, + detail=f"windows: {spans}", + evidence={"freezes": len(windows), "max_freezes": int(self.opt("max_freezes")), + "windows_s": [round(d, 1) for _, d in windows], + "phase": m.phase, "members": len(m.members)}, + note="A migration takes the cutover pause once; the rest are retries of a " + "step that did not take. Every migration observed to freeze more than " + "once has also silently lost writes — treat as corruption until the " + "checksums say otherwise, including when the phase is Completed.", + ) + if not judged: + raise SkipDetector("no ANA samples for any migration") + + +@detector +class CutoverPause(Detector): + """A cutover pause that ran longer than the design window allows. + + A bounded pause is expected — the control plane drives every path inaccessible for about + CUTOVER_PAUSE_DESIGN_S while it moves the volume — so this bounds that window rather + than forbidding it. The default allows the design window plus one sampling interval, + which is the widest a well-behaved pause can be *measured* as. + + Keep it alongside ana.freeze-count rather than instead of it: this catches a single + window that overran, which the count cannot see. + """ + + name = "ana.cutover-pause" + summary = "an all-paths-inaccessible window longer than the cutover pause allows" + + def defaults(self) -> dict: + return {"max_pause_s": CUTOVER_PAUSE_DESIGN_S + 3.0, + "design_pause_s": CUTOVER_PAUSE_DESIGN_S} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + limit = float(self.opt("max_pause_s")) + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples: + continue + judged += 1 + windows = freeze_windows(samples) + if not windows: + continue + worst = max(d for _, d in windows) + if worst <= limit: + continue + yield critical( + self.name, + title=f"every path to some namespace was inaccessible for {worst:.0f}s", + subject=m.name, + detail=f"limit {limit:.0f}s; the cutover pause is meant to last about " + f"{float(self.opt('design_pause_s')):.0f}s", + evidence={"worst_pause_s": round(worst, 1), "max_pause_s": limit, + "freezes": len(windows), "phase": m.phase}, + note="An application that sees an I/O error during a migration sees it " + "inside this window, so its length bounds the blast radius.", + ) + if not judged: + raise SkipDetector("no ANA samples for any migration") + + +@detector +class SplitBrain(Detector): + """Source and target both serving at the same instant. + + Two simultaneously optimized paths to two copies means a read can land on either, and + the one that is behind returns data that was never written there. This is the one ANA + defect that is silent corruption *by construction* rather than by race, so it is worth + checking even though it has not yet been observed. + """ + + name = "ana.split-brain" + summary = "source and target paths both optimized at the same instant (two writers)" + + def defaults(self) -> dict: + return {"optimized_states": ["optimized"]} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + opt = set(self.opt("optimized_states")) + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples or not any(s.role for s in samples): + continue + judged += 1 + per_ts: dict = {} + for s in samples: + if any(a in opt for a in s.ana.values()): + per_ts.setdefault(s.ts, {}).setdefault(s.role or "?", set()).add(s.address) + for ts, roles in sorted(per_ts.items()): + if "source" in roles and "target" in roles: + yield critical( + self.name, + title="source and target both optimized at the same instant", + subject=m.name, + detail=f"at {ts:%H:%M:%S}: source={sorted(roles['source'])} " + f"target={sorted(roles['target'])}", + evidence={"ts": str(ts), "source": sorted(roles["source"]), + "target": sorted(roles["target"])}, + note="Reads can be served by either copy; the one that is behind " + "returns data that was never written at that offset.", + ) + break # one finding per migration is enough to act on + if not judged: + raise SkipDetector("no role-labelled ANA samples (needs source/target roles)") + + +@detector +class UnservedAfterCutover(Detector): + """A completed migration whose target does not serve every namespace. + + The half-moved case seen from the host: the migration reports success, a live controller + exists at the target, and one of the subsystem's namespaces has no path over it. On a + shared subsystem that is one volume left stranded while its siblings moved. + """ + + name = "ana.unserved-after-cutover" + summary = "after a Completed migration, a live target controller serves only some namespaces" + + def defaults(self) -> dict: + return {"completed_phases": ["Completed"]} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = [m for m in ev.migrations() if m.phase in set(self.opt("completed_phases"))] + if not migs: + raise SkipDetector("no completed migrations in this run") + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + targets = [s for s in samples if s.role == "target"] + if not targets: + continue + judged += 1 + last_ts = max(s.ts for s in samples) + final = [s for s in targets if s.ts == last_ts] + live = [s for s in final if s.state == "live"] + served = {n for s in live for n in s.accessible_nsids()} + expected = {n for s in samples for n in s.ana} + missing = expected - served + if live and expected and missing: + yield critical( + self.name, + title=f"{len(missing)} namespace(s) unserved on the target after cutover", + subject=m.name, + detail=f"target serves {sorted(served) or '-'} of {sorted(expected)}; " + f"missing {sorted(missing)}", + evidence={"served": sorted(served), "expected": sorted(expected), + "missing": sorted(missing)}, + ) + elif not live: + yield warning( + self.name, + title="no live target controller after a completed migration", + subject=m.name, + detail=f"controllers at the target: " + f"{sorted({(s.address, s.state) for s in final})}", + note="Non-live controllers at the target address are normal while the " + "old instance is torn down; a persisting one is not.", + ) + if not judged: + raise SkipDetector("no role-labelled ANA samples for completed migrations") + + +@detector +class PathChurn(Detector): + """An unusual number of distinct path addresses for one subsystem on one host. + + A leak indicator that does not need a live cluster: a subsystem accumulating listeners + it never sheds shows up in the ANA samples as a growing address set. Informational by + default, because the healthy count is topology-dependent (primary plus HA replicas per + side) and a threshold that fits one cluster will not fit another. + """ + + name = "ana.path-churn" + summary = "more distinct path addresses per host than the topology should produce" + + def defaults(self) -> dict: + return {"max_addresses": 6, "severity": "info"} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + limit = int(self.opt("max_addresses")) + make = critical if self.opt("severity") == "critical" else ( + warning if self.opt("severity") == "warning" else info) + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples: + continue + judged += 1 + per_node: dict[str, set[str]] = {} + for s in samples: + per_node.setdefault(s.node, set()).add(s.address) + for node, addrs in sorted(per_node.items()): + if len(addrs) > limit: + yield make( + self.name, + title=f"{len(addrs)} distinct path addresses on {node}", + subject=m.name, + detail=", ".join(sorted(addrs)), + evidence={"node": node, "addresses": sorted(addrs), "limit": limit}, + note="An ip names a node; only ip:port names a path. Extra " + "addresses on one node are usually leaked listeners from " + "earlier migrations.", + ) + if not judged: + raise SkipDetector("no ANA samples for any migration") + + +def freeze_summary(ev: Evidence) -> list[tuple[str, int, float]]: + """(migration, freezes, worst window) for every migration with samples. + + Exposed because it is the table that makes the freeze-count result legible at a glance, + and callers other than the detector want it — the CLI prints it and the README quotes it. + """ + out = [] + for m in ev.migrations(): + samples: list[AnaSample] = ev.ana_samples(m.name) + if not samples: + continue + w = freeze_windows(samples) + out.append((m.name, len(w), max((d for _, d in w), default=0.0))) + return out diff --git a/test/framework/sbtest/detectors/control.py b/test/framework/sbtest/detectors/control.py new file mode 100644 index 000000000..319d4dadc --- /dev/null +++ b/test/framework/sbtest/detectors/control.py @@ -0,0 +1,341 @@ +"""Control-plane detectors, from the cluster's own event log. + +The other half of every host-side symptom. "Every path went away" and "the control plane +decided the node was down" are the same incident told from two ends, and only one of those +ends explains *why*. + +Nothing here is migration-specific — these are the failure shapes a distributed control plane +has regardless of what operation is running, which is why they belong in a framework rather +than in a migration test. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Iterable +from datetime import UTC, datetime, timedelta + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector, info, warning + + +@detector +class NodeFlap(Detector): + """A node that went offline and came back inside one run. + + The shape behind a real 9.5-hour outage: a three-second kube-apiserver blip made the + liveness probe conclude SPDK was dead — the probe listed pods through the Kubernetes API — + so three storage nodes were marked offline at once, and nothing re-probed them afterwards. + + A flap is therefore worth reporting *even when it recovered*, because the recovery is + luck: the same trigger with a slower re-probe is an outage. A short flap is the signal, not + the noise — a node that is genuinely gone stays gone. + """ + + name = "control.node-flap" + summary = "a storage node marked offline and back inside one run — liveness, not liveliness" + + RE_STATUS = re.compile( + r"(?:Storage node|Management node) status changed from:?\s*(\S+)\s*to:?\s*(\S+)") + #: States that mean the control plane stopped trusting the node. + DOWN = {"down", "offline", "unreachable", "suspended"} + UP = {"online", "active"} + + def defaults(self) -> dict: + return {"max_flaps": 0, + # A transition back inside this window is a flap rather than a real outage + # followed by a real recovery. + "flap_within_s": 300.0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + window = timedelta(seconds=float(self.opt("flap_within_s"))) + + # subject -> [(ts, went_down)] + transitions: dict[str, list] = {} + for e in events: + m = self.RE_STATUS.search(e.message) + if not m: + continue + frm, to = m.group(1).strip(". "), m.group(2).strip(". ") + if to in self.DOWN: + transitions.setdefault(e.subject or "?", []).append((e.ts, True, frm, to)) + elif to in self.UP and frm in self.DOWN: + transitions.setdefault(e.subject or "?", []).append((e.ts, False, frm, to)) + + flaps = [] + for subject, seq in transitions.items(): + seq.sort() + for i, (ts, down, _f, to) in enumerate(seq): + if not down: + continue + back = next(((t2, f2) for t2, d2, f2, _t2 in seq[i + 1:] if not d2), None) + if back and back[0] - ts <= window: + flaps.append((subject, ts, (back[0] - ts).total_seconds(), to)) + + if len(flaps) <= int(self.opt("max_flaps")): + return + yield critical( + self.name, + title=f"{len(flaps)} node flap(s): marked down and back within " + f"{float(self.opt('flap_within_s')):.0f}s", + subject="control-plane", + detail="; ".join(f"{s[:8]} {t:%H:%M:%S} down->up in {d:.0f}s (as {to})" + for s, t, d, to in sorted(flaps, key=lambda x: x[1])[:8]), + evidence={"flaps": [{"subject": s, "at": str(t), "seconds": round(d), "state": to} + for s, t, d, to in flaps]}, + note="A node that recovers in seconds was probably never gone: the usual cause is " + "a liveness check that depends on something other than the node — an API " + "call, a lock, a lease. Every volume whose paths were on it took a hit for " + "the duration, and the recovery here was luck rather than design.", + ) + + +@detector +class TaskStuck(Detector): + """A control-plane task that never reached a terminal state. + + Generic on purpose. Whatever the task is — migration, rebalance, backup, node add — one + that is created and never resolved holds locks, blocks the next operation of its kind, and + makes the cluster's state un-diagnosable. This is the check that says "the control plane + stopped finishing things", which no per-operation test asks. + """ + + name = "control.task-stuck" + summary = "a task created during the run that never reached a terminal state" + + RE_CREATED = re.compile(r"[Tt]ask created") + RE_UPDATED = re.compile(r"[Tt]ask updated") + TERMINAL = ("done", "completed", "failed", "cancelled", "canceled", "suspended") + + def defaults(self) -> dict: + return {"max_stuck": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + + created = sum(1 for e in events if self.RE_CREATED.search(e.message)) + if not created: + raise SkipDetector("the event log records no task creations to follow") + terminal = sum(1 for e in events + if self.RE_UPDATED.search(e.message) + and any(t in e.message.lower() for t in self.TERMINAL)) + outstanding = created - terminal + if outstanding <= int(self.opt("max_stuck")): + yield info(self.name, title=f"{created} task(s) created, {terminal} resolved", + subject="control-plane", + evidence={"created": created, "terminal": terminal}) + return + yield warning( + self.name, + title=f"{outstanding} task(s) created but never seen to finish", + subject="control-plane", + detail=f"{created} created, {terminal} reached a terminal state", + evidence={"created": created, "terminal": terminal, "outstanding": outstanding}, + note="The event log may simply end before they finished — check it against the run " + "window before treating this as a leak. A task that is genuinely stuck holds " + "its lock and blocks the next operation of its kind.", + ) + + +@detector +class VolumeHealth(Detector): + """A volume whose health went false and never came back. + + Independent of what the run was doing: whatever the operation, a volume that ends the run + unhealthy is a volume someone has to go and look at. On one archived run five volumes went + unhealthy mid-migration and only three recovered — the other two stayed down for the rest + of the run, which is a much more useful statement than the migration's own phase. + """ + + name = "control.volume-health" + summary = "a volume or node whose health went false during the run and never recovered" + + RE_HEALTH = re.compile(r"(\S+) health check changed from:?\s*(\S+)\s*to:?\s*(\S+)") + + def defaults(self) -> dict: + return {"max_unrecovered": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + state: dict[str, bool] = {} + seen = False + for e in events: + m = self.RE_HEALTH.search(e.message) + if not m: + continue + seen = True + to = m.group(3).strip(". ").lower() + state[e.subject or m.group(1)] = to in ("true", "healthy", "online") + if not seen: + raise SkipDetector("the event log records no health transitions") + + bad = sorted(k for k, ok in state.items() if not ok) + if len(bad) <= int(self.opt("max_unrecovered")): + return + yield critical( + self.name, + title=f"{len(bad)} object(s) ended the run unhealthy", + subject="control-plane", + detail=", ".join(b[:12] for b in bad[:16]) + (" ..." if len(bad) > 16 else ""), + evidence={"unhealthy": bad}, + note="Health that goes false and does not return outlives the run: the next run " + "starts from here. Check it against nvme.dirty-start on the following run.", + ) + + +@detector +class RetryStorm(Detector): + """One operation attempted far more often than it should need to be. + + A retry loop is invisible in a pass/fail result and obvious in a count. It matters beyond + the operation retried: each attempt usually re-does whatever the previous one half-did, + which is how a retry turns a failure into damage — the batch-migration corruption is + exactly that shape. + """ + + name = "control.retry-storm" + summary = "the same control-plane operation attempted far more often than it should be" + + def defaults(self) -> dict: + return {"max_repeats": 5, "kinds": ["STATUS_CHANGE"]} + + @staticmethod + def _shape(msg: str) -> str: + s = re.sub(r"[0-9a-f]{8}-[0-9a-f-]{27}", "", msg) + s = re.sub(r"\d+", "N", s) + return s[:120] + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + kinds = set(self.opt("kinds") or []) + counts: dict[tuple[str, str], int] = {} + for e in events: + if kinds and e.kind not in kinds: + continue + counts[(e.subject or "?", self._shape(e.message))] = \ + counts.get((e.subject or "?", self._shape(e.message)), 0) + 1 + + hot = {k: v for k, v in counts.items() if v > int(self.opt("max_repeats"))} + if not hot: + return + worst = sorted(hot.items(), key=lambda kv: -kv[1])[:8] + yield warning( + self.name, + title=f"{len(hot)} operation(s) repeated more than {self.opt('max_repeats')} times", + subject="control-plane", + detail="; ".join(f"{n}x {msg}" for (_subj, msg), n in worst), + evidence={"repeats": [{"subject": s, "shape": m, "count": n} + for (s, m), n in worst]}, + note="Each attempt usually re-does what the last one half-did, which is how a " + "retry turns a failure into damage rather than merely delaying it.", + ) + + +@detector +class NodeAgent(Detector): + """The node-side agent's own account: failed calls, and gaps in the liveness polling. + + The storage-node DaemonSet is what starts and probes the SPDK process, so it sits on the + causal path of every "the node went offline" decision. Its access log is mostly liveness + polling — `/snode/check`, `/snode/spdk_process_is_up`, `/snode/ping_ip`, thousands of each + per run — and two things in it are worth watching: + + * **a non-2xx response.** The control plane asked whether SPDK was alive and got an error + rather than an answer. Whatever it decided next, it decided on no information; this is + the upstream half of `control.node-flap`. + * **a gap in the polling.** Either the control plane stopped asking or the agent stopped + answering, and both precede a node being declared down. A poll that never happened + leaves no other trace, so this is the only place the stall is visible. + + Grounded rather than speculative: the outage this is built for was a liveness check that + concluded SPDK was dead because a Kubernetes API call it depended on blipped for three + seconds, and three nodes went offline at once. + """ + + name = "control.node-agent" + summary = "the node agent returning errors, or gaps in the liveness polling it answers" + + RE_ACCESS = re.compile(r'"(?:GET|POST|PUT|DELETE) (/snode/[a-z_]+)[^"]*" (\d{3})') + RE_TS = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})") + + def defaults(self) -> dict: + return {"logs": ["snode-api-*"], + # A liveness poll is expected every couple of seconds; this is where a gap + # stops being scheduling jitter and starts being a stall. + "max_poll_gap_s": 60.0, + "liveness_endpoints": ["/snode/check", "/snode/spdk_process_is_up"], + "max_errors": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + names = [n for n in ev.container_logs() + if any(fnmatch.fnmatch(n, g) for g in self.opt("logs"))] + if not names: + raise SkipDetector("no node-agent logs collected (enable the storage-node " + "DaemonSet target in logs.collect)") + + live = set(self.opt("liveness_endpoints") or []) + gap_limit = float(self.opt("max_poll_gap_s")) + errors: dict[str, dict[str, int]] = {} + gaps: list[tuple[str, str, float]] = [] + + for name in names: + last_poll: datetime | None = None + for raw in ev.container_log(name): + m = self.RE_ACCESS.search(raw) + if not m: + continue + endpoint, status = m.group(1), m.group(2) + if not status.startswith("2"): + key = f"{endpoint} {status}" + per = errors.setdefault(name, {}) + per[key] = per.get(key, 0) + 1 + if endpoint not in live: + continue + mt = self.RE_TS.match(raw) + if not mt: + continue + try: + ts = datetime.strptime(mt.group(1), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + except ValueError: + continue + if last_poll is not None and (ts - last_poll).total_seconds() > gap_limit: + gaps.append((name, f"{last_poll:%H:%M:%S}", (ts - last_poll).total_seconds())) + last_poll = ts + + total_errors = sum(sum(v.values()) for v in errors.values()) + if total_errors > int(self.opt("max_errors")): + yield critical( + self.name, + title=f"{total_errors} node-agent call(s) returned an error", + subject="control-plane", + detail="; ".join(f"{n}: " + ", ".join(f"{k} x{c}" for k, c in sorted(v.items())) + for n, v in sorted(errors.items())), + evidence={"errors": errors}, + note="The control plane asked and got an error rather than an answer, so " + "whatever it decided next it decided on no information. Read with " + "control.node-flap: this is the upstream half of a false offline.", + ) + + if gaps: + worst = sorted(gaps, key=lambda g: -g[2])[:6] + yield warning( + self.name, + title=f"{len(gaps)} gap(s) in the liveness polling, worst {worst[0][2]:.0f}s", + subject="control-plane", + detail="; ".join(f"{n} after {t} ({d:.0f}s)" for n, t, d in worst), + evidence={"gaps": [{"log": n, "after": t, "seconds": round(d)} + for n, t, d in gaps]}, + note="Either the control plane stopped asking or the agent stopped answering. " + "A poll that never happened leaves no other trace, so this is the only " + "place the stall is visible — and it is what precedes a node being " + "declared down.", + ) diff --git a/test/framework/sbtest/detectors/fio.py b/test/framework/sbtest/detectors/fio.py new file mode 100644 index 000000000..650cefd8d --- /dev/null +++ b/test/framework/sbtest/detectors/fio.py @@ -0,0 +1,355 @@ +"""fio-side detectors: I/O errors, silent corruption, and sustained outages. + +The checksum detector is the one that decides whether a run lost data, so its attribution +matters as much as its detection — see VERIFY_LAG_S. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +from ..core import ( + Detector, + Evidence, + Finding, + SkipDetector, + attribute, + attribute_window, + critical, + detector, + warning, +) + +#: How long after a migration ends one of its lost writes may still surface. +#: +#: fio notices a lost write when it next *reads* that block, not when the write was lost, +#: so detection trails the loss by however long the read pattern takes to come round — +#: measured at 3-34s across runs. Attribution has to allow for that: without it, a +#: migration's own losses get filed under "no migration was running", which is exactly how +#: a Completed-but-corrupting migration stayed invisible for a whole analysis pass. +VERIFY_LAG_S = 45.0 + +#: fio's verify failure lines. "bad magic header" is the signature seen in every event so +#: far: a block that never carried an fio header at all, i.e. a *first* write that was lost +#: rather than a stale or torn overwrite. +_VERIFY_RE = re.compile( + r"verify:? (?:bad magic header|header|crc|md5|pattern)|" + r"verify_(?:header|md5|crc)|" + r"got (?:crc|md5) .*expected", + re.IGNORECASE) + +_OFFSET_RE = re.compile(r"offset[= ](\d+)") +#: CRI log prefix: 2026-08-19T22:23:18.994807954Z stderr F +_CRI_TS_RE = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})") + + +def _cri_ts(line: str) -> datetime | None: + m = _CRI_TS_RE.match(line) + if not m: + return None + return datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + + +@detector +class JobError(Detector): + """An fio job that ended with an error. + + Reported per pod with the errno, because the errno is the diagnosis and they mean very + different things: 84 is a failed *verification* (the data was wrong), 121 is a path that + went away under the I/O (the data never arrived). + + The numbers below are **Linux** errno values, written out rather than looked up. The + analysis usually runs somewhere other than the host that produced the run, and the errno + table is platform-specific — 84 is EILSEQ on Linux and EOVERFLOW on macOS, so resolving it + locally silently mislabels the most important code in the set. + """ + + name = "fio.job-error" + summary = "an fio job ended with a non-zero errno" + + ERRNO_HINT = { + 84: "EILSEQ — fio's verify failed: a block read back did not match what was written. " + "This is silent data corruption, not an I/O failure; see fio.checksum for the " + "blocks and the migration they belong to", + 121: "EREMOTEIO — every path to the namespace was inaccessible under the I/O; " + "correlate with ana.cutover-pause / ana.freeze-count on the same window", + 5: "EIO — the target failed the command outright rather than losing the path", + 28: "ENOSPC — capacity, not connectivity", + 110: "ETIMEDOUT — the command was accepted and never completed", + 108: "ESHUTDOWN — the target went away mid-flight", + } + + def defaults(self) -> dict: + return {"ignore_errnos": []} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + jobs = ev.fio_jobs() + if not jobs: + raise SkipDetector("no fio job results") + ignore = {int(x) for x in self.opt("ignore_errnos")} + for j in jobs: + if not j.error or j.error in ignore: + continue + yield critical( + self.name, + title=f"fio job ended in error {j.error}", + subject=j.pod, + detail=self.ERRNO_HINT.get(j.error, ""), + evidence={"errno": j.error, "total_iops": j.total_iops}, + ) + + +@detector +class Checksum(Detector): + """fio read back a block that does not match what it wrote — silent corruption. + + The read *succeeded*, so nothing outside fio's own verification notices: no I/O error, + no kernel message, no control-plane event. This is the most serious thing a run can + find, and it is why the attribution window is generous rather than exact — a lost write + that cannot be tied to a migration is much harder to act on than one that can. + """ + + name = "fio.checksum" + summary = "fio read back data it never wrote (silent corruption), attributed to a migration" + + def defaults(self) -> dict: + return {"verify_lag_s": VERIFY_LAG_S} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + pods = ev.pods() + if not pods: + raise SkipDetector("no fio pods") + lag = timedelta(seconds=float(self.opt("verify_lag_s"))) + migs = ev.migrations() + seen_any_log = False + # subject -> list of (pod, ts, line) + per_subject: dict[str, list[tuple[str, datetime | None, str]]] = {} + + for pod in pods: + for line in ev.fio_log(pod): + seen_any_log = True + if not _VERIFY_RE.search(line): + continue + ts = _cri_ts(line) + m = attribute(migs, ts, lag) if ts else None + per_subject.setdefault(m.name if m else "outside-any-migration", []).append( + (pod, ts, line.strip())) + + if not seen_any_log: + raise SkipDetector("no fio pod logs available") + + for subject, hits in sorted(per_subject.items()): + pods_hit: dict[str, int] = {} + offsets = [] + for pod, _ts, line in hits: + pods_hit[pod] = pods_hit.get(pod, 0) + 1 + mo = _OFFSET_RE.search(line) + if mo: + offsets.append(int(mo.group(1))) + mig = next((m for m in migs if m.name == subject), None) + lagged = "" + if mig and mig.end: + late = [t for _p, t, _l in hits if t and t > mig.end] + if late: + worst = max((t - mig.end).total_seconds() for t in late) + lagged = (f"; {len(late)} of them detected up to {worst:.0f}s after the " + f"migration ended, inside fio's verify backlog") + yield critical( + self.name, + title=f"{len(hits)} block(s) read back with the wrong contents", + subject=subject, + detail=("pods: " + ", ".join(f"{p}={n}" for p, n in sorted(pods_hit.items())) + + (f"; phase={mig.phase}" if mig else "") + lagged), + evidence={"blocks": len(hits), "pods": pods_hit, + "offsets": sorted(offsets)[:32], + "phase": mig.phase if mig else "", + "verify_lag_s": float(self.opt("verify_lag_s"))}, + artifacts=[f"{p}/fio.log" for p in sorted(pods_hit)], + note="The reads succeeded, so this is silent: the volume served data that " + "was never written at those offsets. A Completed migration can do " + "this — phase is not a filter.", + ) + + +@dataclass(frozen=True) +class _Window: + """One stretch of a pod doing no I/O. + + `recovered` is the whole distinction between a freeze and a loss, so it travels with the + window rather than being re-derived from where it sits in the series. + """ + + seconds: int + from_offset_s: int + to_offset_s: int + migration: str + recovered: bool + + +@detector +class Outage(Detector): + """A sustained window where a pod did no I/O at all. + + Distinct from a checksum failure and from a job error: the I/O neither failed nor + returned wrong data, it stopped. Short dips are normal during a cutover, so only runs + at or above `min_seconds` count. + + A stopped volume is reported as one of two things, because they are not the same defect: + + * an I/O **loss** is a gap that was still open when fio stopped. I/O the volume was + supposed to accept and never did. + * an I/O **freeze** is a gap that recovered. Every write the application issued was + eventually taken, so nothing was lost; what the freeze measures is how long an + application had to survive with the volume gone. A cutover is a freeze by design, + which is why its *length* is the thing under test and its existence is not. + + Both are CRITICAL — a volume that stops for longer than a cutover should cost is a + defect either way — but a run that lost I/O and a run that stalled and recovered need + different next steps, and a report that calls both "outage" hands the reader that work. + + Reported per pod and per kind rather than per window. A pod that stalls repeatedly stalls + *hundreds* of times in a soak — one run produced 781 qualifying windows at a 10s + threshold — and a report with 781 entries for one symptom is a report nobody reads to the + end. The worst windows are named, the rest are counted, and the total downtime is what the + finding leads with, because "18% of the run did no I/O" is the fact that decides anything. + """ + + name = "fio.outage" + summary = "a pod's I/O stopped for longer than a cutover should cost — a freeze if it came back, a loss if it did not" # noqa: E501 + + def defaults(self) -> dict: + return {"min_seconds": 30, "iops_floor": 0.0, "name_worst": 6} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + pods = ev.pods() + if not pods: + raise SkipDetector("no fio pods") + floor = float(self.opt("iops_floor")) + need = int(self.opt("min_seconds")) + migs = ev.migrations() + saw_series = False + + for pod in pods: + series = ev.fio_timeseries(pod) + if not series: + continue + saw_series = True + wall_at = {x.offset_s: x.wall for x in series} + windows: list[_Window] = [] + run_start: int | None = None + prev_off: int | None = None + # None is a sentinel that closes a run still open at the end of the series. + for s in [*series, None]: + down = s is not None and s.total_iops <= floor + if down: + assert s is not None # implied by `down`; stated for the type checker + if run_start is None: + run_start = s.offset_s + elif run_start is not None: + # A run the sentinel closes was still down when fio stopped: nothing + # observed it come back, so it is a loss rather than a freeze. + recovered = s is not None + end = s.offset_s if s is not None else (prev_off or run_start) + dur = end - run_start + if dur >= need: + w0, w1 = wall_at.get(run_start), wall_at.get(end) + mig = attribute_window(migs, w0, w1 or w0) if w0 else None + windows.append(_Window(dur, run_start, end, + mig.name if mig else "", recovered)) + run_start = None + if s is not None: + prev_off = s.offset_s + + if not windows: + continue + span = series[-1].offset_s - series[0].offset_s or 1 + # Losses first: both fail the run, but a gap that never closed outranks one that + # did, and the order findings are emitted in is the order they are read in. + for kind in ("loss", "freeze"): + of_kind = [w for w in windows if (w.recovered != (kind == "loss"))] + if of_kind: + yield self._finding(pod, kind, of_kind, span) + if not saw_series: + raise SkipDetector("no fio time series available") + + def _finding(self, pod: str, kind: str, windows: list[_Window], span: int) -> Finding: + show = int(self.opt("name_worst")) + windows = sorted(windows, key=lambda w: -w.seconds) + downtime = sum(w.seconds for w in windows) + named = ", ".join( + f"{w.seconds}s at +{w.from_offset_s}s" + + (f" ({w.migration})" if w.migration else " (no migration)") + for w in windows[:show]) + during = sorted({w.migration for w in windows if w.migration}) + if kind == "loss": + title = (f"I/O LOSS: stopped {len(windows)}x for {downtime}s total " + f"({downtime / span:.0%} of the run) and never resumed, " + f"worst {windows[0].seconds}s") + meaning = ("I/O was still dead when fio stopped, so these are writes the volume " + "was supposed to accept and never did. ") + else: + title = (f"I/O froze {len(windows)}x for {downtime}s total " + f"({downtime / span:.0%} of the run), worst {windows[0].seconds}s, " + "all recovered") + meaning = ("Every freeze recovered, so no I/O was lost — what this measures is " + "how long an application had to survive with the volume gone. ") + return critical( + self.name, + title=title, + subject=pod, + detail=named + (f", and {len(windows) - show} more" if len(windows) > show else ""), + evidence={"kind": kind, "windows": len(windows), "downtime_s": downtime, + "worst_s": windows[0].seconds, + "fraction_of_run": round(downtime / span, 4), + "migrations": during, + "worst_windows": [{"seconds": w.seconds, + "from_offset_s": w.from_offset_s, + "to_offset_s": w.to_offset_s, + "migration": w.migration, + "recovered": w.recovered} + for w in windows[:show]]}, + note=meaning + ( + "Windows are attributed to the migration they share the most seconds with; " + f"{len(during)} migration(s) are implicated here." + if during else + "None of these windows overlap a migration, so the cause is " + "elsewhere — check the fabric and the node's own health."), + ) + + +@detector +class Throughput(Detector): + """A pod whose average IOPS is far below its peers. + + A weak signal on its own, which is why it is a WARNING: a pod on a shared subsystem that + is being migrated repeatedly will legitimately lag. It earns its place next to the other + detectors — a pod that is both slow and the subject of an ANA finding is a different + story from one that is merely slow. + """ + + name = "fio.throughput-outlier" + summary = "a pod whose average IOPS is far below the run's median" + + def defaults(self) -> dict: + return {"min_fraction_of_median": 0.5, "min_pods": 4} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + jobs = [j for j in ev.fio_jobs() if j.total_iops > 0] + if len(jobs) < int(self.opt("min_pods")): + raise SkipDetector(f"needs at least {self.opt('min_pods')} pods with I/O, " + f"got {len(jobs)}") + vals = sorted(j.total_iops for j in jobs) + median = vals[len(vals) // 2] + frac = float(self.opt("min_fraction_of_median")) + for j in jobs: + if j.total_iops < median * frac: + yield warning( + self.name, + title=f"{j.total_iops:.0f} IOPS against a median of {median:.0f}", + subject=j.pod, + evidence={"total_iops": j.total_iops, "median_iops": median, + "fraction": round(j.total_iops / median, 3)}, + ) diff --git a/test/framework/sbtest/detectors/kernel.py b/test/framework/sbtest/detectors/kernel.py new file mode 100644 index 000000000..4ee85370e --- /dev/null +++ b/test/framework/sbtest/detectors/kernel.py @@ -0,0 +1,637 @@ +"""Kernel-side detectors, from dmesg. + +dmesg is the only source that says what the *host* did about a fabric event, and it turns +out to carry things nothing else does. + +The headline is a severity ladder. When every path to a namespace goes away, the kernel does +not simply fail I/O — it queues, waits, and only fails once a timeout expires: + + all paths inaccessible + -> "block nvmeXnY: no usable path - requeuing I/O" queued; the application waits + -> "nvme nvmeN: failfast expired" fast_io_fail_tmo elapsed + -> "block nvmeXnY: no available path - failing I/O" errors reach the application + -> "XFS (nvmeXnY): log I/O error -5" + -> "XFS (nvmeXnY): Filesystem has been shut down" the volume needs repair + +Every stage was observed across the archived runs, and the ladder is monotone in severity: +one run reached only the first rung (163 requeues, no failures, no filesystem damage) while +a later one reached the last (221 requeues, 30 failures, 20 filesystem shutdowns). That makes +the rung reached a much better statement of blast radius than any count of ANA samples, and +it identifies the knob that decides it: **fast_io_fail_tmo**. A pause shorter than it is +absorbed; a pause longer than it becomes application-visible I/O errors and then filesystem +damage. + +The second thing dmesg alone shows is that leaked controllers **outlive their cluster**. In +one run, 90% of the kernel's NVMe traffic was controllers retrying subsystems belonging to +two clusters that had already been destroyed and reinstalled. An NQN names its cluster, so +that is decidable without any threshold — see ForeignCluster. + +**Everything here is bounded to the run's window**, and that is not a detail. dmesg is a ring +buffer covering hours, so it routinely holds the previous runs' damage and a cluster's worth +of leftover debris. Counting that against this run means a clean run inherits its +predecessor's mess — and, worse, a genuinely broken run hides inside inherited noise. Events +dated before the run are reported separately as Attribution.PRE_EXISTING, which makes the +verdict INCONCLUSIVE rather than FAIL: the action is to clean the environment and run again. + +Timestamps are read from either `--time-format=iso` (which carries an offset, so the +comparison against a UTC run window is sound) or `-T` (local time, no offset — assumed to be +the run's own timezone, and flagged as such). Events with no usable timestamp are +Attribution.UNKNOWN and still count, because "I cannot date this" must not become "not our +problem". Individual migrations are never blamed: the window is the run, not the migration. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Callable, Iterable, Iterator +from datetime import UTC, datetime + +from ..core import ( + Attribution, + Detector, + Evidence, + Finding, + Severity, + SkipDetector, + critical, + detector, + info, + warning, +) + +#: `[Thu Aug 20 05:46:57 2026] ...` — dmesg -T, host-local, no offset. +_TS_CTIME = re.compile(r"^\[(\w{3} \w{3}\s+\d+ \d{2}:\d{2}:\d{2} \d{4})\]\s*(.*)$") +#: `2026-08-20T05:46:57,123456+00:00 ...` — dmesg --time-format=iso, with an offset. +_TS_ISO = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})[,.]\d+([+-]\d{2}:?\d{2})?\s*(.*)$") + +#: Which logs these detectors read. dmesg-.txt by convention. +DEFAULT_LOG_GLOBS = ["dmesg-*"] + + +def _parse_line(raw: str) -> tuple[datetime | None, str]: + """(timestamp, message) from a dmesg line in either format.""" + line = raw.rstrip("\n") + m = _TS_ISO.match(line) + if m: + stamp, offset, msg = m.group(1), m.group(2), m.group(3) + try: + return datetime.fromisoformat(stamp + (offset or "+00:00")), msg + except ValueError: + return None, msg + m = _TS_CTIME.match(line) + if m: + try: + # No offset in this format. Treated as UTC, which is what these hosts run and + # what makes the window comparison meaningful; a host in another zone shifts + # events across the boundary, which is why the ISO form is collected now. + return datetime.strptime(m.group(1), "%a %b %d %H:%M:%S %Y").replace( + tzinfo=UTC), m.group(2) + except ValueError: + return None, m.group(2) + return None, line.strip() + + +def _severity(sev: Severity) -> Callable[..., Finding]: + """The finding constructor for a severity, so a detector can vary it by attribution. + + Which it must: the same observation is a defect when the run caused it and a hygiene note + when the run inherited it, and that difference is severity, not wording. + """ + return {Severity.CRITICAL: critical, Severity.WARNING: warning}.get(sev, info) + + +class _Window: + """Places a dmesg event inside or before the run, and counts each bucket. + + The whole point of this class is that "before the run" is tracked separately rather than + filtered away: inherited damage is worth reporting loudly, just not as this run's fault. + """ + + def __init__(self, ev: Evidence) -> None: + self.start, self.end = ev.run_window() + + def attribute(self, ts: datetime | None) -> Attribution: + if ts is None or self.start is None: + return Attribution.UNKNOWN + return Attribution.PRE_EXISTING if ts < self.start else Attribution.RUN + + @property + def described(self) -> str: + if not self.start: + return "the run window is unknown, so nothing could be dated" + return f"run window {self.start:%b %d %H:%M:%S}..{self.end:%H:%M:%S} UTC" if self.end \ + else f"run started {self.start:%b %d %H:%M:%S} UTC" + + +def _dmesg_lines(ev: Evidence, globs: list[str]) -> Iterator[tuple[str, datetime | None, str]]: + """(log name, timestamp or None, message) for every dmesg line available.""" + names = [n for n in ev.container_logs() if any(fnmatch.fnmatch(n, g) for g in globs)] + for name in names: + for raw in ev.container_log(name): + ts, msg = _parse_line(raw) + yield name, ts, msg + + +def _span(times: list[datetime]) -> str: + if not times: + return "" + lo, hi = min(times), max(times) + return (f"{lo:%b %d %H:%M:%S}" if lo == hi + else f"{lo:%b %d %H:%M:%S}..{hi:%H:%M:%S} host-local") + + +@detector +class PathLoss(Detector): + """How far the kernel got up the path-loss ladder, per namespace device. + + Reports the worst rung reached, because the rungs mean different things: + + * **requeued only** — the pause was absorbed. The application saw latency, not errors. + Worth knowing (it means paths went away at all) but not a failure. + * **failfast expired** — the pause outlasted `fast_io_fail_tmo`. This is the boundary; + past it the kernel stops protecting the application. + * **failing I/O** — errors reached the application. Data loss is now possible and fio + will have noticed too (correlate with fio.job-error / fio.checksum). + + This is stronger evidence than ANA sampling for the same event: it is what the kernel + actually did, not what a sampler happened to catch, so it cannot miss a window shorter + than the sampling interval. + """ + + name = "kernel.path-loss" + summary = "how far the kernel got up the path-loss ladder (requeue -> failfast -> fail I/O)" + + RE_REQUEUE = re.compile(r"block (\S+): no usable path - requeuing I/O") + RE_FAILING = re.compile(r"block (\S+): no available path - failing I/O") + RE_FAILFAST = re.compile(r"(nvme\d+): failfast expired") + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, + # Requeues are expected during a cutover; this bounds how many is normal. + "max_requeues": 0, + # Any failed I/O is a failure — the application saw an error. + "max_failing": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + win = _Window(ev) + # kind -> attribution -> device -> times + seen: dict[str, dict[Attribution, dict[str, list[datetime]]]] = {} + saw_any = False + + for _log, ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + for rx, kind in ((self.RE_FAILING, "failing"), (self.RE_REQUEUE, "requeue"), + (self.RE_FAILFAST, "failfast")): + m = rx.search(msg) + if not m: + continue + bucket = seen.setdefault(kind, {}).setdefault(win.attribute(ts), {}) + stamps = bucket.setdefault(m.group(1), []) + if ts: + stamps.append(ts) + break + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + def count(kind: str, attr: Attribution) -> int: + return sum(len(v) or 1 for v in seen.get(kind, {}).get(attr, {}).values()) + + for attr in (Attribution.RUN, Attribution.UNKNOWN, Attribution.PRE_EXISTING): + devs_fail = seen.get("failing", {}).get(attr, {}) + n_fail = count("failing", attr) + n_req = count("requeue", attr) + n_ff = count("failfast", attr) + if not (devs_fail or n_req or n_ff): + continue + inherited = attr is Attribution.PRE_EXISTING + times = [t for d in seen.values() for v in d.get(attr, {}).values() for t in v] + + if devs_fail and n_fail > int(self.opt("max_failing")): + # Pre-existing damage is reported but never fails the run: I/O the kernel + # gave up on before this run started is the previous run's story. + make = _severity(Severity.WARNING if inherited else Severity.CRITICAL) + yield make( + self.name, + title=(f"the kernel failed I/O on {len(devs_fail)} device(s) for want of " + f"a path" + (" before this run began" if inherited else "")), + subject="fabric", + detail=(f"{n_fail} occurrence(s) on {', '.join(sorted(devs_fail))}" + + (f"; {_span(times)}" if times else "") + + f"; {win.described}"), + evidence={"failing_io": n_fail, "devices": sorted(devs_fail), + "requeues": n_req, "failfast": n_ff, + "attribution": str(attr)}, + artifacts=sorted(set(self.opt("logs"))), + attribution=attr, + note=("Inherited from before the run — worth cleaning up, but it says " + "nothing about this run's code." + if inherited else + "This is the rung past which the kernel stops protecting the " + "application: queued I/O is given up and the error returned. " + "Filesystem damage follows — see kernel.filesystem-shutdown — and " + "fio will have seen it too."), + ) + + if n_ff: + yield _severity(Severity.INFO if inherited else Severity.WARNING)( + self.name, + title=(f"fast_io_fail_tmo expired {n_ff} time(s)" + + (" before this run began" if inherited else "")), + subject="fabric", + detail=f"{len(seen.get('failfast', {}).get(attr, {}))} controller(s)" + + (f"; {_span(times)}" if times else ""), + evidence={"failfast": n_ff, "attribution": str(attr)}, + attribution=attr, + note=("" if inherited else + "The outage outlasted fast_io_fail_tmo, the knob that decides " + "whether an all-paths-inaccessible window is absorbed or becomes " + "application-visible. A cutover pause longer than this value " + "cannot be survived by queueing."), + ) + + if n_req > int(self.opt("max_requeues")): + yield _severity(Severity.INFO if inherited else Severity.WARNING)( + self.name, + title=(f"the kernel had no usable path on " + f"{len(seen.get('requeue', {}).get(attr, {}))} device(s)" + + (" before this run began" if inherited else "")), + subject="fabric", + detail=f"{n_req} requeue(s)" + (f"; {_span(times)}" if times else ""), + evidence={"requeues": n_req, "attribution": str(attr)}, + attribution=attr, + note=("" if inherited else + "I/O was queued rather than failed, so the application survived — " + "but every path to these namespaces was gone. This is the kernel's " + "own record of the window ana.freeze-count samples for, and it " + "cannot miss one shorter than the sampling interval."), + ) + + +@detector +class FilesystemShutdown(Detector): + """A filesystem that took itself offline because its log I/O failed. + + The end of the path-loss ladder, and the most consequential thing a run can leave behind: + the volume is unusable until it is unmounted and repaired, which no amount of retrying + fixes. Separate from kernel.path-loss because it is the *consequence* — someone asking + "did this run damage a filesystem" wants exactly this and nothing else. + """ + + name = "kernel.filesystem-shutdown" + summary = "XFS/ext4 shut down or went read-only after failed log I/O" + + PATTERNS = ( + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*(?:has been shut down|Filesystem has been shut down)"), + "filesystem shut down"), + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*log I/O error"), "log I/O error"), + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*(?:Remounting filesystem read-only|" + r"remounting filesystem read-only)"), "remounted read-only"), + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*metadata I/O error"), "metadata I/O error"), + ) + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + win = _Window(ev) + # attribution -> device -> kinds -> times + hits: dict[Attribution, dict[str, dict[str, list[datetime]]]] = {} + saw_any = False + for _log, ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + for rx, kind in self.PATTERNS: + m = rx.search(msg) + if m: + slot = hits.setdefault(win.attribute(ts), {}).setdefault( + m.group(2), {}).setdefault(kind, []) + if ts: + slot.append(ts) + break + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + for attr, devices in hits.items(): + inherited = attr is Attribution.PRE_EXISTING + shutdown = sorted(d for d, k in devices.items() if "filesystem shut down" in k) + times = [t for k in devices.values() for v in k.values() for t in v] + if shutdown: + # Pre-existing damage is a hygiene warning, not this run's failure: a + # filesystem that died before the run started is the previous run's evidence. + # It only invalidates *this* run if the run went on to use that volume, which + # dmesg alone cannot establish — nvme.dirty-start is the check that can. + yield _severity(Severity.WARNING if inherited else Severity.CRITICAL)( + self.name, + title=(f"{len(shutdown)} filesystem(s) shut down after failed log I/O" + + (" before this run began" if inherited else "")), + subject="fabric", + detail=(", ".join(shutdown) + (f"; {_span(times)}" if times else "") + + f"; {win.described}"), + evidence={"devices": shutdown, "attribution": str(attr), + "kinds": {d: sorted(k) for d, k in sorted(devices.items())}}, + attribution=attr, + note=("Left over from earlier activity on these hosts. Worth cleaning up " + "— the volume needs unmount and repair — but not evidence about " + "this run." + if inherited else + "The volume is unusable until it is unmounted and repaired; " + "retrying does not recover it. This is the end of the path-loss " + "ladder — see kernel.path-loss for how it got here."), + ) + elif devices: + yield _severity(Severity.INFO if inherited else Severity.WARNING)( + self.name, + title=(f"filesystem I/O errors on {len(devices)} device(s), no shutdown" + + (" (before this run)" if inherited else "")), + subject="fabric", + detail=", ".join(f"{d}: {', '.join(sorted(k))}" + for d, k in sorted(devices.items())), + evidence={"kinds": {d: sorted(k) for d, k in sorted(devices.items())}, + "attribution": str(attr)}, + attribution=attr, + ) + + +@detector +class ForeignCluster(Detector): + """A controller retrying a subsystem that belongs to a cluster which no longer exists. + + An NQN names its cluster, so this needs no threshold and no topology knowledge: if the + run is against cluster A and the kernel is retrying NQNs for cluster B, those controllers + are leaked, full stop. + + This is the sharpest form of "controllers not disappearing". Across the archived runs the + leak survived not just the migration that made it but the cluster's *destruction and + reinstallation*: in one run, 90% of the kernel's NVMe log traffic was retries against two + clusters that had already been torn down. Nothing on the host ever removes them, and + because the kernel resets its reconnect counter on a partial reconnect, they are + effectively immortal until the node reboots. + + **Reported as hygiene, never as a run failure.** By construction these belong to a cluster + that no longer exists, so they cannot make a migration of *this* cluster's subsystems fail + — they are noise in the logs and wasted work on the host, which is a real thing to fix and + a bad thing to fail a build over. The leak that does invalidate a run is a leak on the + *live* cluster's subsystems: see nvme.dirty-start. + """ + + name = "nvme.foreign-cluster" + summary = "controllers retrying subsystems of a cluster that is no longer the live one" + + RE_NQN_CLUSTER = re.compile(r"simplyblock:([0-9a-f]{8}-[0-9a-f-]{27}):") + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, "max_lines": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + current = ev.cluster_uuid() + if not current: + raise SkipDetector("the run's cluster uuid is unknown, so a foreign NQN " + "cannot be told from the live one") + per_cluster: dict[str, int] = {} + saw_any = False + for _log, _ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + m = self.RE_NQN_CLUSTER.search(msg) + if m: + per_cluster[m.group(1)] = per_cluster.get(m.group(1), 0) + 1 + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + foreign = {c: n for c, n in per_cluster.items() if c != current} + if not foreign or sum(foreign.values()) <= int(self.opt("max_lines")): + return + total = sum(per_cluster.values()) or 1 + share = sum(foreign.values()) / total + yield warning( + self.name, + title=f"{len(foreign)} dead cluster(s) still being retried by this host", + subject="fabric", + detail=("; ".join(f"{c[:8]}={n} lines" for c, n in + sorted(foreign.items(), key=lambda x: -x[1])) + + f"; live cluster {current[:8]}={per_cluster.get(current, 0)} " + f"({share:.0%} of NVMe log traffic is for dead clusters)"), + evidence={"live_cluster": current, "foreign": foreign, + "foreign_share": round(share, 3)}, + attribution=Attribution.PRE_EXISTING, + note="These controllers outlived the cluster they belong to, so nothing will " + "ever answer them. They cost a reconnect every ctrl_loss_tmo window and " + "bury the live cluster's messages in the log, but they cannot affect a " + "migration of this cluster's subsystems — so this is hygiene, not a " + "verdict. Only a disconnect or a node reboot clears them.", + ) + + +@detector +class ControllerChurn(Detector): + """Controllers created versus removed — "they never disappear", counted. + + A run that connects paths and tears them down again nets out. One that leaks shows a + positive delta, and a reconnect storm shows as a retry count out of all proportion to the + number of controllers involved. + + Both halves are needed. The delta alone misses a leak whose controller was created before + the dmesg ring wrapped; the retry rate alone cannot say whether the retries belong to one + doomed controller or fifty healthy ones. + """ + + name = "nvme.controller-churn" + summary = "controllers created vs removed, and how hard the survivors are retrying" + + RE_NEW = re.compile(r"nvme(\d+): new ctrl|nvme(\d+): creating \d+ I/O queues") + RE_GONE = re.compile(r"nvme(\d+): Removing ctrl") + RE_RETRY = re.compile(r"nvme(\d+): Failed reconnect attempt (\d+)") + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, + # A positive delta this large means controllers are accumulating. + "max_net_created": 4, + # A single controller retrying this many times is not coming back. + "max_retries_per_controller": 100} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + created: set[str] = set() + removed: set[str] = set() + worst_retry: dict[str, int] = {} + saw_any = False + for _log, _ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + m = self.RE_NEW.search(msg) + if m: + created.add(m.group(1) or m.group(2)) + m = self.RE_GONE.search(msg) + if m: + removed.add(m.group(1)) + m = self.RE_RETRY.search(msg) + if m: + ctrl, n = m.group(1), int(m.group(2)) + worst_retry[ctrl] = max(worst_retry.get(ctrl, 0), n) + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + net = len(created - removed) + if net > int(self.opt("max_net_created")): + yield warning( + self.name, + title=f"{net} controller(s) created and never removed", + subject="fabric", + detail=f"created={len(created)} removed={len(removed)}; " + f"never removed: {', '.join(sorted(created - removed)[:16])}", + evidence={"created": len(created), "removed": len(removed), "net": net, + "never_removed": sorted(created - removed)}, + note="dmesg is a ring buffer, so a controller created before it wrapped will " + "look un-created rather than un-removed; the delta is a floor on the " + "leak, not the whole of it.", + ) + + stuck = {c: n for c, n in worst_retry.items() + if n > int(self.opt("max_retries_per_controller"))} + if stuck: + yield warning( + self.name, + title=f"{len(stuck)} controller(s) retrying without ever succeeding", + subject="fabric", + detail="; ".join(f"nvme{c}={n} attempts" for c, n in + sorted(stuck.items(), key=lambda x: -x[1])[:12]), + evidence={"controllers": {f"nvme{c}": n for c, n in sorted(stuck.items())}, + "threshold": int(self.opt("max_retries_per_controller"))}, + note="The reconnect counter resets on a partial reconnect, so a high value " + "means the target is actively refusing this host rather than being " + "briefly unreachable. Check nvme.foreign-cluster: the usual cause is a " + "controller whose subsystem no longer exists.", + ) + if not stuck and net <= int(self.opt("max_net_created")): + yield info(self.name, title=f"{len(created)} created / {len(removed)} removed", + subject="fabric", + evidence={"created": len(created), "removed": len(removed)}) + + +@detector +class FabricErrors(Detector): + """Fabric-level errors that indicate instability short of losing a path. + + None of these is a failure on its own — they are the texture around one, and they are + worth surfacing because they say *where* the fabric hurt: a socket that will not + establish is a different problem from a controller that connects and then cannot be + configured. + """ + + name = "kernel.fabric-errors" + summary = "connect/reset/timeout errors from the NVMe fabric layer" + + PATTERNS = ( + (re.compile(r"starting error recovery"), "error recovery started"), + (re.compile(r"Property Set error"), "property set failed (controller config)"), + (re.compile(r"Identify Descriptors failed"), "identify descriptors failed"), + (re.compile(r"failed to connect socket: -(\d+)"), "socket connect failed"), + (re.compile(r"failed to connect queue: \d+ ret=(\d+)"), "queue connect failed"), + (re.compile(r"queue \d+ socket state (\d+)"), "socket in unexpected state"), + (re.compile(r"I/O \d+ (?:\(\S+\) )?QID \d+ timeout"), "I/O timeout"), + (re.compile(r"Connect Invalid Data Parameter"), "connect refused: no such subsystem"), + (re.compile(r"is not allowed, hostnqn"), "connect refused: host not in allow-list"), + (re.compile(r"rescanning namespaces"), "namespace rescan"), + ) + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, + #: kind -> minimum count before it is worth reporting. A rescan or a refused + #: connect happens in ones and twos on any healthy run. + "min_counts": {"namespace rescan": 50, + "connect refused: no such subsystem": 50, + "connect refused: host not in allow-list": 50, + "socket connect failed": 50, + "queue connect failed": 50}, + "default_min_count": 1} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + counts: dict[str, int] = {} + saw_any = False + for _log, _ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + for rx, kind in self.PATTERNS: + if rx.search(msg): + counts[kind] = counts.get(kind, 0) + 1 + break + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + mins = dict(self.opt("min_counts") or {}) + floor = int(self.opt("default_min_count")) + reportable = {k: v for k, v in counts.items() if v >= mins.get(k, floor)} + if not reportable: + return + yield warning( + self.name, + title=f"{sum(reportable.values())} fabric error(s) across {len(reportable)} kind(s)", + subject="fabric", + detail="; ".join(f"{k}={v}" for k, v in + sorted(reportable.items(), key=lambda x: -x[1])), + evidence={"counts": reportable, "all_counts": counts}, + note="Texture rather than a verdict, and spanning whatever the ring buffer held " + "rather than only this run: read it next to kernel.path-loss to see whether " + "the fabric merely wobbled or actually lost a path.", + ) + + +@detector +class DirtyStart(Detector): + """The fabric was already carrying blocking debris for the **live** cluster at setup. + + This is the one pre-existing condition that forfeits a run, and it is worth being precise + about why. A controller that is live and serves no namespace makes VerifyMigrationPaths + reject the migration of its subsystem — permanently, because nothing removes it. If that + is already true when the run starts, then migrations that should have passed will fail, + and the run's completion rate measures the mess it inherited rather than the code under + test. A 13-of-46 result says nothing in that state. + + Everything about the qualification matters: + + * **live cluster only.** Debris for a destroyed cluster cannot block this cluster's + subsystems; that is nvme.foreign-cluster's hygiene warning, not a forfeit. + * **at setup, not at the end.** Debris the run *created* is the run's own finding — see + nvme.stale-controllers — and failing it is correct. + * **blocking shapes only.** A controller stuck in "connecting" is indistinguishable from + a normal HA reconnect in a snapshot, so it does not qualify. + + Requires the nvme.snapshot component's pre-run snapshot; without it there is nothing to + judge and the detector says so. + """ + + name = "nvme.dirty-start" + summary = ("the fabric already held blocking debris for the live cluster when the run " + "started, so the run's results cannot be trusted") + + def defaults(self) -> dict: + return {"max_blocking": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + pre = getattr(ev, "nvme_controllers_pre", None) + ctrls = pre() if callable(pre) else [] + if not ctrls: + raise SkipDetector("no pre-run fabric snapshot (enable the nvme.snapshot " + "component to judge the state the run started from)") + cluster = ev.cluster_uuid() + if not cluster: + raise SkipDetector("the run's cluster uuid is unknown, so debris for it cannot " + "be told from debris for a dead one") + + blocking = [c for c in ctrls + if cluster in c.nqn and c.state == "live" and c.serves_nothing] + if len(blocking) <= int(self.opt("max_blocking")): + return + by_node: dict[str, list[str]] = {} + for c in blocking: + by_node.setdefault(c.node, []).append(f"{c.name}@{c.address}") + yield critical( + self.name, + title=f"{len(blocking)} blocking controller(s) for the live cluster before the " + f"run started", + subject="environment", + detail="; ".join(f"{n}: {', '.join(sorted(v))}" for n, v in sorted(by_node.items())), + evidence={"count": len(blocking), "live_cluster": cluster, + "per_node": {n: sorted(v) for n, v in by_node.items()}}, + attribution=Attribution.PRE_EXISTING, + note="Each of these makes the pre-cutover path check reject its subsystem, so " + "migrations that should pass will fail and the completion rate measures the " + "inherited mess rather than the code. Clear the fabric and run again; do " + "not read this run's migration results.", + ) diff --git a/test/framework/sbtest/detectors/logs.py b/test/framework/sbtest/detectors/logs.py new file mode 100644 index 000000000..8a5b121ae --- /dev/null +++ b/test/framework/sbtest/detectors/logs.py @@ -0,0 +1,206 @@ +"""Log-pattern detectors — user-definable checks over any collected log. + +This is the extension point for "whatever the user wants". A pattern entry is data, so a +new check costs a config block rather than code: + + detectors: + logs.pattern: + patterns: + - id: spdk.undrained-transfer + regex: "still have outstanding io" + logs: ["spdk-*"] + severity: critical + min_count: 1 + note: "the batch transfer started before in-flight I/O was drained" + +The bundled catalogue below is what past runs turned out to care about, and it doubles as +worked examples. Everything in it is overridable: `patterns` replaces the catalogue, +`extra_patterns` appends to it. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any + +from ..core import Detector, Evidence, Finding, Severity, SkipDetector, detector + + +@dataclass +class Pattern: + """One log check. `id` is the finding subject, so keep it stable and dotted.""" + + id: str + regex: str + #: Glob(s) over the available log names. "spdk-*" covers every storage node. + logs: list[str] = field(default_factory=lambda: ["*"]) + severity: str = "warning" + #: Fire only at or above this many matches. The default of 1 suits a line that should + #: never appear; raise it for a line that is normal in small numbers. + min_count: int = 1 + note: str = "" + #: Cap on example lines kept in the finding, to keep reports readable. + examples: int = 3 + + def compiled(self) -> re.Pattern: + return re.compile(self.regex) + + def matches_log(self, name: str) -> bool: + return any(fnmatch.fnmatch(name, g) for g in self.logs) + + +#: Patterns worth having by default. Each one was the visible symptom of a real defect. +CATALOGUE: list[Pattern] = [ + Pattern( + id="spdk.undrained-transfer", + regex=r"still have outstanding io|Task transfer timeout with outstanding", + logs=["spdk-4*", "spdk-*"], + severity="critical", + note="The batch delta copy started while I/O was still in flight, so the transfer " + "failed and the control plane retried it. Each retry replays a non-idempotent " + "step against a source that has been serving writes in between — this is the " + "line that accompanies silent write loss. Correlate with ana.freeze-count.", + ), + Pattern( + id="spdk.migration-subtask-failed", + regex=r"Sub task failed for migration of lvol", + logs=["spdk-*"], + severity="critical", + note="One member of a batch migration failed its transfer; the group is retried or " + "abandoned as a whole.", + ), + Pattern( + id="nvme.host-not-allowed", + regex=r"does not allow host .* to connect at this address", + logs=["spdk-*"], + severity="warning", + min_count=50, + note="A host is retrying an endpoint whose allow-list no longer includes it — the " + "signature of a leaked controller nobody disconnected. Volume, not presence, " + "is the signal: a steady rate means a reconnect storm.", + ), + Pattern( + id="nvme.write-to-readonly", + regex=r"WRITE TO RO RANGE", + logs=["spdk-*"], + severity="warning", + min_count=1, + note="A write reached a range the target considers read-only, which during a " + "migration means it landed on a copy that was already frozen.", + ), + Pattern( + id="operator.path-validation-failed", + regex=r"NVMe path validation failed", + logs=["operator*"], + severity="warning", + note="The pre-cutover check refused a migration. Persistent failures for one " + "subsystem usually mean a leaked controller the check keeps reporting — see " + "nvme.stale-controllers.", + ), + Pattern( + id="operator.migration-group-stuck", + regex=r"is already past pre-create|is not active \(status=cancelled\)", + logs=["operator*"], + severity="warning", + note="The operator and the control plane disagree about a migration group's state; " + "the group needs /continue or an explicit cancel.", + ), + Pattern( + id="kernel.controller-reconnect-loop", + regex=r"Failed reconnect attempt (\d{3,})", + logs=["dmesg-*"], + severity="warning", + note="A controller has retried hundreds of times. The counter resets on a partial " + "reconnect, so a high number means it is being refused rather than merely " + "waiting.", + ), +] + + +@detector +class LogPattern(Detector): + """Count regex hits across collected logs and report the ones over threshold. + + One pass per log, all patterns evaluated per line, because these logs are tens of + megabytes and reading them once per pattern is what makes the difference between a check + that runs and one that gets disabled. + """ + + name = "logs.pattern" + summary = "configurable regex checks over any collected log (ships a catalogue)" + + def defaults(self) -> dict: + return {"patterns": None, "extra_patterns": [], "logs": None} + + def _patterns(self) -> list[Pattern]: + given = self.opt("patterns") + pats = [self._as_pattern(p) for p in given] if given is not None else list(CATALOGUE) + pats += [self._as_pattern(p) for p in (self.opt("extra_patterns") or [])] + return pats + + @staticmethod + def _as_pattern(p: Any) -> Pattern: + if isinstance(p, Pattern): + return p + if not isinstance(p, dict): + raise ValueError(f"pattern must be a mapping, got {type(p).__name__}") + unknown = set(p) - set(Pattern.__dataclass_fields__) + if unknown: + raise ValueError(f"pattern {p.get('id', '?')!r}: unknown key(s) {sorted(unknown)}") + return Pattern(**p) + + def detect(self, ev: Evidence) -> Iterable[Finding]: + available = ev.container_logs() + if not available: + raise SkipDetector("no container logs collected in this run") + pats = self._patterns() + only = self.opt("logs") + if only: + available = [n for n in available if any(fnmatch.fnmatch(n, g) for g in only)] + + # (pattern id, log name) -> [count, examples] + hits: dict[tuple[str, str], list] = {} + for log_name in available: + active = [(p, p.compiled()) for p in pats if p.matches_log(log_name)] + if not active: + continue + for line in ev.container_log(log_name): + for p, rx in active: + if rx.search(line): + slot = hits.setdefault((p.id, log_name), [0, []]) + slot[0] += 1 + if len(slot[1]) < p.examples: + slot[1].append(line.strip()[:220]) + + by_id = {p.id: p for p in pats} + # Aggregate across logs per pattern, but keep the per-log breakdown: "which node" + # is usually the first question, and for the undrained-transfer line it was the + # answer (the node stuck in the retry loop). + per_pattern: dict[str, dict[str, int]] = {} + examples: dict[str, list[str]] = {} + for (pid, log_name), (count, ex) in hits.items(): + per_pattern.setdefault(pid, {})[log_name] = count + examples.setdefault(pid, []).extend(ex) + + for pid, counts in sorted(per_pattern.items()): + p = by_id[pid] + total = sum(counts.values()) + if total < p.min_count: + continue + sev = {"critical": Severity.CRITICAL, "warning": Severity.WARNING}.get( + p.severity.lower(), Severity.INFO) + yield Finding( + detector=self.name, + severity=sev, + title=f"{total} match(es) for {pid}", + subject=pid, + detail="; ".join(f"{k}={v}" for k, v in sorted(counts.items(), key=lambda x: -x[1])) + + ("\n" + "\n".join(examples.get(pid, [])[:p.examples]) if examples.get(pid) else ""), + evidence={"pattern": p.regex, "total": total, "per_log": counts, + "min_count": p.min_count}, + artifacts=[f"{k}.txt" for k in sorted(counts)], + note=p.note, + ) diff --git a/test/framework/sbtest/detectors/meta.py b/test/framework/sbtest/detectors/meta.py new file mode 100644 index 000000000..f74f8dfd8 --- /dev/null +++ b/test/framework/sbtest/detectors/meta.py @@ -0,0 +1,190 @@ +"""Meta-detectors: checks on the evidence itself. + +These answer "can this run be judged at all?", which is a different question from "did it +pass" and one that nothing else asks. They exist because the alternative kept happening: a +log-based verdict was drawn over a log that covered a fraction of the run, and nothing in the +report said so. + +The framework already distinguishes a detector that *skipped* from one that found nothing. +This module extends that to partial evidence, which is the harder and more common case — a log +that exists, opens fine, and is missing the two hours you needed. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, detector, info, warning + + +@detector +class LogCoverage(Detector): + """A collected log that does not span the run. + + Container logs are rotated by the kubelet against a fixed byte budget, so a busy + component's log covers the last N minutes rather than the run. That is survivable if you + know it and misleading if you do not: every `logs.pattern` count, every kernel finding and + every manual grep is silently scoped to whatever survived. + + Reported as a WARNING rather than a failure — incomplete evidence is not a defect in the + system under test — but reported *loudly*, because it bounds what the rest of the report + is allowed to claim. + """ + + name = "evidence.log-coverage" + summary = "a collected log that does not cover the whole run, bounding what can be claimed" + + def defaults(self) -> dict: + return { + # Below this, a gap is startup jitter rather than rotation. + "min_gap_s": 60.0, + # Logs that legitimately begin mid-run: a component created by the run itself has + # nothing to say before it existed. + "ignore": [], + } + + def detect(self, ev: Evidence) -> Iterable[Finding]: + spans = ev.log_spans() + if not spans: + raise SkipDetector("no collected logs to measure") + start, end = ev.run_window() + if not start: + raise SkipDetector("the run window is unknown, so coverage cannot be measured") + + min_gap = float(self.opt("min_gap_s")) + ignore = set(self.opt("ignore") or []) + short: list[tuple[str, float, float]] = [] # name, missing-at-start, covered fraction + empty: list[str] = [] + total = (end - start).total_seconds() if end else 0.0 + + for sp in spans: + if sp.name in ignore: + continue + if not sp.first or not sp.last: + empty.append(sp.name) + continue + missing = (sp.first - start).total_seconds() + covered = max(0.0, (sp.last - max(sp.first, start)).total_seconds()) + if missing > min_gap: + short.append((sp.name, missing, covered / total if total else 0.0)) + + if short: + short.sort(key=lambda x: -x[1]) + worst = short[0] + yield warning( + self.name, + title=f"{len(short)} log(s) do not cover the start of the run", + subject="evidence", + detail="; ".join(f"{n}: missing first {m / 60:.0f} min" + f"{f', covers {c:.0%}' if c else ''}" for n, m, c in short), + evidence={"logs": {n: {"missing_start_s": round(m), + "covered_fraction": round(c, 3)} + for n, m, c in short}, + "worst": worst[0]}, + note="Log-derived findings are scoped to what survived rotation, so an absence " + "in these files is not evidence of absence. Raise the kubelet's " + "containerLogMaxSize/Files, or follow the log live (logs.stream) for the " + "components that outrun it.", + ) + if empty: + yield warning( + self.name, + title=f"{len(empty)} collected log(s) carry no usable timestamp", + subject="evidence", + detail=", ".join(sorted(empty)), + evidence={"logs": sorted(empty)}, + note="Either the collection produced nothing or the format is unrecognised; " + "either way nothing in these can be placed in time.", + ) + + +@detector +class MigrationBlindSpot(Detector): + """A migration that happened while a log was not covering it. + + The sharper form of the coverage problem, and the one that actually costs time: on the run + this was written for, the migration that silently corrupted data ran 09:28:32-09:31:00 and + one node's SPDK log began at 09:31:06 — six seconds after it finished. There is no + post-mortem to do for that node, and the only thing worse than knowing that is not knowing + it. + + Deliberately per (migration, log): "the run is 80% covered" is useless when the missing + 20% is the interesting part. + """ + + name = "evidence.blind-spot" + summary = "a migration that no log covers, so it cannot be post-mortemed" + + def defaults(self) -> dict: + return {"logs": ["spdk-*", "operator*"], "max_blind": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + import fnmatch + migs = ev.migrations() + spans = [s for s in ev.log_spans() + if any(fnmatch.fnmatch(s.name, g) for g in self.opt("logs"))] + if not migs: + raise SkipDetector("no migrations in this run") + if not spans: + raise SkipDetector("none of the requested logs were collected") + + blind: dict[str, list[str]] = {} + for m in migs: + end = m.end or m.start + for sp in spans: + if not sp.first or not sp.last: + continue + # No overlap at all between the migration and what the log holds. + if sp.first > end or sp.last < m.start: + blind.setdefault(m.name, []).append(sp.name) + + if len(blind) <= int(self.opt("max_blind")): + return + worst = sorted(blind.items(), key=lambda kv: -len(kv[1])) + yield warning( + self.name, + title=f"{len(blind)} migration(s) fall outside at least one log's coverage", + subject="evidence", + detail="; ".join(f"{name}: no {', '.join(sorted(logs))}" + for name, logs in worst[:8]) + + (" ..." if len(worst) > 8 else ""), + evidence={"blind": {k: sorted(v) for k, v in blind.items()}}, + note="These migrations cannot be investigated from those logs whatever they turn " + "out to have done. If one of them also carries a CRITICAL finding, collect " + "again with logs.stream before spending time on the analysis.", + ) + + +@detector +class EvidenceInventory(Detector): + """What evidence this run actually produced. Always INFO; never a verdict. + + Worth a finding of its own because "which of the fourteen checks could even run" is the + first thing anyone asks of a report they did not generate, and reconstructing it from the + skip list is guesswork. + """ + + name = "evidence.inventory" + summary = "what evidence the run produced (informational)" + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + have = { + "migrations": len(migs), + "ana_sampled_migrations": sum(1 for m in migs if ev.ana_samples(m.name)), + "fio_pods": len(ev.pods()), + "fio_jobs": len(ev.fio_jobs()), + "container_logs": len(ev.container_logs()), + "control_events": len(ev.control_events()), + "nvme_controllers": len(ev.nvme_controllers()), + } + start, end = ev.run_window() + yield info( + self.name, + title=", ".join(f"{k}={v}" for k, v in have.items() if v), + subject="evidence", + detail=(f"run window {start:%H:%M:%S}..{end:%H:%M:%S} UTC" + if start and end else "run window unknown"), + evidence={**have, "cluster": ev.cluster_uuid(), + "window_known": bool(start)}, + ) diff --git a/test/framework/sbtest/detectors/migration.py b/test/framework/sbtest/detectors/migration.py new file mode 100644 index 000000000..ba951ed4e --- /dev/null +++ b/test/framework/sbtest/detectors/migration.py @@ -0,0 +1,115 @@ +"""Migration-outcome detectors — the run's success rate, and how it failed. + +Deliberately rate-based rather than per-migration: one failed migration in a long run is +noise, and a run where half of them time out is a different defect from a run where one did. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector, info, warning + + +@detector +class Outcomes(Detector): + """Too few migrations completed, or too many ended one particular way. + + The phase breakdown is the finding, not just the count. A run of 46 migrations with 13 + Completed and 25 TIMEOUT says something quite specific — none of the timeouts ever + reached a cutover — that "28% success" alone does not. + """ + + name = "migration.outcomes" + summary = "completion rate and phase breakdown across the run" + + def defaults(self) -> dict: + return {"min_completed_fraction": 0.8, + "max_timeout_fraction": 0.1, + "completed_phases": ["Completed"], + "timeout_phases": ["TIMEOUT", "Timeout"]} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + total = len(migs) + done = set(self.opt("completed_phases")) + tmo = set(self.opt("timeout_phases")) + + phases: dict[str, int] = {} + for m in migs: + phases[m.phase or "?"] = phases.get(m.phase or "?", 0) + 1 + breakdown = ", ".join(f"{k}={v}" for k, v in sorted(phases.items(), key=lambda x: -x[1])) + + completed = sum(v for k, v in phases.items() if k in done) + timeouts = sum(v for k, v in phases.items() if k in tmo) + frac = completed / total + yield info(self.name, title=f"{completed}/{total} migrations completed", + subject="run", detail=breakdown, + evidence={"total": total, "completed": completed, "phases": phases}) + + if frac < float(self.opt("min_completed_fraction")): + yield critical( + self.name, + title=f"only {frac:.0%} of migrations completed", + subject="run", + detail=breakdown, + evidence={"completed_fraction": round(frac, 3), + "min_completed_fraction": float(self.opt("min_completed_fraction")), + "phases": phases}, + ) + if timeouts / total > float(self.opt("max_timeout_fraction")): + yield critical( + self.name, + title=f"{timeouts}/{total} migrations timed out", + subject="run", + detail=breakdown, + evidence={"timeouts": timeouts, "total": total, + "max_timeout_fraction": float(self.opt("max_timeout_fraction"))}, + note="A timeout that never reached a cutover leaves the volume on the " + "source with the target's objects still allocated; check whether the " + "cutover was attempted at all before blaming the copy.", + ) + + +@detector +class Errors(Detector): + """Distinct migration error messages, grouped. + + Grouping matters more than counting here: 16 identical "NVMe path validation failed" + errors are one defect, and the value is in seeing that they are identical. + """ + + name = "migration.errors" + summary = "distinct migration error messages, grouped by shape" + + def defaults(self) -> dict: + return {"max_distinct": 0} + + @staticmethod + def _shape(err: str) -> str: + import re + s = re.sub(r"[0-9a-f]{8}-[0-9a-f-]{27}", "", err) + s = re.sub(r"\d+", "N", s) + return s[:200] + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + groups: dict[str, list[str]] = {} + for m in migs: + if m.error: + groups.setdefault(self._shape(m.error), []).append(m.name) + if not groups: + return + for shape, names in sorted(groups.items(), key=lambda x: -len(x[1])): + yield warning( + self.name, + title=f"{len(names)} migration(s) failed with the same error", + subject=shape[:60], + detail=f"{shape}\nmigrations: {', '.join(sorted(names)[:12])}" + + (" ..." if len(names) > 12 else ""), + evidence={"count": len(names), "migrations": sorted(names), "shape": shape}, + ) diff --git a/test/framework/sbtest/detectors/nvme.py b/test/framework/sbtest/detectors/nvme.py new file mode 100644 index 000000000..3c57e4261 --- /dev/null +++ b/test/framework/sbtest/detectors/nvme.py @@ -0,0 +1,140 @@ +"""Host NVMe fabric detectors — leaked and non-contributing controllers. + +These judge the state a run *leaves behind*, which turned out to matter as much as what it +does: a leaked controller is invisible to the run that created it and breaks the next one. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector, warning + + +@detector +class StaleControllers(Detector): + """Controllers that cannot carry I/O and will not recover on their own. + + Two shapes, both from the migration path-leak investigation: + + * **live, serving no namespace** — the admin queue is up and enumeration finished, and + the controller was told about nothing. It looks connected from every angle a connect + checks, so `nvme connect` returns "already connected" and a reconciler that counts + paths sees nothing wrong. This is the state that made a single abandoned migration + block every later migration of the same subsystem, because the pre-cutover check + reports it forever. + * **stuck connecting** — retrying an endpoint that has stopped answering for this host. + Bounded only by ctrl_loss_tmo, and the kernel resets its reconnect counter whenever a + reconnect gets far enough, so "bounded" can mean "until the node reboots". + + Run this at the *end* of a run, and on the next run's setup: it is the check that turns + "the last run left a mess" from a guess into a finding. + """ + + name = "nvme.stale-controllers" + summary = "controllers that are live with no namespace, or stuck connecting" + + def defaults(self) -> dict: + return {"max_zero_namespace": 0, "max_connecting": 0, + # Expected paths per subsystem per host: primary plus HA replicas. Above + # this is leak territory, but the healthy number is topology-dependent, so + # it warns rather than fails. + "expected_paths_per_subsystem": 3} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + ctrls = ev.nvme_controllers() + if not ctrls: + raise SkipDetector("no host NVMe controller snapshot in this run") + + zero = [c for c in ctrls if c.state == "live" and c.serves_nothing] + conn = [c for c in ctrls if c.state == "connecting"] + + if len(zero) > int(self.opt("max_zero_namespace")): + by_node: dict[str, list[str]] = {} + for c in zero: + by_node.setdefault(c.node, []).append(f"{c.name}@{c.address}") + yield critical( + self.name, + title=f"{len(zero)} live controller(s) serving no namespace", + subject="fabric", + detail="; ".join(f"{n}: {', '.join(sorted(v))}" for n, v in sorted(by_node.items())), + evidence={"count": len(zero), + "per_node": {n: sorted(v) for n, v in by_node.items()}}, + note="Each of these makes the pre-cutover path check fail for its " + "subsystem, so they block later migrations until something tears them " + "down. Nothing routes I/O over them, so removing them is safe.", + ) + + if len(conn) > int(self.opt("max_connecting")): + by_node = {} + for c in conn: + by_node.setdefault(c.node, []).append(f"{c.name}@{c.address}") + yield warning( + self.name, + title=f"{len(conn)} controller(s) stuck connecting", + subject="fabric", + detail="; ".join(f"{n}: {', '.join(sorted(v))}" for n, v in sorted(by_node.items())), + evidence={"count": len(conn), + "per_node": {n: sorted(v) for n, v in by_node.items()}, + "ctrl_loss_tmo": sorted({c.ctrl_loss_tmo for c in conn + if c.ctrl_loss_tmo is not None})}, + note="A snapshot cannot tell one of these from a normal HA reconnect, hence " + "a warning. What bounds them is ctrl_loss_tmo — a large value here " + "means they can outlive the run.", + ) + + # Path count per (host, subsystem): the leak seen from a different angle, and the + # one that catches an accumulation whose members all still look individually fine. + limit = int(self.opt("expected_paths_per_subsystem")) + per: dict[tuple[str, str], list[str]] = {} + for c in ctrls: + per.setdefault((c.node, c.nqn), []).append(c.address) + for (node, nqn), addrs in sorted(per.items()): + if len(addrs) > limit: + yield warning( + self.name, + title=f"{len(addrs)} paths to one subsystem on {node} (expected <= {limit})", + subject=f"{node}/{nqn.rsplit(':', 1)[-1]}", + detail=", ".join(sorted(addrs)), + evidence={"node": node, "nqn": nqn, "addresses": sorted(addrs), + "limit": limit}, + ) + + +@detector +class LossTimeout(Detector): + """A controller whose ctrl_loss_tmo lets it outlive the run that created it. + + Worth checking because it is the difference between a leak that expires on its own and + one that has to be cleaned up: the control plane answered migration connects with an + hour, while the CSI driver connects every other path with a minute. A probe path that + may be abandoned should expire quickly. + """ + + name = "nvme.loss-timeout" + summary = "a controller connected with a ctrl_loss_tmo long enough to outlive the run" + + def defaults(self) -> dict: + return {"max_ctrl_loss_tmo_s": 60} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + ctrls = [c for c in ev.nvme_controllers() if c.ctrl_loss_tmo is not None] + if not ctrls: + raise SkipDetector("no controller snapshot with ctrl_loss_tmo") + limit = int(self.opt("max_ctrl_loss_tmo_s")) + bad = [c for c in ctrls + if c.ctrl_loss_tmo is not None and c.ctrl_loss_tmo > limit] + if not bad: + return + vals = sorted({c.ctrl_loss_tmo for c in bad if c.ctrl_loss_tmo is not None}) + yield warning( + self.name, + title=f"{len(bad)} controller(s) with ctrl_loss_tmo above {limit}s", + subject="fabric", + detail=f"values seen: {vals}", + evidence={"count": len(bad), "values": vals, "limit": limit, + "controllers": sorted(f"{c.node}/{c.name}@{c.address}" for c in bad)[:32]}, + note="A path that may be abandoned should expire on its own. The kernel also " + "resets its reconnect counter on a partial reconnect, so a large value is " + "a floor on how long a leak survives, not a bound.", + ) diff --git a/test/framework/sbtest/detectors/security.py b/test/framework/sbtest/detectors/security.py new file mode 100644 index 000000000..99c4bf3cf --- /dev/null +++ b/test/framework/sbtest/detectors/security.py @@ -0,0 +1,85 @@ +"""Security detectors over collected evidence. + +One check for now, and it is the one that keeps being true: artifacts get attached to tickets, +copied into chat and committed to repositories, so a credential that reaches a log reaches all +of those. This is cheap to check and expensive to miss, and it is entirely independent of what +the run was testing. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector + +#: Shapes worth refusing to ship. Each is deliberately narrow: a pattern that fires on ordinary +#: text gets disabled, and a disabled check finds nothing. +SECRET_PATTERNS: tuple[tuple[str, str], ...] = ( + ("private key block", r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), + ("nvme dhchap secret", r"DHHC-1:[0-9]{2}:[A-Za-z0-9+/=]{20,}"), + ("bearer token", r"[Bb]earer\s+[A-Za-z0-9._-]{24,}"), + ("kubernetes sa token", r"eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\."), + ("aws access key", r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), + ("url with credentials", r"[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s:@]+@"), + ("password assignment", r"(?i)\b(?:password|passwd|secret[_-]?key)\s*[=:]\s*['\"]?[^\s'\"]{8,}"), +) + + +@detector +class SecretExposure(Detector): + """A credential-shaped string in a collected log. + + Reports the pattern, the log and the line number — never the match itself. A finding that + quotes the secret has copied it into a second file, and findings.json travels at least as + widely as the log did. + """ + + name = "security.secret-exposure" + summary = "credential-shaped strings in collected logs (reports location, never the value)" + + def defaults(self) -> dict: + return {"logs": ["*"], "extra_patterns": [], "max_reported": 12} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + names = [n for n in ev.container_logs() + if any(fnmatch.fnmatch(n, g) for g in self.opt("logs"))] + if not names: + raise SkipDetector("no collected logs to scan") + + patterns = [(label, re.compile(rx)) for label, rx in SECRET_PATTERNS] + patterns += [(str(p.get("id", "custom")), re.compile(str(p["regex"]))) + for p in (self.opt("extra_patterns") or [])] + + # label -> log -> [line numbers] + hits: dict[str, dict[str, list[int]]] = {} + for name in names: + for lineno, line in enumerate(ev.container_log(name), start=1): + for label, rx in patterns: + if rx.search(line): + hits.setdefault(label, {}).setdefault(name, []).append(lineno) + if not hits: + return + + cap = int(self.opt("max_reported")) + for label, per_log in sorted(hits.items()): + total = sum(len(v) for v in per_log.values()) + where = "; ".join( + f"{log}:{','.join(str(n) for n in lines[:cap])}" + f"{f' (+{len(lines) - cap} more)' if len(lines) > cap else ''}" + for log, lines in sorted(per_log.items())) + yield critical( + self.name, + title=f"{total} line(s) matching {label}", + subject=label, + detail=where, + evidence={"pattern": label, "total": total, + "per_log": {k: len(v) for k, v in per_log.items()}}, + # The file, not the log's name: the run directory holds .txt, and + # a pointer a reader has to complete is not a pointer. + artifacts=[f"{log}.txt" for log in sorted(per_log)], + note="The value is deliberately not quoted here — findings.json travels at " + "least as widely as the log. Rotate whatever this is, then stop the " + "component logging it; artifacts get attached to tickets and committed.", + ) diff --git a/test/framework/sbtest/suites/analyze-only.yaml b/test/framework/sbtest/suites/analyze-only.yaml new file mode 100644 index 000000000..b4d1fb562 --- /dev/null +++ b/test/framework/sbtest/suites/analyze-only.yaml @@ -0,0 +1,15 @@ +# Judge an existing artifact directory. No components, so nothing is collected and nothing on +# the cluster is touched — this is what `sbtest analyze` does by default, written out as a +# suite for when it needs to be said explicitly. +# +# make analyze RUN=../../operator/fio-mig-1787211447 SUITE=analyze-only +description: >- + Judge an existing artifact directory. Nothing is collected and nothing is touched. + +# Empty rather than absent: an empty mapping says "no components, deliberately", where a +# missing key would rely on the reader knowing that components default to off. +components: {} + +# Empty means every registered detector runs on its defaults — detectors default to *on*, +# because judging is not something you should have to remember to switch on. +detectors: {} diff --git a/test/framework/sbtest/suites/corruption-hunt.yaml b/test/framework/sbtest/suites/corruption-hunt.yaml new file mode 100644 index 000000000..781aa696c --- /dev/null +++ b/test/framework/sbtest/suites/corruption-hunt.yaml @@ -0,0 +1,48 @@ +# Narrowed to the one question: did this run lose or corrupt data? +# +# Everything that measures *quality* is off, so a FAIL from this suite means data — not +# latency, not a slow cutover, not a timed-out migration. That makes it the suite to bisect +# with, where a verdict has to mean exactly one thing. +# +# make analyze RUN= SUITE=corruption-hunt +description: >- + Narrowed to the data-integrity question, for a bisect or a quick verdict. + +components: + ana.sample: + # 1s rather than 2: the freeze count is only as sharp as the interval measuring it, and + # the count is this suite's primary signal. + interval_s: 1.0 + logs.stream: + containers: + - spdk-container + nvme.snapshot: true + +detectors: + ana.freeze-count: true + ana.split-brain: true + fio.checksum: true + fio.job-error: true + + logs.pattern: + patterns: + # SPDK's own admission that the delta copy began before in-flight I/O had drained. + # This is the root cause the corruption RCA landed on, so it is worth matching + # verbatim rather than inferring from the ANA side-effects. + - id: spdk.undrained-transfer + regex: "still have outstanding io|Task transfer timeout with outstanding" + logs: ["spdk-*"] + severity: critical + note: the delta copy started before in-flight I/O was drained + + # Off: these measure how *well* it went, not whether data survived. Keeping them on would + # make a bisect ambiguous — a slow cutover would fail the same run a lost write does. + ana.cutover-pause: false + ana.path-churn: false + ana.unserved-after-cutover: false + fio.outage: false + fio.throughput-outlier: false + migration.outcomes: false + migration.errors: false + nvme.stale-controllers: false + nvme.loss-timeout: false diff --git a/test/framework/sbtest/suites/migration-full.yaml b/test/framework/sbtest/suites/migration-full.yaml new file mode 100644 index 000000000..9233cd954 --- /dev/null +++ b/test/framework/sbtest/suites/migration-full.yaml @@ -0,0 +1,105 @@ +# The full driven migration run: the sbtest equivalent of +# +# fio_migration_test.py --keep --pods 0 --runtime 7200 --ns-pods 25 \ +# --ns-per-subsys 6 --snapshot-chance 0 --target-policy alternate +# +# make run SUITE=migration-full DURATION=7200 KEEP=1 +description: >- + The full driven migration run: 25 namespaced volumes packed 6 to a subsystem, so every + migration is a batch migration — the case that corrupts. + +run: + id_prefix: fiomig + outdir: ./runs + +components: + workload.fio: + # pods=0: the single-namespace path is not what is under investigation. Raise it to + # compare single against batch in one run — they share the geometry, so the comparison + # is meaningful only if both classes format their filesystems identically, which is why + # the workload derives both from the same base parameters. + pods: 0 + ns_pods: 25 + ns_per_subsys: 6 + volume_size_gb: 10 + # Small on purpose: fio always lays out a file-backed target before random I/O and + # cannot be told to skip it. The point is continuous I/O, not capacity. + file_size_gb: 1 + # Longer than the run's DURATION so fio never exits first and leaves the last migrations + # measuring an idle volume. + runtime_s: 7500 + iodepth: 8 + # 1, and it must stay 1 for an integrity run: numjobs > 1 cannot serialize overlapping + # writes without io_submit_mode=offload, so verify would report corruption that never + # happened — and the workload switches verification off rather than lie. + numjobs: 1 + + migration.driver: + # `alternate` puts the harder case (target also hosts a consumer) on the odd migrations, + # so a run cut short still exercised it. + target_policy: alternate + gap_s: 30.0 + timeout_s: 600.0 + + ana.sample: + # 2s. The freeze windows being measured are 2-8s, so a coarser interval cannot count + # them and a finer one costs an exec per node per sample. + interval_s: 2.0 + + # The SPDK containers outrun kubelet rotation within the hour — measured on vm04: rotation + # every ~2 min, so the whole 50 MiB budget bought about 10 minutes — so they are followed + # live. Everything else is still complete from a one-shot grab at the end. + logs.stream: + containers: + - spdk-container + - spdk-proxy-container + ttl_s: 21600 + logs.collect: true + host.dmesg: true + cluster.events: true + nvme.snapshot: true + +detectors: + ana.freeze-count: + # 1. More than one ANA freeze in a migration meant lost writes on 5 of 5 archived runs: + # each extra freeze is a revert-and-retry, and the retry can copy nothing. + max_freezes: 1 + ana.cutover-pause: + # The control plane makes all paths inaccessible for ~2s by design; 5 leaves headroom + # without reaching fast_io_fail_tmo, past which the pause becomes application-visible. + max_pause_s: 5.0 + ana.split-brain: true + ana.unserved-after-cutover: true + + fio.job-error: true + fio.checksum: + # A lost write surfaces when fio next reads the block, which is seconds to tens of + # seconds after the migration that lost it — without the lag, a migration's own + # corruption files itself under "no migration was running". + verify_lag_s: 45 + fio.outage: + min_seconds: 30 + + migration.outcomes: + min_completed_fraction: 0.8 + max_timeout_fraction: 0.1 + migration.errors: true + + nvme.stale-controllers: true + nvme.loss-timeout: true + nvme.controller-churn: true + nvme.foreign-cluster: true + + kernel.path-loss: true + kernel.filesystem-shutdown: true + + control.node-flap: true + control.task-stuck: true + control.volume-health: true + + logs.pattern: true + + # Bounds what the rest of the report is allowed to claim. + evidence.log-coverage: true + evidence.blind-spot: true + evidence.inventory: true diff --git a/test/framework/sbtest/suites/migration-soak.yaml b/test/framework/sbtest/suites/migration-soak.yaml new file mode 100644 index 000000000..93d8e1242 --- /dev/null +++ b/test/framework/sbtest/suites/migration-soak.yaml @@ -0,0 +1,47 @@ +# Observation only: no workload, no driver. For watching a cluster someone else is loading — +# including the existing operator/test/fio_migration_test.py — and for checking that +# collection works before committing a long run to it. +# +# make collect OUT=./runs/soak SUITE=migration-soak DURATION=600 +# +# For a run this framework drives itself, use migration-full. +description: >- + Long migration soak — the collection configuration the corruption work needed. Both log + components are on deliberately. + +run: + id_prefix: fiomig + outdir: ./runs + +components: + # The SPDK containers outrun kubelet rotation within the hour, so they are followed live; + # everything else is still complete from a one-shot grab at the end. That split is the + # whole reason both log components exist. + logs.stream: + containers: + - spdk-container + - spdk-proxy-container + ttl_s: 21600 + logs.collect: true + host.dmesg: true + cluster.events: true + # Before *and* after: "did the last run leave a mess?" is a real question, because a leaked + # controller breaks the next run rather than this one. + nvme.snapshot: true + ana.sample: + interval_s: 2.0 + +detectors: + ana.freeze-count: + # More than one freeze in a migration meant lost writes on 5 of 5 archived runs. + max_freezes: 1 + ana.cutover-pause: + max_pause_s: 5.0 + fio.checksum: + # Corruption surfaces when fio next reads the block, after the migration has ended. + verify_lag_s: 45 + fio.outage: + min_seconds: 30 + migration.outcomes: + min_completed_fraction: 0.8 + max_timeout_fraction: 0.1 diff --git a/test/framework/tests/test_core.py b/test/framework/tests/test_core.py new file mode 100644 index 000000000..e9c57a8d2 --- /dev/null +++ b/test/framework/tests/test_core.py @@ -0,0 +1,595 @@ +"""Core tests: config resolution, lifecycle ordering, findings/report, archive round-trip. + +The lifecycle tests matter more than they look. A component that starts pods must be torn +down even when the run explodes, and a collector that fails must not take the run's +judgement with it — both are properties of the runner, not of any component, so this is the +only place they can be pinned. +""" + +from __future__ import annotations + +import csv +import json +import os +import sys +import tempfile +import unittest +from datetime import UTC, datetime, timedelta +from typing import cast +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import sbtest # noqa: E402,F401 (registers the bundled plugins) +from sbtest.adapters import ArchiveEvidence # noqa: E402 +from sbtest.components import kube # noqa: E402 +from sbtest.core import ( # noqa: E402 + Component, + Detector, + Logger, + Report, + RunContext, + Runner, + Severity, + SkipDetector, + apply_cli_toggles, + component, + critical, + detector, + known_components, + known_detectors, + load, + now_utc, +) +from sbtest.core.config import _resolve # noqa: E402 + + +class ConfigResolution(unittest.TestCase): + KNOWN_C = ["logs.stream", "logs.collect"] + KNOWN_D = ["ana.freeze-count", "fio.checksum"] + + def test_absent_component_is_off_and_absent_detector_is_on(self): + """Collecting is a cost you opt into; judging should not need remembering.""" + cfg = load(None, self.KNOWN_C, self.KNOWN_D) + self.assertEqual(cfg.components.enabled, {}) + self.assertEqual(sorted(cfg.components.disabled), sorted(self.KNOWN_C)) + self.assertEqual(sorted(cfg.detectors.enabled), sorted(self.KNOWN_D)) + + def test_bool_and_mapping_forms(self): + sel = _resolve({"logs.stream": True, "logs.collect": {"ttl_s": 99}}, + self.KNOWN_C, False) + self.assertEqual(sel.enabled["logs.stream"], {}) + self.assertEqual(sel.enabled["logs.collect"], {"ttl_s": 99}) + + def test_enabled_false_inside_a_mapping_disables(self): + sel = _resolve({"logs.stream": {"enabled": False, "ttl_s": 5}}, self.KNOWN_C, True) + self.assertIn("logs.stream", sel.disabled) + self.assertNotIn("logs.stream", sel.enabled) + + def test_unknown_name_fails_loudly(self): + with self.assertRaises(ValueError) as cm: + _resolve({"logs.stremm": True}, self.KNOWN_C, False) + self.assertIn("logs.stremm", str(cm.exception)) + + def test_unknown_option_fails_at_build_not_at_use(self): + with self.assertRaises(KeyError): + sbtest.build_detector("nope.not-a-detector") + with self.assertRaises(ValueError): + sbtest.build_detector("ana.freeze-count", wrong_option=1) + + def test_cli_disable_beats_enable(self): + sel = _resolve({}, self.KNOWN_C, True) + apply_cli_toggles(sel, ["logs.stream"], ["logs.stream"]) + self.assertIn("logs.stream", sel.disabled) + self.assertNotIn("logs.stream", sel.enabled) + + def test_yaml_suite_round_trip(self): + """Comments and all — the reason suites are YAML is that a threshold wants a sentence + saying why it is that number, and the parser must not care that one is there.""" + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "suite.yaml") + with open(p, "w") as fh: + fh.write("components:\n" + " logs.stream:\n" + " ttl_s: 7 # six hours was not enough on vm04\n" + "detectors:\n" + " fio.checksum: false\n") + cfg = load(p, self.KNOWN_C, self.KNOWN_D) + self.assertEqual(cfg.components.enabled["logs.stream"], {"ttl_s": 7}) + self.assertIn("fio.checksum", cfg.detectors.disabled) + self.assertIn("ana.freeze-count", cfg.detectors.enabled) + + def test_json_suite_still_loads(self): + """Kept working for anything generating suites programmatically.""" + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "suite.json") + with open(p, "w") as fh: + json.dump({"components": {"logs.stream": {"ttl_s": 7}}, + "detectors": {"fio.checksum": False}}, fh) + cfg = load(p, self.KNOWN_C, self.KNOWN_D) + self.assertEqual(cfg.components.enabled["logs.stream"], {"ttl_s": 7}) + self.assertIn("fio.checksum", cfg.detectors.disabled) + + def test_malformed_yaml_names_the_file(self): + """A parse error must say which suite and where, not surface as a bare YAMLError.""" + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "broken.yaml") + with open(p, "w") as fh: + fh.write("components:\n logs.stream: {ttl_s: 7\n") + with self.assertRaises(ValueError) as cm: + load(p, self.KNOWN_C, self.KNOWN_D) + self.assertIn("broken.yaml", str(cm.exception)) + + def test_a_suite_that_is_not_a_mapping_is_refused(self): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "list.yaml") + with open(p, "w") as fh: + fh.write("- logs.stream\n- logs.collect\n") + with self.assertRaises(ValueError) as cm: + load(p, self.KNOWN_C, self.KNOWN_D) + self.assertIn("mapping", str(cm.exception)) + + def test_every_bundled_suite_loads_against_the_real_registry(self): + """The suites ship with the package, so a typo in one is a broken release. Loading + resolves names against the registry, so this also pins that no suite references a + detector that has since been renamed.""" + names = ["analyze-only", "corruption-hunt", "migration-soak", "migration-full"] + for name in names: + p = sbtest.suite_path(name) + self.assertIsNotNone(p, f"bundled suite {name} not found") + assert p is not None + self.assertTrue(p.endswith(".yaml"), f"{name} should be YAML, got {p}") + cfg = load(p, list(known_components()), list(known_detectors())) + self.assertTrue(cfg.raw.get("description"), f"{name} has no description") + # migration-full is the one that must actually drive a run. + full = load(sbtest.suite_path("migration-full"), + list(known_components()), list(known_detectors())) + self.assertIn("workload.fio", full.components.enabled) + self.assertIn("migration.driver", full.components.enabled) + + +class Lifecycle(unittest.TestCase): + def ctx(self, d): + return RunContext(run_id="t", outdir=d, log=Logger(None)) + + def test_hooks_run_in_order(self): + calls = [] + + @component + class Ordered(Component): + name = "test.ordered" + def setup(self, ctx): calls.append("setup") + def start(self, ctx): calls.append("start") + def tick(self, ctx): calls.append("tick") + def stop(self, ctx): calls.append("stop") + def collect(self, ctx): calls.append("collect") + def teardown(self, ctx): calls.append("teardown") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.ordered"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, self.ctx(d)).build() + for phase in ("setup", "start", "tick", "stop", "collect", "teardown"): + getattr(r, phase)() + self.assertEqual(calls, ["setup", "start", "tick", "stop", "collect", "teardown"]) + + def test_teardown_runs_even_when_setup_failed(self): + """A component that allocates in setup must still get its teardown.""" + torn = [] + + @component + class Exploding(Component): + name = "test.exploding" + required = True # this component *is* the run; failing it must abort + def setup(self, ctx): raise RuntimeError("boom") + def teardown(self, ctx): torn.append(self.name) + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.exploding"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, self.ctx(d)).build() + with self.assertRaises(RuntimeError): + r.setup() + r.teardown() + self.assertEqual(torn, ["test.exploding"]) + + def test_a_failing_collector_does_not_stop_the_others(self): + done = [] + + @component + class BadCollect(Component): + name = "test.badcollect" + def collect(self, ctx): raise RuntimeError("nope") + + @component + class GoodCollect(Component): + name = "test.goodcollect" + def collect(self, ctx): done.append("good") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.badcollect", "test.goodcollect"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, self.ctx(d)).build() + r.collect() + self.assertEqual(done, ["good"]) + # and the failure is recorded rather than swallowed + self.assertTrue(any(f.detector == "component/test.badcollect" for f in r.report.findings)) + + +class Judging(unittest.TestCase): + def test_a_raising_detector_is_reported_not_treated_as_clean(self): + @detector + class Boom(Detector): + name = "test.boom" + def detect(self, ev): raise ValueError("bug in me") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + cfg.components.enabled = {} + cfg.detectors.enabled = {"test.boom": {}} + r = Runner(cfg, RunContext(run_id="t", outdir=d, log=Logger(None))).build() + # Evidence is never touched: the detector raises first. That is the point. + rep = r.judge(cast("sbtest.core.Evidence", object())) + self.assertIn("test.boom", rep.skipped) + self.assertTrue(any("raised" in f.title for f in rep.findings)) + + def test_skip_is_distinguishable_from_clean(self): + @detector + class Skipper(Detector): + name = "test.skipper" + def detect(self, ev): raise SkipDetector("no evidence here") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + cfg.components.enabled = {} + cfg.detectors.enabled = {"test.skipper": {}} + r = Runner(cfg, RunContext(run_id="t", outdir=d, log=Logger(None))).build() + rep = r.judge(cast("sbtest.core.Evidence", object())) + self.assertEqual(rep.findings, []) + self.assertEqual(rep.skipped["test.skipper"], "no evidence here") + self.assertEqual(rep.verdict, "PASS") # nothing found, but the gap is on record + + +class Findings(unittest.TestCase): + def test_critical_fails_the_verdict(self): + r = Report(run_id="x") + self.assertEqual(r.verdict, "PASS") + r.add(critical("d", "bad thing", subject="s")) + self.assertEqual(r.verdict, "FAIL") + + def test_by_subject_groups_and_orders_worst_first(self): + r = Report() + r.add(sbtest.info("a", "note", subject="mig-1"), + critical("b", "boom", subject="mig-1")) + self.assertEqual([f.severity for f in r.by_subject()["mig-1"]], + [Severity.CRITICAL, Severity.INFO]) + + def test_json_is_serialisable(self): + r = Report(run_id="x") + r.add(critical("d", "t", subject="s", evidence={"n": 1})) + parsed = json.loads(r.to_json()) + self.assertEqual(parsed["verdict"], "FAIL") + self.assertEqual(parsed["findings"][0]["severity"], "CRITICAL") + + +class Archive(unittest.TestCase): + """The archive reader is the seam that makes a check testable offline, so its + tolerance for missing files is a property worth pinning.""" + + def _write_run(self, d: str) -> None: + t0 = datetime(2026, 8, 19, 22, 0, 0, tzinfo=UTC) + state = {"run_id": "fiomig-test", "pods": ["fiomig-test-fio-0"], "migrations": [ + {"name": "mig-1", "start": t0.isoformat().replace("+00:00", "Z"), + "end": (t0 + timedelta(seconds=30)).isoformat().replace("+00:00", "Z"), + "phase": "Completed", "source": "srcuuid", "target": "tgtuuid", + "pv": "pvc-a", "group_pvs": ["pvc-a", "pvc-b"]}]} + with open(os.path.join(d, "state.json"), "w") as fh: + json.dump(state, fh) + os.makedirs(os.path.join(d, "ana"), exist_ok=True) + with open(os.path.join(d, "ana", "mig-1.csv"), "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["ts", "node", "phase", "address", "role", "ctrl_state", "nsid", "ana_state"]) + for off, st in ((0, "optimized"), (2, "inaccessible"), (4, "optimized")): + stamp = (t0 + timedelta(seconds=off)).strftime("%Y-%m-%dT%H:%M:%SZ") + for nsid in (1, 2): + w.writerow([stamp, "vm03", "Running", "10.0.0.1:4420", "source", + "live", nsid, st]) + pod = os.path.join(d, "fiomig-test-fio-0") + os.makedirs(pod, exist_ok=True) + with open(os.path.join(pod, "result.json"), "w") as fh: + json.dump({"jobs": [{"error": 121, "read": {"iops": 100.0}, + "write": {"iops": 50.0}}]}, fh) + with open(os.path.join(pod, "fio.log"), "w") as fh: + fh.write("2026-08-19T22:00:05.000000000Z stderr F all fine\n") + with open(os.path.join(pod, "timeseries.csv"), "w") as fh: + fh.write("t,wall,total_iops\n0,,100\n1,,0\n") + with open(os.path.join(d, "spdk-4420.txt"), "w") as fh: + fh.write("boring line\n") + + def test_reads_a_run_directory(self): + with tempfile.TemporaryDirectory() as d: + self._write_run(d) + ev = ArchiveEvidence(d) + self.assertEqual(ev.run_id, "fiomig-test") + migs = ev.migrations() + self.assertEqual(len(migs), 1) + self.assertTrue(migs[0].batch) # two group_pvs + self.assertEqual(len(ev.ana_samples("mig-1")), 3) # one per (ts, node, address) + self.assertEqual(ev.fio_jobs()[0].error, 121) + self.assertAlmostEqual(ev.fio_jobs()[0].total_iops, 150.0) + self.assertEqual(len(ev.fio_timeseries("fiomig-test-fio-0")), 2) + self.assertIn("spdk-4420", ev.container_logs()) + + def test_missing_files_yield_empty_not_an_exception(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "state.json"), "w") as fh: + json.dump({"run_id": "r", "migrations": []}, fh) + ev = ArchiveEvidence(d) + self.assertEqual(ev.migrations(), []) + self.assertEqual(ev.ana_samples("nope"), []) + self.assertEqual(ev.fio_jobs(), []) + self.assertEqual(list(ev.fio_log("nope")), []) + self.assertEqual(ev.nvme_controllers(), []) + + def test_falls_back_to_test_log_when_state_is_absent(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "test.log"), "w") as fh: + fh.write( + "2026-08-19T22:00:00Z [EVENT ] MIGRATION START mig-9 kind=namespaced " + "pod=p pv=pvc-x source=s target=t\n" + "2026-08-19T22:01:00Z [EVENT ] MIGRATION STOP mig-9 phase=TIMEOUT " + "error='timed out'\n") + ev = ArchiveEvidence(d) + migs = ev.migrations() + self.assertEqual(len(migs), 1) + self.assertEqual(migs[0].phase, "TIMEOUT") + self.assertEqual(migs[0].error, "timed out") + + def test_absent_directory_is_an_error(self): + with self.assertRaises(FileNotFoundError): + ArchiveEvidence("/definitely/not/here") + + +class Registry(unittest.TestCase): + @staticmethod + def _bundled(reg): + # Other tests register throwaway plugins into the same global registry. + return {k: v for k, v in reg.items() if not k.startswith("test.")} + + def test_every_bundled_detector_has_a_name_and_a_summary(self): + for name, cls in self._bundled(known_detectors()).items(): + self.assertTrue(name, cls) + self.assertTrue(cls.summary, f"{name} needs a summary") + + def test_every_bundled_component_has_a_name_and_a_summary(self): + for name, cls in self._bundled(known_components()).items(): + self.assertTrue(name, cls) + self.assertTrue(cls.summary, f"{name} needs a summary") + + def test_detector_defaults_are_the_option_allow_list(self): + for name, cls in self._bundled(known_detectors()).items(): + d = cls() + self.assertEqual(set(d.options), set(d.defaults()), + f"{name}: options must start from defaults") + + def test_duplicate_registration_is_rejected(self): + with self.assertRaises(ValueError): + @detector + class Dup(Detector): + name = "ana.freeze-count" + + +if __name__ == "__main__": + unittest.main(verbosity=2) + + +class RunWindowRecording(unittest.TestCase): + """A live run must record its own window. + + Found by running the collector against a real cluster: with no window, nothing in the + artifact directory says when the run was, so every detector reading a ring buffer marks + what it finds UNKNOWN — and UNKNOWN counts. The collector failed on twenty-nine filesystem + shutdowns that had happened hours before it started. + """ + + def test_run_json_is_written_and_read_back(self): + with tempfile.TemporaryDirectory() as d: + ctx = RunContext(run_id="t", outdir=d, log=Logger(None)) + ctx.mark_window() + ctx.mark_window(end=now_utc()) + with open(os.path.join(d, "run.json")) as fh: + rec = json.load(fh) + self.assertEqual(rec["run_id"], "t") + self.assertTrue(rec["start"] and rec["end"]) + start, end = ArchiveEvidence(d).run_window() + self.assertIsNotNone(start) + self.assertIsNotNone(end) + + def test_a_later_mark_does_not_erase_the_recorded_end(self): + """Regression: 2026-08-26-mark-window-erases-end (PR #445 review). + + mark_window persisted its `end` *argument* rather than the end it had stored, so any + later call without one rewrote run.json with end: null. Every detector that bounds + what it may blame on the run reads that window; with no end, evidence from after the + run counts as the run's. + """ + with tempfile.TemporaryDirectory() as d: + ctx = RunContext(run_id="t", outdir=d, log=Logger(None)) + ctx.mark_window() + ctx.mark_window(end=now_utc()) + ctx.mark_window() # a second collect phase, a component, a retry + with open(os.path.join(d, "run.json")) as fh: + rec = json.load(fh) + self.assertIsNotNone(rec["end"]) + self.assertIsNotNone(ArchiveEvidence(d).run_window()[1]) + + def test_run_json_wins_over_inference_from_test_log(self): + """The run's own record beats whatever happened to get logged.""" + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "test.log"), "w") as fh: + fh.write("1999-01-01T00:00:00Z [INFO] ancient\n") + with open(os.path.join(d, "run.json"), "w") as fh: + json.dump({"run_id": "t", "start": "2026-08-20T11:40:09Z", + "end": "2026-08-20T11:42:20Z"}, fh) + start, _end = ArchiveEvidence(d).run_window() + assert start is not None + self.assertEqual(start.year, 2026) + + def test_a_recorded_window_demotes_older_damage(self): + """The end-to-end point: with a window, inherited damage stops failing the run.""" + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "run.json"), "w") as fh: + json.dump({"run_id": "t", "start": "2026-08-20T11:40:09Z", + "end": "2026-08-20T11:42:20Z"}, fh) + # A filesystem shutdown from hours before the run began. + with open(os.path.join(d, "dmesg-vm03.txt"), "w") as fh: + fh.write("[Thu Aug 20 05:46:57 2026] XFS (nvme0n1): " + "Filesystem has been shut down due to log error (0x2).\n") + ev = ArchiveEvidence(d) + found = list(sbtest.build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, sbtest.Attribution.PRE_EXISTING) + rep = Report() + rep.add(*found) + self.assertFalse(rep.failed) # must not fail the run + self.assertEqual(rep.verdict, "PASS") # a WARNING, so not even inconclusive + + +class GrabberNaming(unittest.TestCase): + """Two components wanting a grabber on one node must not collide. + + Found by running it: a Pod is immutable, so the second component to apply the same name + failed on a field it may not change — and logs.collect silently produced empty files for + every node logs.stream had already claimed. + """ + + def test_stream_and_collect_pick_different_pod_names(self): + from sbtest.components.logs import LogCollect, LogStream + ctx = RunContext(run_id="run1", outdir="/tmp", log=Logger(None)) + node = "vm02.example.com" + stream = LogStream() + collect = LogCollect() + n1 = stream._grabber_manifest(ctx, node, f"sbtest-{stream.name.replace('.', '-')}" + f"-vm02-run1", 100) + n2 = collect._grabber_manifest(ctx, node, f"sbtest-{collect.name.replace('.', '-')}" + f"-vm02-run1", 100) + name1 = json.loads(n1)["metadata"]["name"] + name2 = json.loads(n2)["metadata"]["name"] + self.assertNotEqual(name1, name2) + self.assertIn("logs-stream", name1) + self.assertIn("logs-collect", name2) + + +class GrabberReuse(unittest.TestCase): + """logs.collect must reuse logs.stream's grabbers, and must not delete them. + + Two components wanting a privileged pod on the same node is the normal case, and the + first attempt at fixing their collision only gave them different names — which works but + leaves two pods per node doing the same job. Reuse is the intended behaviour; the distinct + names are the backstop for when reuse is not possible. + """ + + def _probe(self): + """A LogCollect with the cluster faked at the component's own boundaries. + + Everything below `collect()` is stubbed and everything inside it runs, because the + previous version of these cases re-implemented the reuse arithmetic in the test body + and never called `collect()` at all — so it went on passing while the component + borrowed nothing and recorded nothing to tear down. + """ + from sbtest.components import logs as logs_mod + started: list[list[str]] = [] + deleted: list[list[str]] = [] + + class Probe(logs_mod.LogCollect): + def _start_grabbers(self, ctx, nodes, ttl_s): + started.append(list(nodes)) + return {n: f"own-{n}" for n in nodes} + + def _delete_grabbers(self, ctx, names): + deleted.append(list(names)) + + pods = [kube.Pod(name=f"snode-spdk-{i}", namespace="default", node=node, + containers=("spdk-container",)) + for i, node in enumerate(("vm02", "vm03", "vm04"))] + c = Probe(targets=[{"pods": ["snode-spdk"], "containers": ["spdk-container"], + "name_from": "pod-container"}]) + return c, started, deleted, pods + + def test_collect_reuses_published_grabbers_and_starts_only_the_rest(self): + """Regression: 2026-08-26-logcollect-ignores-published-grabbers (PR #445 review). + + logs.collect started its own privileged pod on every node regardless of what + logs.stream had already published in ctx.shared["logs.grabbers"], so a run carried two + privileged pods per node doing the same job. + """ + c, started, _deleted, pods = self._probe() + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(kube, "list_pods", lambda *a, **k: pods), \ + mock.patch.object(kube, "run_bytes", lambda *a, **k: b"log line\n"): + ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) + ctx.shared["logs.grabbers"] = {"vm02": "stream-vm02", "vm03": "stream-vm03"} + c.collect(ctx) + self.assertEqual(started, [["vm04"]]) # only the uncovered node + self.assertEqual(c._grabbers["vm02"], "stream-vm02") # borrowed, not replaced + self.assertEqual(c._grabbers["vm04"], "own-vm04") + + def test_teardown_removes_the_grabbers_collect_started(self): + """Regression: 2026-08-26-logcollect-grabber-leak (PR #445 review). + + collect() never recorded what it started in `_own`, so teardown deleted nothing and + every privileged pod it created outlived the run, up to its TTL. + """ + c, _started, deleted, pods = self._probe() + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(kube, "list_pods", lambda *a, **k: pods), \ + mock.patch.object(kube, "run_bytes", lambda *a, **k: b"log line\n"): + ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) + ctx.shared["logs.grabbers"] = {"vm02": "stream-vm02"} + c.collect(ctx) + c.teardown(ctx) + # Its own three, and none of logs.stream's: deleting a borrowed pod would pull it + # out from under the component that owns it. + self.assertEqual(deleted, [["own-vm03", "own-vm04"]]) + + def test_collect_works_standalone_when_nothing_published(self): + """logs.stream may be disabled — collect must still start what it needs.""" + c, started, _deleted, pods = self._probe() + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(kube, "list_pods", lambda *a, **k: pods), \ + mock.patch.object(kube, "run_bytes", lambda *a, **k: b"log line\n"): + ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) + c.collect(ctx) + self.assertEqual(started, [["vm02", "vm03", "vm04"]]) + self.assertEqual(sorted(c._grabbers), ["vm02", "vm03", "vm04"]) + + def test_lifecycle_order_makes_reuse_safe(self): + """stop precedes collect, and teardown follows it — so the borrowed pod is idle and + still alive exactly when collection needs it.""" + order: list[str] = [] + + @component + class Streamish(Component): + name = "test.streamish" + def start(self, ctx): order.append("stream.start") + def stop(self, ctx): order.append("stream.stop") + def teardown(self, ctx): order.append("stream.teardown") + + @component + class Collectish(Component): + name = "test.collectish" + def collect(self, ctx): order.append("collect.collect") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.streamish", "test.collectish"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, RunContext(run_id="r", outdir=d, log=Logger(None))).build() + for phase in ("setup", "start", "stop", "collect", "teardown"): + getattr(r, phase)() + + self.assertLess(order.index("stream.stop"), order.index("collect.collect")) + self.assertLess(order.index("collect.collect"), order.index("stream.teardown")) diff --git a/test/framework/tests/test_detectors.py b/test/framework/tests/test_detectors.py new file mode 100644 index 000000000..1345d5aec --- /dev/null +++ b/test/framework/tests/test_detectors.py @@ -0,0 +1,946 @@ +"""Detector unit tests. + +Detectors are pure functions from Evidence to findings, which is what makes this file +possible: every check below is a handful of synthetic samples rather than a cluster. The +cases are the ones the real runs taught — a healthy single freeze, a retried cutover, a +completed-but-corrupting migration, a verify failure that surfaces after its migration +ended. +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sbtest.core import ( # noqa: E402 + AnaSample, + Attribution, + ControlEvent, + FioJob, + IopsSample, + LogSpan, + Migration, + NvmeController, + Report, + Severity, + SkipDetector, + attribute_window, + build_detector, + freeze_windows, +) + +T0 = datetime(2026, 8, 19, 22, 0, 0, tzinfo=UTC) + + +def ts(sec: int) -> datetime: + return T0 + timedelta(seconds=sec) + + +class FakeEvidence: + """Evidence from literals. Only what a test sets is present; the rest is empty. + + Typed explicitly rather than **kwargs so that mypy checks it against the Evidence + protocol — a test double that has drifted from the real contract is worse than no double, + because every detector test keeps passing while the detector itself has stopped matching. + """ + + def __init__( + self, + run_id: str = "test-run", + outdir: str = "/nonexistent", + migrations: list[Migration] | None = None, + ana: dict[str, list[AnaSample]] | None = None, + jobs: list[FioJob] | None = None, + series: dict[str, list[IopsSample]] | None = None, + fio_logs: dict[str, list[str]] | None = None, + logs: dict[str, list[str]] | None = None, + controllers: list[NvmeController] | None = None, + cluster: str = "", + window: tuple[datetime | None, datetime | None] = (None, None), + events: list[ControlEvent] | None = None, + spans: list[LogSpan] | None = None, + ) -> None: + self.run_id = run_id + self.outdir = outdir + self._migrations = migrations or [] + self._ana = ana or {} + self._jobs = jobs or [] + self._series = series or {} + self._fio_logs = fio_logs or {} + self._logs = logs or {} + self._ctrls = controllers or [] + self._cluster = cluster + self._window = window + self._events = events or [] + self._spans = spans + + def migrations(self) -> list[Migration]: + return list(self._migrations) + + def ana_samples(self, migration: str) -> list[AnaSample]: + return list(self._ana.get(migration, [])) + + def fio_jobs(self) -> list[FioJob]: + return list(self._jobs) + + def fio_timeseries(self, pod: str) -> list[IopsSample]: + return list(self._series.get(pod, [])) + + def fio_log(self, pod: str) -> Iterator[str]: + return iter(self._fio_logs.get(pod, [])) + + def container_logs(self) -> list[str]: + return sorted(self._logs) + + def container_log(self, name: str) -> Iterator[str]: + return iter(self._logs.get(name, [])) + + def nvme_controllers(self) -> list[NvmeController]: + return list(self._ctrls) + + def pods(self) -> list[str]: + return sorted(set(self._fio_logs) | {j.pod for j in self._jobs} | set(self._series)) + + def cluster_uuid(self) -> str: + return self._cluster + + def run_window(self) -> tuple[datetime | None, datetime | None]: + return self._window + + def control_events(self) -> list[ControlEvent]: + return list(self._events) + + def log_spans(self) -> list[LogSpan]: + if self._spans is not None: + return list(self._spans) + return [LogSpan(name=n, first=None, last=None, lines=len(self._logs[n])) + for n in sorted(self._logs)] + + +def ana_series(node: str, address: str, spec: list[tuple[int, str]], nsids=(1, 2), + state="live", role="") -> list[AnaSample]: + """Samples for one path: spec is [(offset_seconds, ana_state), ...].""" + return [AnaSample(ts=ts(off), node=node, address=address, state=state, + ana=dict.fromkeys(nsids, st), role=role) + for off, st in spec] + + +class FreezeWindows(unittest.TestCase): + def test_single_healthy_freeze(self): + s = ana_series("vm03", "10.0.0.113:4430", + [(0, "optimized"), (2, "inaccessible"), (4, "optimized"), + (6, "optimized")]) + w = freeze_windows(s) + self.assertEqual(len(w), 1) + self.assertAlmostEqual(w[0][1], 2.0) + + def test_zero_length_window_is_not_a_freeze(self): + # One sample inaccessible then immediately accessible again at the same instant is + # a sampling artefact, not a window. + s = [AnaSample(ts=ts(0), node="v", address="a", state="live", ana={1: "optimized"}), + AnaSample(ts=ts(0), node="v", address="b", state="live", ana={1: "inaccessible"})] + self.assertEqual(freeze_windows(s), []) + + def test_counts_per_node_not_summed(self): + # Two nodes see the same freeze; the count is one, not two. + s = ana_series("vm02", "a", [(0, "optimized"), (2, "inaccessible"), (4, "optimized")]) + s += ana_series("vm03", "b", [(0, "optimized"), (2, "inaccessible"), (4, "optimized")]) + self.assertEqual(len(freeze_windows(s)), 1) + + def test_trailing_freeze_is_counted(self): + s = ana_series("vm03", "a", [(0, "optimized"), (2, "inaccessible"), (6, "inaccessible")]) + w = freeze_windows(s) + self.assertEqual(len(w), 1) + self.assertAlmostEqual(w[0][1], 4.0) + + +class AnaFreezeCount(unittest.TestCase): + def detector(self, **opts): + return build_detector("ana.freeze-count", **opts) + + def test_one_freeze_is_clean(self): + m = Migration(name="mig-1", start=ts(0), end=ts(10), phase="Completed") + ev = FakeEvidence(migrations=[m], ana={"mig-1": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (4, "optimized")])}) + self.assertEqual(list(self.detector().detect(ev)), []) + + def test_two_freezes_is_critical(self): + """The mig-20 shape: froze, reverted, froze again — and it Completed.""" + m = Migration(name="mig-20", start=ts(0), end=ts(30), phase="Completed") + ev = FakeEvidence(migrations=[m], ana={"mig-20": ana_series( + "vm03", "a", + [(0, "optimized"), (2, "inaccessible"), (8, "optimized"), + (12, "inaccessible"), (18, "optimized")])}) + found = list(self.detector().detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].subject, "mig-20") + self.assertEqual(found[0].evidence["freezes"], 2) + # Phase must not exempt it: a Completed migration can still have lost writes. + self.assertEqual(found[0].evidence["phase"], "Completed") + + def test_threshold_is_configurable(self): + m = Migration(name="mig-20", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"mig-20": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (8, "optimized"), + (12, "inaccessible"), (18, "optimized")])}) + self.assertEqual(list(self.detector(max_freezes=2).detect(ev)), []) + + def test_skips_when_no_samples(self): + ev = FakeEvidence(migrations=[Migration(name="m", start=ts(0))]) + with self.assertRaises(SkipDetector): + list(self.detector().detect(ev)) + + def test_skips_when_no_migrations(self): + with self.assertRaises(SkipDetector): + list(self.detector().detect(FakeEvidence())) + + def test_rejects_unknown_option(self): + with self.assertRaises(ValueError): + build_detector("ana.freeze-count", maxfreezes=2) + + +class AnaCutoverPause(unittest.TestCase): + def test_pause_within_budget_is_clean(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (5, "optimized")])}) + self.assertEqual(list(build_detector("ana.cutover-pause").detect(ev)), []) + + def test_overlong_pause_is_critical(self): + m = Migration(name="m", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (14, "optimized")])}) + found = list(build_detector("ana.cutover-pause").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["worst_pause_s"], 12.0) + + def test_single_long_pause_is_caught_where_freeze_count_is_not(self): + """Why both detectors exist: one window, too long. Count says fine, pause does not.""" + m = Migration(name="m", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (20, "optimized")])}) + self.assertEqual(list(build_detector("ana.freeze-count").detect(ev)), []) + self.assertEqual(len(list(build_detector("ana.cutover-pause").detect(ev))), 1) + + def test_two_short_freezes_are_caught_where_pause_is_not(self): + """And the converse: two design-length windows. Pause says fine, count does not.""" + m = Migration(name="m", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (5, "optimized"), + (10, "inaccessible"), (13, "optimized")])}) + self.assertEqual(list(build_detector("ana.cutover-pause").detect(ev)), []) + self.assertEqual(len(list(build_detector("ana.freeze-count").detect(ev))), 1) + + +class AnaSplitBrain(unittest.TestCase): + def test_both_sides_optimized_is_critical(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + s = ana_series("vm03", "10.0.0.112:4426", [(0, "optimized")], role="source") + s += ana_series("vm03", "10.0.0.114:4428", [(0, "optimized")], role="target") + found = list(build_detector("ana.split-brain").detect( + FakeEvidence(migrations=[m], ana={"m": s}))) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + + def test_target_non_optimized_is_normal_ha_standby(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + s = ana_series("vm03", "10.0.0.112:4426", [(0, "optimized")], role="source") + s += ana_series("vm03", "10.0.0.114:4428", [(0, "non-optimized")], role="target") + self.assertEqual(list(build_detector("ana.split-brain").detect( + FakeEvidence(migrations=[m], ana={"m": s}))), []) + + def test_skips_without_roles(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + s = ana_series("vm03", "a", [(0, "optimized")]) + with self.assertRaises(SkipDetector): + list(build_detector("ana.split-brain").detect( + FakeEvidence(migrations=[m], ana={"m": s}))) + + +class FioChecksum(unittest.TestCase): + LINE = ("2026-08-19T22:00:{sec:02d}.123456789Z stderr F verify: bad magic header 0, " + "wanted acca at file /data/fiotest offset 43999232, length 4096") + + def test_attributes_a_lagged_detection_to_its_migration(self): + """The mig-20 case: the loss surfaces after the migration ended, inside the backlog. + + Without the lag this lands in "outside-any-migration", which is how a + Completed-but-corrupting migration hid. + """ + m = Migration(name="mig-20", start=ts(0), end=ts(10), phase="Completed") + ev = FakeEvidence(migrations=[m], + fio_logs={"fio-16": [self.LINE.format(sec=28)]}) + found = list(build_detector("fio.checksum").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].subject, "mig-20") + self.assertIn("verify backlog", found[0].detail) + + def test_beyond_the_lag_is_not_attributed(self): + m = Migration(name="mig-20", start=ts(0), end=ts(10), phase="Completed") + ev = FakeEvidence(migrations=[m], + fio_logs={"fio-16": [self.LINE.format(sec=59)]}) + found = list(build_detector("fio.checksum", verify_lag_s=5).detect(ev)) + self.assertEqual(found[0].subject, "outside-any-migration") + + def test_groups_blocks_per_migration_and_counts_pods(self): + m = Migration(name="mig-29", start=ts(0), end=ts(10), phase="TIMEOUT") + ev = FakeEvidence(migrations=[m], fio_logs={ + "fio-6": [self.LINE.format(sec=5), self.LINE.format(sec=6)], + "fio-7": [self.LINE.format(sec=7)]}) + found = list(build_detector("fio.checksum").detect(ev)) + self.assertEqual(found[0].evidence["blocks"], 3) + self.assertEqual(found[0].evidence["pods"], {"fio-6": 2, "fio-7": 1}) + + def test_clean_log_yields_nothing(self): + ev = FakeEvidence(fio_logs={"fio-1": ["all good\n"]}) + self.assertEqual(list(build_detector("fio.checksum").detect(ev)), []) + + def test_skips_without_logs(self): + with self.assertRaises(SkipDetector): + list(build_detector("fio.checksum").detect(FakeEvidence(jobs=[FioJob(pod="p")]))) + + +class FioJobError(unittest.TestCase): + def test_eremoteio_carries_the_hint_that_points_at_ana(self): + ev = FakeEvidence(jobs=[FioJob(pod="fio-0", error=121)]) + found = list(build_detector("fio.job-error").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIn("EREMOTEIO", found[0].detail) + + def test_eilseq_is_named_as_corruption_not_an_io_failure(self): + """Linux errno 84. Worth pinning: it is EOVERFLOW on macOS, so a local lookup would + mislabel the one code that means the data was wrong.""" + ev = FakeEvidence(jobs=[FioJob(pod="fio-13", error=84)]) + found = list(build_detector("fio.job-error").detect(ev)) + self.assertIn("EILSEQ", found[0].detail) + self.assertIn("corruption", found[0].detail) + + def test_clean_jobs_yield_nothing(self): + ev = FakeEvidence(jobs=[FioJob(pod="fio-0", error=0)]) + self.assertEqual(list(build_detector("fio.job-error").detect(ev)), []) + + def test_ignore_list(self): + ev = FakeEvidence(jobs=[FioJob(pod="fio-0", error=121)]) + self.assertEqual(list(build_detector("fio.job-error", ignore_errnos=[121]).detect(ev)), []) + + +class FioOutage(unittest.TestCase): + def series(self, pattern: str) -> list[IopsSample]: + # "1" = doing I/O, "0" = stopped; one character per second. + return [IopsSample(offset_s=i, wall=ts(i), total_iops=100.0 if c == "1" else 0.0) + for i, c in enumerate(pattern)] + + def test_short_dip_is_not_an_outage(self): + ev = FakeEvidence(series={"p": self.series("1" * 10 + "0" * 5 + "1" * 10)}) + self.assertEqual(list(build_detector("fio.outage", min_seconds=30).detect(ev)), []) + + def test_sustained_stop_is_critical(self): + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40 + "1" * 5)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["worst_s"], 40) + self.assertEqual(found[0].evidence["downtime_s"], 40) + + def test_trailing_outage_is_reported(self): + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40)}) + self.assertEqual(len(list(build_detector("fio.outage", min_seconds=30).detect(ev))), 1) + + def test_many_windows_collapse_into_one_finding_per_pod(self): + """A repeatedly stalling pod stalls hundreds of times in a soak — one archived run + produced 781 qualifying windows. Per-window findings make the report unreadable, so + the count and the total are the finding and the worst windows are named.""" + # Trailing "1" so all 20 windows are closed by a resumption: a window still open at + # the end of the series is measured to the last sample, which is one second short. + ev = FakeEvidence(series={"p": self.series(("1" * 5 + "0" * 31) * 20 + "1")}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["windows"], 20) + self.assertEqual(found[0].evidence["downtime_s"], 20 * 31) + self.assertIn("20x", found[0].title) + + def test_windows_outside_any_migration_say_so(self): + """Attribution is the difference between "migration cost this" and "the cluster is + unwell", and the note must not imply the former when it was neither.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["migrations"], []) + self.assertIn("elsewhere", found[0].note) + + def test_a_gap_that_recovered_is_a_freeze_not_a_loss(self): + """A cutover is a freeze by design: every write was eventually taken. It still fails + the run at this length, but calling it loss says data went missing when none did.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40 + "1" * 5)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["kind"], "freeze") + self.assertIn("froze", found[0].title) + self.assertIn("no I/O was lost", found[0].note) + self.assertTrue(found[0].evidence["worst_windows"][0]["recovered"]) + + def test_a_gap_still_open_when_fio_stopped_is_a_loss(self): + """Nothing observed the volume come back, so this is I/O it was supposed to accept + and never did — the finding a reader must not have to dig for.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["kind"], "loss") + self.assertIn("LOSS", found[0].title) + self.assertFalse(found[0].evidence["worst_windows"][0]["recovered"]) + + def test_a_pod_with_both_reports_the_loss_first(self): + """Both fail the run; the order they are emitted in is the order they are read in, + and a gap that never closed outranks one that did.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40 + "1" * 5 + "0" * 40)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual([f.evidence["kind"] for f in found], ["loss", "freeze"]) + self.assertEqual([f.evidence["windows"] for f in found], [1, 1]) + + def test_a_gap_beginning_before_the_migration_still_belongs_to_it(self): + """The host goes dry before the operator records the migration as started. Testing + only the window's first second files those gaps under no migration at all.""" + mig = Migration(name="mig-1", start=ts(20), end=ts(60)) + ev = FakeEvidence(series={"p": self.series("1" * 10 + "0" * 40 + "1")}, + migrations=[mig]) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["migrations"], ["mig-1"]) + + def test_a_window_spanning_two_migrations_names_the_one_holding_most_of_it(self): + ev = FakeEvidence( + series={"p": self.series("1" * 10 + "0" * 40 + "1")}, + migrations=[Migration(name="mig-a", start=ts(5), end=ts(20)), + Migration(name="mig-b", start=ts(30), end=ts(70))]) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["migrations"], ["mig-b"]) + + +class AttributeWindow(unittest.TestCase): + """The primitive behind outage attribution: a symptom that lasted, not one that fired.""" + + def named(self, migs: list[Migration], start: datetime, end: datetime) -> str: + m = attribute_window(migs, start, end) + return m.name if m else "" + + def test_a_window_wholly_inside_a_migration(self): + migs = [Migration(name="m", start=ts(0), end=ts(100))] + self.assertEqual(self.named(migs, ts(10), ts(20)), "m") + + def test_a_window_that_misses_every_migration(self): + migs = [Migration(name="m", start=ts(0), end=ts(10))] + self.assertIsNone(attribute_window(migs, ts(20), ts(30))) + + def test_touching_counts_as_overlapping(self): + """Zero is an overlap: a gap that begins the second a migration ends is the + migration's, and the sampling interval must not decide that.""" + migs = [Migration(name="m", start=ts(0), end=ts(10))] + self.assertEqual(self.named(migs, ts(10), ts(50)), "m") + + def test_the_largest_overlap_wins_not_the_first(self): + migs = [Migration(name="early", start=ts(0), end=ts(15)), + Migration(name="late", start=ts(20), end=ts(60))] + self.assertEqual(self.named(migs, ts(10), ts(50)), "late") + + def test_a_running_migration_is_a_point_in_time(self): + """`end` is None while it is in flight — the same convention `covers` uses.""" + migs = [Migration(name="m", start=ts(30), end=None)] + self.assertEqual(self.named(migs, ts(10), ts(50)), "m") + + +class SecretExposureArtifacts(unittest.TestCase): + def test_the_artifacts_are_the_files_a_reader_can_open(self): + """Regression: 2026-08-26-secret-artifacts-miss-the-extension (PR #445 review). + + The finding listed log *names* ("operator") where the run directory holds + "operator.txt", so the one field that says where to look did not name a file. Every + other detector reporting a container log names it with its extension. + """ + ev = FakeEvidence(logs={ + "operator": ["password=hunter2hunter2"], + "webappapi": ["nothing to see"]}) + found = list(build_detector("security.secret-exposure").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].artifacts, ["operator.txt"]) + + +class NvmeStaleControllers(unittest.TestCase): + def ctrl(self, name, state, ns, node="vm03", nqn="nqn:lvol:x", addr="10.0.0.1:4420", clt=60): + return NvmeController(node=node, name=name, nqn=nqn, address=addr, state=state, + namespaces=ns, ctrl_loss_tmo=clt) + + def test_live_with_no_namespace_is_critical(self): + """The state that blocked every later migration of a subsystem.""" + ev = FakeEvidence(controllers=[ + self.ctrl("nvme12", "live", {}), + self.ctrl("nvme13", "live", {1: "optimized"}, addr="10.0.0.2:4420")]) + found = [f for f in build_detector("nvme.stale-controllers").detect(ev) + if f.severity == Severity.CRITICAL] + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["count"], 1) + + def test_connecting_is_only_a_warning(self): + # A snapshot cannot tell a leak from a normal HA reconnect. + ev = FakeEvidence(controllers=[self.ctrl("nvme6", "connecting", {})]) + sevs = {f.severity for f in build_detector("nvme.stale-controllers").detect(ev)} + self.assertNotIn(Severity.CRITICAL, sevs) + self.assertIn(Severity.WARNING, sevs) + + def test_healthy_fabric_is_clean(self): + ev = FakeEvidence(controllers=[ + self.ctrl("nvme1", "live", {1: "optimized"}, addr="10.0.0.1:4420"), + self.ctrl("nvme2", "live", {1: "non-optimized"}, addr="10.0.0.2:4420")]) + self.assertEqual(list(build_detector("nvme.stale-controllers").detect(ev)), []) + + def test_skips_without_a_snapshot(self): + with self.assertRaises(SkipDetector): + list(build_detector("nvme.stale-controllers").detect(FakeEvidence())) + + def test_loss_timeout_flags_a_value_that_outlives_the_run(self): + ev = FakeEvidence(controllers=[self.ctrl("nvme1", "live", {1: "optimized"}, clt=3600)]) + found = list(build_detector("nvme.loss-timeout").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["values"], [3600]) + + +class LogPattern(unittest.TestCase): + def test_catalogue_catches_the_undrained_transfer(self): + ev = FakeEvidence(logs={"spdk-4422": [ + "transfer task failed: ----- but still have outstanding io 1\n"] * 3}) + found = [f for f in build_detector("logs.pattern").detect(ev) + if f.subject == "spdk.undrained-transfer"] + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].evidence["total"], 3) + + def test_min_count_suppresses_a_line_that_is_normal_in_small_numbers(self): + ev = FakeEvidence(logs={"spdk-4422": ["does not allow host X to connect at this address\n"]}) + found = [f for f in build_detector("logs.pattern").detect(ev) + if f.subject == "nvme.host-not-allowed"] + self.assertEqual(found, []) # catalogue min_count is 50 + + def test_user_defined_pattern_replaces_the_catalogue(self): + ev = FakeEvidence(logs={"mylog": ["something odd happened\n"]}) + d = build_detector("logs.pattern", patterns=[ + {"id": "my.check", "regex": "something odd", "logs": ["mylog"], + "severity": "critical"}]) + found = list(d.detect(ev)) + self.assertEqual([f.subject for f in found], ["my.check"]) + + def test_log_glob_scopes_the_pattern(self): + ev = FakeEvidence(logs={"spdk-4420": ["boom\n"], "operator": ["boom\n"]}) + d = build_detector("logs.pattern", patterns=[ + {"id": "only.spdk", "regex": "boom", "logs": ["spdk-*"], "severity": "warning"}]) + found = list(d.detect(ev)) + self.assertEqual(found[0].evidence["per_log"], {"spdk-4420": 1}) + + def test_rejects_a_malformed_pattern(self): + ev = FakeEvidence(logs={"x": ["y"]}) + d = build_detector("logs.pattern", patterns=[{"id": "a", "rgex": "y"}]) + with self.assertRaises(ValueError): + list(d.detect(ev)) + + def test_skips_without_logs(self): + with self.assertRaises(SkipDetector): + list(build_detector("logs.pattern").detect(FakeEvidence())) + + +class MigrationOutcomes(unittest.TestCase): + def test_low_completion_is_critical(self): + migs = ([Migration(name=f"m{i}", start=ts(i), phase="TIMEOUT") for i in range(25)] + + [Migration(name=f"c{i}", start=ts(100 + i), phase="Completed") for i in range(13)]) + found = list(build_detector("migration.outcomes").detect(FakeEvidence(migrations=migs))) + crit = [f for f in found if f.severity == Severity.CRITICAL] + self.assertEqual(len(crit), 2) # completion rate and timeout rate + self.assertTrue(any("timed out" in f.title for f in crit)) + + def test_healthy_run_reports_info_only(self): + migs = [Migration(name=f"c{i}", start=ts(i), phase="Completed") for i in range(10)] + found = list(build_detector("migration.outcomes").detect(FakeEvidence(migrations=migs))) + self.assertTrue(all(f.severity == Severity.INFO for f in found)) + + def test_errors_group_by_shape(self): + migs = [Migration(name=f"m{i}", start=ts(i), phase="Failed", + error=f"NVMe path validation failed on node vm0{i}; cancelled") + for i in range(3)] + found = list(build_detector("migration.errors").detect(FakeEvidence(migrations=migs))) + self.assertEqual(len(found), 1) # three messages, one shape + self.assertEqual(found[0].evidence["count"], 3) + + +if __name__ == "__main__": + unittest.main(verbosity=2) + + +def dmesg(*msgs: str, day: int = 20, start: int = 0) -> list[str]: + """dmesg -T lines. Local time, which is why these detectors do not attribute to migrations.""" + return [f"[Thu Aug {day} 05:{46 + (start + i) // 60:02d}:{(start + i) % 60:02d} 2026] {m}\n" + for i, m in enumerate(msgs)] + + +class KernelPathLoss(unittest.TestCase): + """The ladder: requeue (absorbed) -> failfast -> failing I/O (application-visible).""" + + def test_requeue_only_is_a_warning_not_a_failure(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "block nvme0n1: no usable path - requeuing I/O", + "block nvme0n1: no usable path - requeuing I/O")}) + found = list(build_detector("kernel.path-loss").detect(ev)) + self.assertEqual([f.severity for f in found], [Severity.WARNING]) + self.assertEqual(found[0].evidence["requeues"], 2) + + def test_failing_io_is_critical(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "block nvme0n1: no usable path - requeuing I/O", + "nvme nvme10: failfast expired", + "block nvme0n1: no available path - failing I/O")}) + found = list(build_detector("kernel.path-loss").detect(ev)) + crit = [f for f in found if f.severity == Severity.CRITICAL] + self.assertEqual(len(crit), 1) + self.assertEqual(crit[0].evidence["failing_io"], 1) + + def test_failfast_is_reported_separately_as_the_boundary(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg("nvme nvme10: failfast expired")}) + found = list(build_detector("kernel.path-loss").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIn("fast_io_fail_tmo", found[0].title) + + def test_clean_dmesg_yields_nothing(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg("nvme nvme1: creating 3 I/O queues.")}) + self.assertEqual(list(build_detector("kernel.path-loss").detect(ev)), []) + + def test_skips_without_dmesg(self): + with self.assertRaises(SkipDetector): + list(build_detector("kernel.path-loss").detect(FakeEvidence())) + + +class KernelFilesystemShutdown(unittest.TestCase): + def test_shutdown_is_critical_and_names_the_devices(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "XFS (nvme0n1): log I/O error -5", + "XFS (nvme0n1): Filesystem has been shut down due to log error (0x2).", + "XFS (nvme1n6): Filesystem has been shut down due to log error (0x2).")}) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].evidence["devices"], ["nvme0n1", "nvme1n6"]) + + def test_io_errors_without_a_shutdown_are_only_a_warning(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg("XFS (nvme0n1): metadata I/O error")}) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual([f.severity for f in found], [Severity.WARNING]) + + def test_healthy_mount_messages_are_not_findings(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "XFS (nvme0n1): Ending clean mount", + "XFS (nvme0n1): Unmounting Filesystem abc")}) + self.assertEqual(list(build_detector("kernel.filesystem-shutdown").detect(ev)), []) + + +class NvmeForeignCluster(unittest.TestCase): + LIVE = "d26b8f37-2b45-47c0-9d20-983e6c5ee3fe" + DEAD = "5fd9ad70-3cd1-4fc8-b8e6-e085081601f6" + + def test_a_dead_cluster_being_retried_is_reported_as_hygiene(self): + """The sharpest form of 'controllers never disappear': they outlive the cluster. + + Reported, but never as this run's failure — a controller for a destroyed cluster + cannot affect a migration of the live cluster's subsystems. See NvmeDirtyStart for + the leak that does invalidate a run. + """ + ev = FakeEvidence(cluster=self.LIVE, logs={"dmesg-vm03": dmesg( + f'nvme nvme6: Connect Invalid Data Parameter, subsysnqn "nqn.2023-02.io.simplyblock:{self.DEAD}:lvol:x"', + f'nvme nvme6: Connect Invalid Data Parameter, subsysnqn "nqn.2023-02.io.simplyblock:{self.DEAD}:lvol:x"', + f'nvme nvme7: connected to nqn.2023-02.io.simplyblock:{self.LIVE}:lvol:y')}) + found = list(build_detector("nvme.foreign-cluster").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.WARNING) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + self.assertFalse(found[0].counts_against_the_run) + self.assertEqual(found[0].evidence["foreign"], {self.DEAD: 2}) + self.assertEqual(found[0].evidence["live_cluster"], self.LIVE) + + def test_only_the_live_cluster_is_clean(self): + ev = FakeEvidence(cluster=self.LIVE, logs={"dmesg-vm03": dmesg( + f'nvme nvme7: connected to nqn.2023-02.io.simplyblock:{self.LIVE}:lvol:y')}) + self.assertEqual(list(build_detector("nvme.foreign-cluster").detect(ev)), []) + + def test_skips_when_the_live_cluster_is_unknown(self): + """Without it a foreign NQN cannot be told from the live one, so do not guess.""" + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + f'nvme nvme6: nqn.2023-02.io.simplyblock:{self.DEAD}:lvol:x')}) + with self.assertRaises(SkipDetector): + list(build_detector("nvme.foreign-cluster").detect(ev)) + + +class NvmeControllerChurn(unittest.TestCase): + def test_created_but_never_removed_is_reported(self): + msgs = [f"nvme nvme{i}: new ctrl: NQN \"nqn.x\"" for i in range(8)] + msgs.append("nvme nvme0: Removing ctrl: NQN \"nqn.x\"") + found = list(build_detector("nvme.controller-churn").detect( + FakeEvidence(logs={"dmesg-vm03": dmesg(*msgs)}))) + churn = [f for f in found if "never removed" in f.title] + self.assertEqual(len(churn), 1) + self.assertEqual(churn[0].evidence["net"], 7) + + def test_balanced_churn_is_only_info(self): + msgs = ["nvme nvme0: new ctrl: NQN \"nqn.x\"", "nvme nvme0: Removing ctrl: NQN \"nqn.x\""] + found = list(build_detector("nvme.controller-churn").detect( + FakeEvidence(logs={"dmesg-vm03": dmesg(*msgs)}))) + self.assertTrue(all(f.severity == Severity.INFO for f in found)) + + def test_a_controller_retrying_forever_is_reported(self): + found = list(build_detector("nvme.controller-churn").detect( + FakeEvidence(logs={"dmesg-vm03": dmesg( + "nvme nvme9: Failed reconnect attempt 5", + "nvme nvme9: Failed reconnect attempt 834")}))) + stuck = [f for f in found if "without ever succeeding" in f.title] + self.assertEqual(len(stuck), 1) + self.assertEqual(stuck[0].evidence["controllers"], {"nvme9": 834}) + + +class KernelFabricErrors(unittest.TestCase): + def test_groups_by_kind_and_respects_per_kind_floors(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "nvme nvme1: starting error recovery", + "nvme nvme1: Property Set error: 880, offset 0x14", + "nvme nvme2: rescanning namespaces.")}) + found = list(build_detector("kernel.fabric-errors").detect(ev)) + self.assertEqual(len(found), 1) + counts = found[0].evidence["counts"] + # a single rescan is normal and must not be reported; the two errors must be + self.assertNotIn("namespace rescan", counts) + self.assertIn("error recovery started", counts) + self.assertIn("property set failed (controller config)", counts) + + +class Attribution_(unittest.TestCase): + """Old, unrelated damage must not be counted against a run. + + dmesg spans hours and a cluster outlives its runs, so evidence routinely contains the + previous runs' mess. These pin the rule: only what happened inside the window counts. + """ + + RUN_START = datetime(2026, 8, 20, 6, 0, 0, tzinfo=UTC) + RUN_END = datetime(2026, 8, 20, 7, 0, 0, tzinfo=UTC) + + def ev(self, *msgs: str, hour: int = 6) -> FakeEvidence: + lines = [f"2026-08-20T{hour:02d}:30:0{i},000000+00:00 {m}\n" for i, m in enumerate(msgs)] + return FakeEvidence(logs={"dmesg-vm03": lines}, + window=(self.RUN_START, self.RUN_END)) + + def test_damage_inside_the_window_is_critical_and_fails(self): + ev = self.ev("block nvme0n1: no available path - failing I/O", hour=6) + found = [f for f in build_detector("kernel.path-loss").detect(ev) + if f.severity == Severity.CRITICAL] + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, Attribution.RUN) + self.assertTrue(found[0].counts_against_the_run) + + def test_the_same_damage_before_the_window_does_not_fail(self): + ev = self.ev("block nvme0n1: no available path - failing I/O", hour=5) + found = list(build_detector("kernel.path-loss").detect(ev)) + self.assertTrue(found) + self.assertTrue(all(f.attribution is Attribution.PRE_EXISTING for f in found)) + self.assertFalse(any(f.severity == Severity.CRITICAL for f in found)) + self.assertFalse(any(f.counts_against_the_run for f in found)) + + def test_a_filesystem_killed_before_the_run_is_hygiene_not_failure(self): + ev = self.ev("XFS (nvme0n1): Filesystem has been shut down due to log error (0x2).", + hour=5) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + self.assertEqual(found[0].severity, Severity.WARNING) + + def test_a_filesystem_killed_during_the_run_is_critical(self): + ev = self.ev("XFS (nvme0n1): Filesystem has been shut down due to log error (0x2).", + hour=6) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertIs(found[0].attribution, Attribution.RUN) + + def test_undated_events_still_count(self): + """'I cannot date this' must not become 'not our problem'.""" + ev = FakeEvidence(logs={"dmesg-vm03": ["block nvme0n1: no available path - failing I/O\n"]}, + window=(self.RUN_START, self.RUN_END)) + found = [f for f in build_detector("kernel.path-loss").detect(ev) + if f.severity == Severity.CRITICAL] + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, Attribution.UNKNOWN) + self.assertTrue(found[0].counts_against_the_run) + + def test_dead_cluster_debris_is_hygiene_never_a_verdict(self): + """It cannot make a different cluster's migration fail, so it must not fail the run.""" + live, dead = "d26b8f37-2b45-47c0-9d20-983e6c5ee3fe", "5fd9ad70-3cd1-4fc8-b8e6-e085081601f6" + ev = FakeEvidence(cluster=live, window=(self.RUN_START, self.RUN_END), logs={ + "dmesg-vm03": [f'nvme nvme6: subsysnqn "nqn.2023-02.io.simplyblock:{dead}:lvol:x"\n'] * 100}) + found = list(build_detector("nvme.foreign-cluster").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.WARNING) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + + +class NvmeDirtyStart(unittest.TestCase): + """The one pre-existing condition that does forfeit a run.""" + + LIVE = "d26b8f37-2b45-47c0-9d20-983e6c5ee3fe" + DEAD = "5fd9ad70-3cd1-4fc8-b8e6-e085081601f6" + + class WithPre(FakeEvidence): + """FakeEvidence plus the optional pre-run snapshot nvme.dirty-start asks for.""" + + def __init__(self, pre: list[NvmeController], cluster: str = "") -> None: + super().__init__(cluster=cluster) + self._pre = pre + + def nvme_controllers_pre(self) -> list[NvmeController]: + return self._pre + + def ctrl(self, nqn: str, state: str, ns: dict) -> NvmeController: + return NvmeController(node="vm03", name="nvme12", nqn=nqn, address="10.0.0.1:4420", + state=state, namespaces=ns, ctrl_loss_tmo=60) + + def test_live_cluster_debris_at_setup_forfeits_the_run(self): + ev = self.WithPre([self.ctrl(f"nqn:{self.LIVE}:lvol:x", "live", {})], + cluster=self.LIVE) + found = list(build_detector("nvme.dirty-start").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + # critical + pre-existing => the run is inconclusive, not failed + rep = Report() + rep.add(*found) + self.assertEqual(rep.verdict, "INCONCLUSIVE") + self.assertFalse(rep.failed) + + def test_dead_cluster_debris_at_setup_does_not_forfeit(self): + ev = self.WithPre([self.ctrl(f"nqn:{self.DEAD}:lvol:x", "live", {})], + cluster=self.LIVE) + self.assertEqual(list(build_detector("nvme.dirty-start").detect(ev)), []) + + def test_a_connecting_controller_does_not_forfeit(self): + ev = self.WithPre([self.ctrl(f"nqn:{self.LIVE}:lvol:x", "connecting", {})], + cluster=self.LIVE) + self.assertEqual(list(build_detector("nvme.dirty-start").detect(ev)), []) + + def test_skips_without_a_pre_snapshot(self): + with self.assertRaises(SkipDetector): + list(build_detector("nvme.dirty-start").detect(FakeEvidence(cluster=self.LIVE))) + + +# ── control-plane and evidence families ───────────────────────────────────────────── + +def cevent(sec: int, msg: str, subject: str = "node-a", level: str = "Info") -> ControlEvent: + return ControlEvent(ts=ts(sec), level=level, kind="STATUS_CHANGE", message=msg, + subject=subject) + + +class ControlNodeFlap(unittest.TestCase): + """The vela shape: a node marked down and back in seconds because the liveness check + depended on something other than the node.""" + + def test_a_short_flap_is_critical(self): + ev = FakeEvidence(events=[ + cevent(0, "Storage node status changed from: online to: down"), + cevent(13, "Storage node status changed from: down to: online")]) + found = list(build_detector("control.node-flap").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].evidence["flaps"][0]["seconds"], 13) + + def test_a_node_that_stays_down_is_not_a_flap(self): + ev = FakeEvidence(events=[ + cevent(0, "Storage node status changed from: online to: down")]) + self.assertEqual(list(build_detector("control.node-flap").detect(ev)), []) + + def test_a_slow_recovery_is_not_a_flap(self): + ev = FakeEvidence(events=[ + cevent(0, "Storage node status changed from: online to: down"), + cevent(9000, "Storage node status changed from: down to: online")]) + self.assertEqual(list(build_detector("control.node-flap").detect(ev)), []) + + def test_skips_without_the_event_log(self): + with self.assertRaises(SkipDetector): + list(build_detector("control.node-flap").detect(FakeEvidence())) + + +class ControlVolumeHealth(unittest.TestCase): + def test_health_that_never_returns_is_critical(self): + ev = FakeEvidence(events=[ + cevent(0, "LVol health check changed from: True to: False", subject="vol-1"), + cevent(5, "LVol health check changed from: True to: False", subject="vol-2"), + cevent(60, "LVol health check changed from: False to: True", subject="vol-2")]) + found = list(build_detector("control.volume-health").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["unhealthy"], ["vol-1"]) + + def test_all_recovered_is_clean(self): + ev = FakeEvidence(events=[ + cevent(0, "LVol health check changed from: True to: False", subject="vol-1"), + cevent(60, "LVol health check changed from: False to: True", subject="vol-1")]) + self.assertEqual(list(build_detector("control.volume-health").detect(ev)), []) + + +class EvidenceCoverage(unittest.TestCase): + """Partial evidence is not the same as clean evidence.""" + + START = datetime(2026, 8, 20, 7, 0, 0, tzinfo=UTC) + END = datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC) + + def test_a_log_that_misses_the_start_is_reported(self): + ev = FakeEvidence(window=(self.START, self.END), spans=[ + LogSpan("spdk-4424", self.START + timedelta(minutes=114), self.END, 100), + LogSpan("operator", self.START, self.END, 100)]) + found = list(build_detector("evidence.log-coverage").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIn("spdk-4424", found[0].evidence["logs"]) + self.assertNotIn("operator", found[0].evidence["logs"]) + + def test_full_coverage_is_clean(self): + ev = FakeEvidence(window=(self.START, self.END), spans=[ + LogSpan("operator", self.START, self.END, 100)]) + self.assertEqual(list(build_detector("evidence.log-coverage").detect(ev)), []) + + def test_skips_without_a_run_window(self): + ev = FakeEvidence(spans=[LogSpan("x", self.START, self.END, 1)]) + with self.assertRaises(SkipDetector): + list(build_detector("evidence.log-coverage").detect(ev)) + + def test_a_migration_no_log_covers_is_a_blind_spot(self): + """The real case: the corrupting migration ended six seconds before a log began.""" + mig_start = self.START + timedelta(minutes=110) + ev = FakeEvidence( + window=(self.START, self.END), + migrations=[Migration(name="mig-19", start=mig_start, + end=mig_start + timedelta(minutes=2))], + spans=[LogSpan("spdk-4424", mig_start + timedelta(minutes=2, seconds=6), + self.END, 100)]) + found = list(build_detector("evidence.blind-spot").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["blind"], {"mig-19": ["spdk-4424"]}) + + +class SecuritySecretExposure(unittest.TestCase): + def test_finds_a_private_key_without_quoting_it(self): + secret = "-----BEGIN RSA PRIVATE KEY-----" + ev = FakeEvidence(logs={"operator": [f"oops {secret} MIIEow\n"]}) + found = list(build_detector("security.secret-exposure").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + blob = json.dumps(found[0].to_dict()) + self.assertNotIn("BEGIN RSA", blob) # the value must not travel with the finding + self.assertIn("operator:1", found[0].detail) + + def test_finds_a_dhchap_secret(self): + ev = FakeEvidence(logs={"operator": [ + "connect --dhchap-secret DHHC-1:00:abcdefghijklmnopqrstuvwxyz012345+/=\n"]}) + found = list(build_detector("security.secret-exposure").detect(ev)) + self.assertEqual([f.subject for f in found], ["nvme dhchap secret"]) + + def test_ordinary_logs_are_clean(self): + ev = FakeEvidence(logs={"operator": ["migration started for volume abc\n"]}) + self.assertEqual(list(build_detector("security.secret-exposure").detect(ev)), []) diff --git a/test/framework/tests/test_driver.py b/test/framework/tests/test_driver.py new file mode 100644 index 000000000..1e60296bd --- /dev/null +++ b/test/framework/tests/test_driver.py @@ -0,0 +1,442 @@ +"""Driver and workload tests. + +These cover the decisions the driver makes on its own — which node to migrate to, which +volumes move together, what a non-terminal poll means — because those are the parts that +silently produce a *valid-looking* run when they are wrong. A driver that always picks the +same target still completes migrations and still reports PASS; the run just never exercised +the case it claimed to. + +Everything here fakes kubectl at the module boundary. That is the whole point of routing +cluster access through one `kube.run`: the logic above it stays testable without a cluster. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from datetime import UTC, datetime, timedelta + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import sbtest # noqa: E402,F401 +from sbtest.components import kube, migration, workload # noqa: E402 +from sbtest.core import Logger, Migration, RunContext # noqa: E402 + + +def _cp(stdout: str = "", rc: int = 0) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(args=["kubectl"], returncode=rc, stdout=stdout, + stderr="") + + +def _nodes_json(*specs: tuple[str, str, str]) -> str: + """specs: (uuid, k8s host, status).""" + return json.dumps({"items": [ + {"spec": {"workerNode": host}, "status": {"uuid": uuid, "status": status}} + for uuid, host, status in specs]}) + + +class _FakeKube: + """Records every kubectl invocation and answers from a scripted table.""" + + def __init__(self, answers: dict[str, str] | None = None) -> None: + self.calls: list[list[str]] = [] + self.stdins: list[str | None] = [] + self.answers = answers or {} + + def run(self, args: list[str], timeout: int = 60, check: bool = True, + stdin: str | None = None) -> subprocess.CompletedProcess[str]: + self.calls.append(list(args)) + self.stdins.append(stdin) + for key, out in self.answers.items(): + if key in " ".join(args): + return _cp(out) + return _cp("") + + +class _Ctx: + """A RunContext with a real temp dir, for components that write artifacts.""" + + def __enter__(self) -> RunContext: + self._d = tempfile.TemporaryDirectory() + self.ctx = RunContext(run_id="run1", outdir=self._d.__enter__(), log=Logger(None)) + return self.ctx + + def __exit__(self, *exc: object) -> None: + self._d.__exit__(*exc) # type: ignore[arg-type] + + +class TargetPolicy(unittest.TestCase): + """The policy decides whether the target also hosts a consumer — the harder case.""" + + def _driver(self, ctx: RunContext, policy: str, + consumers: dict[str, list[str]]) -> migration.MigrationDriver: + d = migration.MigrationDriver(target_policy=policy) + d._nodes = ["uuid-a", "uuid-b", "uuid-c"] + d._node_host = {"uuid-a": "vm02", "uuid-b": "vm03", "uuid-c": "vm04"} + # pv -> pod -> node, which is how the driver learns where consumers run. + ctx.shared["workload.pod_of"] = {"pv1": "fio-0"} + ctx.shared["workload.node_of"] = {"fio-0": next(iter(consumers), "")} + return d + + def test_consumer_policy_picks_a_node_running_a_consumer(self): + with _Ctx() as ctx: + d = self._driver(ctx, "consumer", {"vm03": ["fio-0"]}) + target, policy, named = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertEqual(target, "uuid-b") # vm03, where fio-0 runs + self.assertEqual(policy, "consumer") + self.assertEqual(named, ["fio-0"]) + + def test_no_consumer_policy_avoids_it(self): + with _Ctx() as ctx: + d = self._driver(ctx, "no-consumer", {"vm03": ["fio-0"]}) + target, policy, named = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertEqual(target, "uuid-c") # vm04: the only other non-consumer + self.assertEqual(policy, "no-consumer") + self.assertEqual(named, []) + + def test_alternate_starts_with_the_harder_case(self): + """Odd migrations get `consumer`, so a run cut short still exercised it.""" + with _Ctx() as ctx: + d = self._driver(ctx, "alternate", {"vm03": ["fio-0"]}) + _, first, _ = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + _, second, _ = d._pick_target(ctx, ["pv1"], idx=2, source="uuid-a") + self.assertEqual(first, "consumer") + self.assertEqual(second, "no-consumer") + + def test_unmet_policy_falls_back_and_says_so(self): + """A migration under the other condition is still evidence; skipping it is not. + + The recorded policy has to show it was unmet, or the run's own record claims a case + it never exercised. + """ + with _Ctx() as ctx: + d = self._driver(ctx, "consumer", {}) # no consumer anywhere + target, policy, _ = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertIn(target, ("uuid-b", "uuid-c")) + self.assertEqual(policy, "consumer(unmet)") + + def test_the_source_is_never_the_target(self): + with _Ctx() as ctx: + d = self._driver(ctx, "random", {}) + for _ in range(20): + target, _, _ = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertNotEqual(target, "uuid-a") + + def test_consumers_are_counted_across_the_whole_subsystem(self): + """Every pod holding any namespace of the subsystem has its paths moved.""" + with _Ctx() as ctx: + d = migration.MigrationDriver(target_policy="consumer") + d._nodes = ["uuid-a", "uuid-b"] + d._node_host = {"uuid-a": "vm02", "uuid-b": "vm03"} + ctx.shared["workload.pod_of"] = {"pv1": "fio-0", "pv2": "fio-1"} + ctx.shared["workload.node_of"] = {"fio-0": "vm02", "fio-1": "vm03"} + _, _, named = d._pick_target(ctx, ["pv1", "pv2"], idx=1, source="uuid-a") + self.assertEqual(named, ["fio-1"]) # the sibling on the target counts + + +class Grouping(unittest.TestCase): + """A migration moves the whole subsystem, so the group is what must be tracked.""" + + def test_group_is_the_shared_subsystem(self): + d = migration.MigrationDriver() + d._nqn_of = {"pv1": "nqn.a", "pv2": "nqn.a", "pv3": "nqn.b"} + d._groups = d._regroup() + self.assertEqual(d._group_of("pv1"), ["pv1", "pv2"]) + self.assertEqual(d._group_of("pv3"), ["pv3"]) + + def test_an_unknown_subsystem_migrates_alone(self): + d = migration.MigrationDriver() + self.assertEqual(d._group_of("pv9"), ["pv9"]) + + def test_a_changed_subsystem_regroups(self): + """A previous migration can repack the subsystems; a stale group samples the wrong + nodes and verifies the wrong volumes.""" + class FakeSb: + def subsystem_of(self, lvol: str) -> tuple[str, int]: + return "nqn.new", 1 + + with _Ctx() as ctx: + d = migration.MigrationDriver() + d._sb = FakeSb() # type: ignore[assignment] + d._volume_of = {"pv1": "lvol-1"} + d._nqn_of = {"pv1": "nqn.old", "pv2": "nqn.old"} + d._groups = d._regroup() + self.assertEqual(d._group_of("pv1"), ["pv1", "pv2"]) + d._reread_subsystem(ctx, "pv1") + self.assertEqual(d._nqn_of["pv1"], "nqn.new") + self.assertEqual(d._group_of("pv1"), ["pv1"]) # no longer with pv2 + + +class PollLoop(unittest.TestCase): + def test_terminal_phase_ends_the_wait(self): + fake = _FakeKube({"get volumemigration": json.dumps( + {"status": {"phase": "Completed", "sourceNodeUUID": "uuid-real"}})}) + with _Ctx() as ctx, _patch(kube, "run", fake.run): + d = migration.MigrationDriver(poll_s=0.01) + rec = Migration(name="m1", start=datetime.now(UTC), pv="pv1", source="uuid-guess") + d._await_terminal(ctx, rec, "m1", sampler=None) + self.assertEqual(rec.phase, "Completed") + self.assertIsNotNone(rec.end) + # The operator's resolved source overrides the driver's guess. + self.assertEqual(rec.source, "uuid-real") + + def test_never_reaching_a_terminal_phase_is_a_timeout_not_a_failure(self): + """A rejected migration and one that never finished are different defects, and only + one of them has an error to read.""" + fake = _FakeKube({"get volumemigration": json.dumps( + {"status": {"phase": "Migrating"}})}) + with _Ctx() as ctx, _patch(kube, "run", fake.run): + d = migration.MigrationDriver(poll_s=0.01, timeout_s=0.05) + rec = Migration(name="m1", start=datetime.now(UTC), pv="pv1") + d._await_terminal(ctx, rec, "m1", sampler=None) + self.assertEqual(rec.phase, "TIMEOUT") + self.assertEqual(rec.error, "") + + def test_phase_changes_reach_the_sampler_and_the_timeline(self): + class Sampler: + def __init__(self) -> None: + self.phases: list[str] = [] + + def set_phase(self, p: str) -> None: + self.phases.append(p) + + seq = [json.dumps({"status": {"phase": p}}) + for p in ("Preparing", "Preparing", "Cutover", "Completed")] + + def run(args: list[str], timeout: int = 60, check: bool = True, + stdin: str | None = None) -> subprocess.CompletedProcess[str]: + return _cp(seq.pop(0) if seq else "") + + s = Sampler() + with _Ctx() as ctx, _patch(kube, "run", run): + d = migration.MigrationDriver(poll_s=0.01) + rec = Migration(name="m1", start=datetime.now(UTC), pv="pv1") + d._await_terminal(ctx, rec, "m1", sampler=s) + # Deduplicated: only transitions, not every poll. + self.assertEqual(s.phases, ["Preparing", "Cutover", "Completed"]) + self.assertEqual([e.data["phase"] for e in ctx.timeline.of_kind("migration.phase")], + ["Preparing", "Cutover", "Completed"]) + + +class Manifest(unittest.TestCase): + def test_the_cr_carries_the_run_label_so_teardown_can_find_it(self): + d = migration.MigrationDriver(namespace="default") + m = json.loads(d._manifest("run1-mig-3", "pvc-abc", "uuid-b")) + self.assertEqual(m["kind"], "VolumeMigration") + self.assertEqual(m["spec"], {"pvName": "pvc-abc", "targetNodeUUID": "uuid-b"}) + self.assertEqual(m["metadata"]["labels"]["sbtest-run"], "true") + + def test_online_nodes_only(self): + """Migrating to an offline node is a rejected request, not a test.""" + fake = _FakeKube({"get storagenodes": _nodes_json( + ("uuid-a", "vm02", "online"), ("uuid-b", "vm03", "offline"), + ("uuid-c", "vm04", "online"))}) + with _patch(kube, "run", fake.run): + uuids, hosts = migration.MigrationDriver()._storage_nodes() + self.assertEqual(uuids, ["uuid-a", "uuid-c"]) + self.assertEqual(hosts["uuid-c"], "vm04") + + +class SetupGuards(unittest.TestCase): + def test_one_node_cannot_be_migrated_between(self): + fake = _FakeKube({"get storagenodes": _nodes_json(("uuid-a", "vm02", "online"))}) + with _Ctx() as ctx, _patch(kube, "run", fake.run), \ + self.assertRaises(RuntimeError) as e: + migration.MigrationDriver().setup(ctx) + self.assertIn("at least two", str(e.exception)) + + def test_no_volumes_is_an_explicit_error_not_an_empty_run(self): + """Silently migrating nothing would report PASS for a test that never ran.""" + fake = _FakeKube({"get storagenodes": _nodes_json( + ("uuid-a", "vm02", "online"), ("uuid-b", "vm03", "online"))}) + with _Ctx() as ctx, _patch(kube, "run", fake.run), \ + self.assertRaises(RuntimeError) as e: + migration.MigrationDriver().setup(ctx) + self.assertIn("no volumes to migrate", str(e.exception)) + + +class Persistence(unittest.TestCase): + def test_migrations_round_trip_through_the_file(self): + """What the driver writes is what the analyser reads — the seam that lets a run be + re-judged later against a detector that did not exist when it ran.""" + with _Ctx() as ctx: + d = migration.MigrationDriver() + start = datetime.now(UTC).replace(microsecond=0) + d._records = [Migration(name="run1-mig-1", start=start, + end=start + timedelta(seconds=42), phase="Completed", + source="uuid-a", target="uuid-b", pv="pv1", pod="fio-0", + members=["pv1", "pv2"]), + Migration(name="run1-mig-2", start=start + timedelta(minutes=1), + phase="TIMEOUT", pv="pv3")] + d._nqn_of = {"pv1": "nqn.a"} + d.collect(ctx) + back = migration.migrations_from_file(os.path.join(ctx.outdir, + "migrations.json")) + self.assertEqual([m.name for m in back], ["run1-mig-1", "run1-mig-2"]) + self.assertEqual(back[0].members, ["pv1", "pv2"]) + self.assertTrue(back[0].batch) + self.assertEqual(back[0].phase, "Completed") + self.assertIsNotNone(back[0].end) + assert back[0].end is not None + self.assertEqual((back[0].end - back[0].start).total_seconds(), 42) + self.assertIsNone(back[1].end) + + def test_a_record_without_a_start_is_dropped_rather_than_guessed(self): + with _Ctx() as ctx: + p = os.path.join(ctx.outdir, "migrations.json") + with open(p, "w") as fh: + json.dump([{"name": "broken", "phase": "Completed"}], fh) + self.assertEqual(migration.migrations_from_file(p), []) + + +class WorkloadFio(unittest.TestCase): + def test_verification_is_off_when_it_cannot_be_trusted(self): + """numjobs>1 cannot serialize overlapping writes, so verify would report corruption + that never happened. A throughput run must not look like an integrity run.""" + with _Ctx() as ctx: + single = workload.FioWorkload(numjobs=1, iodepth=8)._fio_script(ctx) + multi = workload.FioWorkload(numjobs=4, iodepth=8)._fio_script(ctx) + self.assertIn("--verify=md5", single) + self.assertIn("--serialize_overlap=1", single) + self.assertNotIn("--verify=md5", multi) + + def test_verify_is_not_fatal_by_default_so_every_bad_block_is_counted(self): + with _Ctx() as ctx: + s = workload.FioWorkload(numjobs=1)._fio_script(ctx) + self.assertIn("--verify_fatal=0", s) + + def test_the_file_stays_inside_the_volume(self): + with _Ctx() as ctx: + s = workload.FioWorkload(volume_size_gb=10, file_size_gb=50)._fio_script(ctx) + self.assertIn("--size=8G", s) # 10 - 2 of filesystem headroom + + def test_logs_live_off_the_volume_under_test(self): + """Collecting the evidence must not depend on the health of what it is about.""" + with _Ctx() as ctx: + s = workload.FioWorkload()._fio_script(ctx) + self.assertIn("--output=/logs/result.json", s) + self.assertIn("--filename=/data/fiotest", s) + + def test_timeseries_is_written_where_the_analyser_reads_it(self): + raw = "\n".join([ + "1000, 500, 0, 4096", # 1s: 500 read + "1000, 100, 1, 4096", # 1s: 100 write + "2000, 400, 0, 4096", + ]) + start = datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC) + migs = [Migration(name="mig-1", start=start + timedelta(seconds=1), + end=start + timedelta(seconds=1), pv="pv1")] + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=start) + w = workload.FioWorkload() + d = ctx.dir("fio-0") + w._write_timeseries(ctx, "default", "fio-0", d, migs) + with open(os.path.join(d, "timeseries.csv")) as fh: + rows = list(__import__("csv").DictReader(fh)) + self.assertEqual(rows[0]["second"], "1") + self.assertEqual(float(rows[0]["total_iops"]), 600.0) + self.assertEqual(rows[0]["wall_clock"], "2026-08-20T09:00:01Z") + # The migration column is the point: it makes a dip attributable. + self.assertEqual(rows[0]["active_migration"], "mig-1") + self.assertEqual(rows[1]["active_migration"], "") + + def test_the_analyser_reads_back_what_the_workload_wrote(self): + """Guards the column-name seam. Reading only fio's own names silently placed every + sample at offset 0 — the series stayed the right length with the whole run collapsed + onto one instant, so nothing looked like a parse failure.""" + from sbtest.adapters import ArchiveEvidence + raw = "1000, 500, 0, 4096\n2000, 400, 0, 4096" + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC)) + workload.FioWorkload()._write_timeseries( + ctx, "default", "run1-fio-0", ctx.dir("run1-fio-0"), []) + series = ArchiveEvidence(ctx.outdir).fio_timeseries("run1-fio-0") + self.assertEqual([s.offset_s for s in series], [1, 2]) + self.assertEqual([s.total_iops for s in series], [500.0, 400.0]) + self.assertIsNotNone(series[0].wall) + + def test_the_clock_comes_from_fio_not_from_the_run(self): + """fio counts from its own launch, which is minutes after the run's start: the PVCs, + the pods and the fio processes all have to exist first. Basing the wall clock on the + run shifts every sample by that gap and hands an outage to the wrong migration.""" + raw = "1000, 500, 0, 4096" + run_start = datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC) + fio_start = run_start + timedelta(seconds=180) + # The migration ran while fio was running, i.e. nowhere near the run's own start. + migs = [Migration(name="mig-1", start=fio_start, end=fio_start + timedelta(seconds=30))] + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=run_start) + d = ctx.dir("fio-0") + with open(os.path.join(d, "result.json"), "w") as fh: + json.dump({"jobs": [{"job_start": int(fio_start.timestamp() * 1000)}]}, fh) + workload.FioWorkload()._write_timeseries(ctx, "default", "fio-0", d, migs) + with open(os.path.join(d, "timeseries.csv")) as fh: + rows = list(__import__("csv").DictReader(fh)) + self.assertEqual(rows[0]["wall_clock"], "2026-08-20T09:03:01Z") + self.assertEqual(rows[0]["active_migration"], "mig-1") + + def test_a_result_without_job_start_falls_back_to_the_run(self): + """A wrong base still beats an empty wall_clock column — but it is reported.""" + raw = "1000, 500, 0, 4096" + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC)) + d = ctx.dir("fio-0") + with open(os.path.join(d, "result.json"), "w") as fh: + json.dump({"jobs": [{}]}, fh) + workload.FioWorkload()._write_timeseries(ctx, "default", "fio-0", d, []) + with open(os.path.join(d, "timeseries.csv")) as fh: + rows = list(__import__("csv").DictReader(fh)) + self.assertEqual(rows[0]["wall_clock"], "2026-08-20T09:00:01Z") + + def test_the_analyser_re_derives_the_base_from_fio(self): + """Replay has to correct archives written before the base was fixed: the wall_clock + column is only as right as whatever wrote it, and job_start is right by + construction.""" + from sbtest.adapters import ArchiveEvidence + fio_start = datetime(2026, 8, 20, 9, 3, 0, tzinfo=UTC) + with _Ctx() as ctx: + d = ctx.dir("run1-fio-0") + with open(os.path.join(d, "result.json"), "w") as fh: + json.dump({"jobs": [{"job_start": int(fio_start.timestamp() * 1000)}]}, fh) + with open(os.path.join(d, "timeseries.csv"), "w") as fh: + fh.write("second,wall_clock,total_iops\n" + "1,2026-08-20T09:00:01Z,500.0\n") # the pre-fix, run-based clock + series = ArchiveEvidence(ctx.outdir).fio_timeseries("run1-fio-0") + self.assertEqual(series[0].wall, fio_start + timedelta(seconds=1)) + + def test_the_wall_clock_column_is_used_when_fio_says_nothing(self): + from sbtest.adapters import ArchiveEvidence + with _Ctx() as ctx: + d = ctx.dir("run1-fio-0") + with open(os.path.join(d, "timeseries.csv"), "w") as fh: + fh.write("second,wall_clock,total_iops\n1,2026-08-20T09:00:01Z,500.0\n") + series = ArchiveEvidence(ctx.outdir).fio_timeseries("run1-fio-0") + self.assertEqual(series[0].wall, datetime(2026, 8, 20, 9, 0, 1, tzinfo=UTC)) + + def test_a_workload_with_no_pods_is_refused(self): + with _Ctx() as ctx, self.assertRaises(RuntimeError) as e: + workload.FioWorkload(pods=0, ns_pods=0)._create(ctx) + self.assertIn("no I/O", str(e.exception)) + + +class _patch: + """Minimal attribute patcher — the stdlib one needs a dotted target string.""" + + def __init__(self, obj: object, attr: str, value: object) -> None: + self.obj, self.attr, self.value = obj, attr, value + + def __enter__(self) -> object: + self.old = getattr(self.obj, self.attr) + setattr(self.obj, self.attr, self.value) + return self.value + + def __exit__(self, *exc: object) -> None: + setattr(self.obj, self.attr, self.old) + + +if __name__ == "__main__": + unittest.main()