From a979758c037e7318074d693fa89bd37932979611 Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Thu, 20 Aug 2026 18:40:37 +0200 Subject: [PATCH 1/3] Added initial sbtest framework to build / compose long-running tests --- test/framework/.gitignore | 6 + test/framework/Makefile | 171 ++++ test/framework/README.md | 528 +++++++++++ test/framework/pyproject.toml | 67 ++ test/framework/requirements-dev.txt | 3 + test/framework/sbtest/__init__.py | 17 + test/framework/sbtest/__main__.py | 5 + test/framework/sbtest/adapters/__init__.py | 11 + test/framework/sbtest/adapters/archive.py | 432 +++++++++ test/framework/sbtest/adapters/live.py | 37 + test/framework/sbtest/cli.py | 241 +++++ test/framework/sbtest/components/__init__.py | 9 + test/framework/sbtest/components/events.py | 60 ++ test/framework/sbtest/components/kube.py | 68 ++ test/framework/sbtest/components/logs.py | 477 ++++++++++ test/framework/sbtest/components/migration.py | 445 +++++++++ test/framework/sbtest/components/nvme.py | 247 +++++ test/framework/sbtest/components/sbctl.py | 194 ++++ test/framework/sbtest/components/workload.py | 534 +++++++++++ test/framework/sbtest/core/__init__.py | 39 + test/framework/sbtest/core/config.py | 162 ++++ test/framework/sbtest/core/context.py | 166 ++++ test/framework/sbtest/core/evidence.py | 277 ++++++ test/framework/sbtest/core/findings.py | 223 +++++ test/framework/sbtest/core/plugin.py | 182 ++++ test/framework/sbtest/core/runner.py | 234 +++++ test/framework/sbtest/detectors/__init__.py | 23 + test/framework/sbtest/detectors/ana.py | 311 +++++++ test/framework/sbtest/detectors/control.py | 341 +++++++ test/framework/sbtest/detectors/fio.py | 288 ++++++ test/framework/sbtest/detectors/kernel.py | 637 +++++++++++++ test/framework/sbtest/detectors/logs.py | 206 +++++ test/framework/sbtest/detectors/meta.py | 190 ++++ test/framework/sbtest/detectors/migration.py | 115 +++ test/framework/sbtest/detectors/nvme.py | 140 +++ test/framework/sbtest/detectors/security.py | 83 ++ .../framework/sbtest/suites/analyze-only.yaml | 15 + .../sbtest/suites/corruption-hunt.yaml | 48 + .../sbtest/suites/migration-full.yaml | 105 +++ .../sbtest/suites/migration-soak.yaml | 47 + test/framework/tests/test_core.py | 551 +++++++++++ test/framework/tests/test_detectors.py | 852 ++++++++++++++++++ test/framework/tests/test_driver.py | 384 ++++++++ 43 files changed, 9171 insertions(+) create mode 100644 test/framework/.gitignore create mode 100644 test/framework/Makefile create mode 100644 test/framework/README.md create mode 100644 test/framework/pyproject.toml create mode 100644 test/framework/requirements-dev.txt create mode 100644 test/framework/sbtest/__init__.py create mode 100644 test/framework/sbtest/__main__.py create mode 100644 test/framework/sbtest/adapters/__init__.py create mode 100644 test/framework/sbtest/adapters/archive.py create mode 100644 test/framework/sbtest/adapters/live.py create mode 100644 test/framework/sbtest/cli.py create mode 100644 test/framework/sbtest/components/__init__.py create mode 100644 test/framework/sbtest/components/events.py create mode 100644 test/framework/sbtest/components/kube.py create mode 100644 test/framework/sbtest/components/logs.py create mode 100644 test/framework/sbtest/components/migration.py create mode 100644 test/framework/sbtest/components/nvme.py create mode 100644 test/framework/sbtest/components/sbctl.py create mode 100644 test/framework/sbtest/components/workload.py create mode 100644 test/framework/sbtest/core/__init__.py create mode 100644 test/framework/sbtest/core/config.py create mode 100644 test/framework/sbtest/core/context.py create mode 100644 test/framework/sbtest/core/evidence.py create mode 100644 test/framework/sbtest/core/findings.py create mode 100644 test/framework/sbtest/core/plugin.py create mode 100644 test/framework/sbtest/core/runner.py create mode 100644 test/framework/sbtest/detectors/__init__.py create mode 100644 test/framework/sbtest/detectors/ana.py create mode 100644 test/framework/sbtest/detectors/control.py create mode 100644 test/framework/sbtest/detectors/fio.py create mode 100644 test/framework/sbtest/detectors/kernel.py create mode 100644 test/framework/sbtest/detectors/logs.py create mode 100644 test/framework/sbtest/detectors/meta.py create mode 100644 test/framework/sbtest/detectors/migration.py create mode 100644 test/framework/sbtest/detectors/nvme.py create mode 100644 test/framework/sbtest/detectors/security.py create mode 100644 test/framework/sbtest/suites/analyze-only.yaml create mode 100644 test/framework/sbtest/suites/corruption-hunt.yaml create mode 100644 test/framework/sbtest/suites/migration-full.yaml create mode 100644 test/framework/sbtest/suites/migration-soak.yaml create mode 100644 test/framework/tests/test_core.py create mode 100644 test/framework/tests/test_detectors.py create mode 100644 test/framework/tests/test_driver.py diff --git a/test/framework/.gitignore b/test/framework/.gitignore new file mode 100644 index 000000000..b7325183a --- /dev/null +++ b/test/framework/.gitignore @@ -0,0 +1,6 @@ +.venv/ +__pycache__/ +*.egg-info/ +.ruff_cache/ +runs/ +.mypy_cache/ diff --git a/test/framework/Makefile b/test/framework/Makefile new file mode 100644 index 000000000..c4b5d7d57 --- /dev/null +++ b/test/framework/Makefile @@ -0,0 +1,171 @@ +# sbtest — component/detector framework for large-scale test runs. +# +# Every target that needs the virtualenv depends on it, so there is no separate setup step +# to remember: `make test` on a clean checkout creates .venv, installs, and runs. +# +# make test unit tests (no cluster) +# make analyze RUN= [SUITE=] judge a finished run directory +# make collect OUT= [SUITE=] [DURATION=] collect from a live cluster, then judge +# make run SUITE= [DURATION=] [KEEP=1] drive a full test run, then judge it +# make detectors | make components what is available, with options +# make gate quality gate: ruff + mypy + tests +# make lint | make types | make fmt the gate's parts, individually +# make clean | make distclean +# +# SUITE takes a bundled suite name (see `make suites`) or a path to your own file. + +SHELL := /bin/bash +.DEFAULT_GOAL := help + +VENV := .venv +PY := $(VENV)/bin/python +PIP := $(VENV)/bin/pip +SBTEST := $(VENV)/bin/sbtest +RUFF := $(VENV)/bin/ruff +MYPY := $(VENV)/bin/mypy +STAMP := $(VENV)/.install-stamp +PYTHON ?= python3 + +# ── parameters ──────────────────────────────────────────────────────────────── +# SUITE is optional everywhere: with no suite, detectors all run with their defaults and no +# components do, which is exactly what judging an archive wants. +SUITE ?= +RUN ?= +OUT ?= +DURATION ?= 0 +JSON_NAME ?= findings.json +# KEEP=1 leaves the volumes, pods and CRs behind. Most post-mortems need this, so it is worth +# knowing the flag exists before the run rather than after. +KEEP ?= +EXTRA ?= + +SUITE_ARG := $(if $(SUITE),--suite $(SUITE),) + +# ── venv ────────────────────────────────────────────────────────────────────── + +$(VENV): + @echo "==> creating $(VENV)" + @$(PYTHON) -m venv $(VENV) + +# Re-installs when pyproject changes; the stamp keeps every other target from paying for it. +$(STAMP): pyproject.toml | $(VENV) + @echo "==> installing sbtest (editable) + dev extras" + @$(PIP) install -q --upgrade pip + @$(PIP) install -q -e '.[dev]' + @touch $@ + +.PHONY: venv +venv: $(STAMP) ## Create the virtualenv and install sbtest. + @$(PY) -c "import sbtest; print(f'sbtest {sbtest.__version__} ready in $(VENV)')" + +# ── tests and lint ──────────────────────────────────────────────────────────── + +.PHONY: test +test: $(STAMP) ## Run the unit tests (no cluster needed). + @$(PY) -m unittest discover -s tests -p 'test_*.py' -v + +.PHONY: lint +lint: $(STAMP) ## Lint with ruff. + @$(RUFF) check sbtest tests + +.PHONY: types +types: $(STAMP) ## Type-check with mypy. + @$(MYPY) + +.PHONY: fmt +fmt: $(STAMP) ## Apply ruff's safe fixes. + @$(RUFF) check --fix sbtest tests + +# The gate runs every check even when an earlier one fails, then reports once. Stopping at +# the first failure means three round trips to find out you had three problems; a detector +# framework whose own checks are annoying to run is a framework nobody runs. +.PHONY: gate +gate: $(STAMP) ## Quality gate: ruff + mypy + tests. Non-zero if any part fails. + @rc=0; \ + printf '\n==> ruff\n'; $(RUFF) check sbtest tests || rc=1; \ + printf '\n==> mypy\n'; $(MYPY) || rc=1; \ + printf '\n==> tests\n'; $(PY) -m unittest discover -s tests -p 'test_*.py' || rc=1; \ + printf '\n'; \ + if [ $$rc -eq 0 ]; then echo "GATE: PASS"; else echo "GATE: FAIL"; fi; \ + exit $$rc + +.PHONY: check +check: gate ## Alias for `gate`. + +# ── using it ────────────────────────────────────────────────────────────────── + +.PHONY: detectors +detectors: $(STAMP) ## List the available detectors and their options. + @$(SBTEST) detectors + +.PHONY: components +components: $(STAMP) ## List the available components and their options. + @$(SBTEST) components + +.PHONY: suites +suites: $(STAMP) ## List the bundled suites. + @for f in sbtest/suites/*.yaml; do \ + name=$$(basename $$f .yaml); \ + desc=$$($(PY) -c "import yaml; print(' '.join(yaml.safe_load(open('$$f')).get('description','').split()))"); \ + printf ' %-18s %s\n' "$$name" "$$desc"; \ + done + +.PHONY: analyze +analyze: $(STAMP) ## Judge a finished run directory. RUN= [SUITE=] +ifndef RUN + $(error RUN is required, e.g. make analyze RUN=../../operator/fio-mig-1787171993 SUITE=corruption-hunt) +endif + @$(SBTEST) analyze "$(RUN)" $(SUITE_ARG) --json-name "$(JSON_NAME)" --freeze-table $(EXTRA) + +.PHONY: collect +collect: $(STAMP) ## Collect from a live cluster, then judge. OUT= [SUITE=] [DURATION=] +ifndef OUT + $(error OUT is required, e.g. make collect OUT=./runs/soak SUITE=migration-soak DURATION=600) +endif + @$(SBTEST) collect "$(OUT)" $(SUITE_ARG) --duration "$(DURATION)" --judge \ + --json-name "$(JSON_NAME)" $(EXTRA) + +.PHONY: run +run: $(STAMP) ## Drive a full test run against a cluster, then judge it. SUITE= [DURATION=] [KEEP=1] [OUT=] +ifndef SUITE + $(error SUITE is required, e.g. make run SUITE=migration-full DURATION=7200 KEEP=1) +endif + @$(SBTEST) run $(SUITE_ARG) --duration "$(DURATION)" \ + $(if $(OUT),--outdir "$(OUT)",) $(if $(KEEP),--keep,) \ + --json-name "$(JSON_NAME)" $(EXTRA) + +# Judging every archived run at once: the cheapest way to see whether a new or changed +# detector fires where it should and stays quiet where it should not. +.PHONY: analyze-all +analyze-all: $(STAMP) ## Judge every operator/fio-mig-* run directory. [SUITE=] + @shopt -s nullglob; \ + found=0; \ + for d in ../../operator/fio-mig-*/; do \ + found=1; \ + echo "════════ $$d"; \ + $(SBTEST) analyze "$$d" $(SUITE_ARG) --json-name "$(JSON_NAME)" 2>&1 \ + | grep -E 'RESULT|skipped \(' || true; \ + done; \ + [ $$found -eq 1 ] || echo "no operator/fio-mig-* run directories found" + +# ── housekeeping ────────────────────────────────────────────────────────────── + +.PHONY: clean +clean: ## Remove caches and build leftovers (keeps the venv). + @rm -rf .ruff_cache .mypy_cache **/__pycache__ */__pycache__ */*/__pycache__ *.egg-info src/*.egg-info + @echo "cleaned" + +.PHONY: distclean +distclean: clean ## Also remove the virtualenv. + @rm -rf $(VENV) + @echo "removed $(VENV)" + +.PHONY: help +help: ## Show this help. + @echo "sbtest — component/detector framework for large-scale test runs" + @echo + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}' + @echo + @echo "Parameters: RUN, OUT, SUITE, DURATION, JSON_NAME, EXTRA" + @echo "Example: make analyze RUN=../../operator/fio-mig-1787171993 SUITE=corruption-hunt" \ No newline at end of file diff --git a/test/framework/README.md b/test/framework/README.md new file mode 100644 index 000000000..edd135306 --- /dev/null +++ b/test/framework/README.md @@ -0,0 +1,528 @@ +# sbtest — a component/detector framework for large-scale test runs + +`sbtest` splits a test run into two kinds of pluggable pieces: + +* **Components** do the side-effecting work — drive a workload, sample host paths, follow + logs, snapshot the fabric. Each can be enabled or disabled independently. +* **Detectors** judge the evidence a run produced. Each is a pure function from `Evidence` + to findings: no cluster access, no ordering, no shared state. + +The split is the point. Every check worth having came out of an incident, and adding one +should not mean touching collection code; conversely, turning streaming log collection on or +off should not disturb a single check. + +It drives `kubectl` as a subprocess and depends on nothing but PyYAML, so it runs wherever +the existing harness runs. + +> `operator/test/fio_migration_test.py` is untouched and still the way to run the migration +> soak today. This framework is the reusable substrate underneath it — its detector set is +> already a strict superset of that harness's checks, and it can judge every run that +> harness has ever produced. + +--- + +## Quick start + +Everything goes through `make`, which creates and populates `.venv` on first use — there is +no separate setup step to remember. + +```bash +cd test/framework + +make # help, with the parameters listed +make gate # quality gate: ruff + mypy + 142 tests + +make detectors # what can judge a run, and with which knobs +make components # what can run during a run +make suites # the bundled suites + +# judge any existing run directory — no cluster needed +make analyze RUN=../../operator/fio-mig-1787171993 +make analyze RUN= SUITE=corruption-hunt + +# judge every archived run at once: the cheapest check on a new detector +make analyze-all SUITE=corruption-hunt + +# collect from a live cluster for 10 minutes, then judge +make collect OUT=./runs/my-run SUITE=migration-soak DURATION=600 + +# drive a full migration run — volumes, fio, VolumeMigration CRs — then judge it +make run SUITE=migration-full DURATION=7200 KEEP=1 +``` + +`collect` observes a cluster someone else is loading; `run` creates the load itself. The half +after the components stop is identical in both, which is the point: a live run and an +archived one are judged by the same detectors, so a verdict from either means the same thing. + +`SUITE` takes a bundled name or a path to your own file, and is optional everywhere: with no +suite every detector runs on its defaults and no component does, which is exactly what +judging an archive wants. + +The venv also installs an `sbtest` entry point, if you would rather not go through make: + +```bash +.venv/bin/sbtest analyze --suite corruption-hunt --freeze-table +``` + +### Quality gate + +`make gate` runs ruff, mypy and the unit tests, and — deliberately — **runs all three even +when an earlier one fails**, then reports once and exits non-zero. Finding out you had three +problems should not take three round trips; a framework whose own checks are tedious to run +is one nobody runs. `make lint`, `make types` and `make test` are the parts individually. + +The only runtime dependency is `pyyaml`, for the suites; `ruff` and `mypy` are dev extras. +Everything runs out of the venv that `make` builds, so the dependency costs nothing — and a +suite is a document to be read and argued with, where every threshold wants a sentence saying +why it is that number. JSON cannot carry that sentence. + +`analyze` is the one to reach for first. It runs the whole detector set over an artifact +directory — including every archived `fio-mig-*` run — which means a check can be written or +fixed and tried **immediately against the run that motivated it**, instead of against the +next four-hour run. Not being able to do that was the single biggest gap in the harness this +grew from. + +## It reproduces the findings it was built from + +Run against `operator/fio-mig-1787171993` (20 pods, 46 migrations, 4h08m), the detectors +independently derive the result that took a manual investigation to find: + +``` +ana.freeze-count mig-20 (2 freezes) mig-29 (4) mig-38 (5) mig-42 (5) +fio.checksum mig-20 (5 blocks) mig-29 (4) mig-38 (2) mig-42 (2) + +subjects with more than one finding: + mig-20: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) + mig-29: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) + mig-38: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) + mig-42: ana.cutover-pause(CRITICAL); ana.freeze-count(CRITICAL); fio.checksum(CRITICAL) +``` + +Two unrelated detectors — one reading host ANA samples, one reading fio's own output — land +on the same four subjects out of 46. That correlation is the strongest signal these runs +produce, and surfacing it automatically is why findings carry a `subject`. + +The same detector set on the earlier `fio-mig-1787159565` reports a *different* signature +(8 × EREMOTEIO, 3 overlong pauses, 3481 `does-not-allow-host` matches from the path leak, no +freeze-count findings) and correctly reports `nvme.stale-controllers` as **skipped** rather +than clean, because that run has no fabric snapshot. + +## The path-loss ladder + +dmesg carries something no other source does: when every path to a namespace goes away, the +kernel does not simply fail I/O. It queues, waits, and only fails once a timeout expires — +and each rung is a different amount of damage. + +``` +all paths inaccessible + -> block nvmeXnY: no usable path - requeuing I/O queued; the application waits + -> nvme nvmeN: failfast expired fast_io_fail_tmo elapsed + -> block nvmeXnY: no available path - failing I/O errors reach the application + -> XFS (nvmeXnY): log I/O error -5 + -> XFS (nvmeXnY): Filesystem has been shut down the volume needs repair +``` + +**`fast_io_fail_tmo` is the knob that decides which rung a cutover pause reaches.** A pause +shorter than it is absorbed by queueing; a pause longer becomes application-visible errors and +then filesystem damage. That is why `ana.cutover-pause` and `kernel.path-loss` belong in the +same report: one measures the pause, the other says whether the host survived it. + +Measured across three archived runs, counting only what happened **inside each run's window**: + +| run | requeued | `failfast expired` | failing I/O | filesystems shut down | +|---|---|---|---|---| +| `-1787159565` | 2 | 18 | 0 | 0 | +| `-1787171993` | 163 | 84 | 0 | 0 | +| `-1787205545` | 35 (+186 before) | 20 (+118 before) | 0 (+30 before) | 0 (+20 before) | + +The parenthesised numbers are the reason attribution exists. Read without a window, that last +run looks catastrophic — 30 failed I/Os and 19 filesystems shut down. All of it happened +between 05:46 and 05:48, eleven minutes *before* the run started at 05:59: it is the previous +run's teardown and the cluster reinstall, not this run's doing. (Worth chasing separately — +tearing a cluster down should not kill mounted filesystems — but it is not a migration defect.) + +## Attribution: old damage must not fail a new run + +dmesg is a ring buffer covering hours and a cluster outlives its runs, so evidence routinely +contains the previous runs' mess. Counting it twice over is a trap: a clean run inherits its +predecessor's failure, and — worse — a genuinely broken run hides inside inherited noise. + +Every finding therefore carries an `Attribution`: + +| attribution | meaning | counts against the run? | +|---|---|---| +| `run` | happened inside the run's window | yes | +| `unknown` | no usable timestamp, or no known window | **yes** — "I cannot date this" must not become "not our problem" | +| `pre-existing` | positively dated before the run began | no | + +Severity says *does this matter*; attribution says *whose fault*. The two are orthogonal, so +the same observation is CRITICAL when the run caused it and a hygiene WARNING when it did not. + +### What actually forfeits a run + +Almost nothing, and the distinction is worth stating because the temptation is to fail on any +inherited mess: + +| pre-existing condition | affects this run? | verdict | +|---|---|---| +| Dead-cluster controllers, old reconnect storms, old fabric errors | No — they cannot make a *different* cluster's migration fail | hygiene WARNING | +| A filesystem killed before the run, on a volume the run does not use | No | hygiene WARNING | +| **Live-cluster** controllers already `live`-with-no-namespace at setup (`nvme.dirty-start`) | **Yes** — `VerifyMigrationPaths` will reject migrations that should pass, so the completion rate measures the inherited mess | **INCONCLUSIVE** | + +That last row is the only thing that produces `INCONCLUSIVE`, and it is a third verdict rather +than a softer FAIL because it calls for a different action: clean the fabric and run again, +rather than go looking through the code. It needs the `nvme.snapshot` component's pre-run +snapshot; without one the detector reports itself **skipped** instead of guessing. + +`dmesg --time-format=iso` is collected in preference to `dmesg -T` for exactly this reason: the +ISO form carries a UTC offset, so an event can be placed against the run window soundly rather +than by assuming the host's clock agrees with the harness's. + +## Concepts + +### Evidence + +The only thing a detector may read. Everything is lazy — a run's SPDK logs are tens of MiB +per node and most detectors never open them. + +```python +ev.migrations() -> list[Migration] # the timeline +ev.ana_samples("mig-20") -> list[AnaSample] # host path state, per migration +ev.fio_jobs() -> list[FioJob] # per-pod outcome and errno +ev.fio_timeseries(pod) -> list[IopsSample] # per-second IOPS +ev.fio_log(pod) -> Iterator[str] # where verify failures live +ev.container_logs() -> list[str] # "spdk-4420", "operator", ... +ev.container_log(name) -> Iterator[str] +ev.nvme_controllers() -> list[NvmeController] # fabric snapshot +``` + +Two implementations, kept deliberately close so a check that passes live cannot fail on +replay: `ArchiveEvidence` reads a finished run directory, and `LiveEvidence` is the same +reader pointed at the directory being written, overlaid with in-memory state. + +### Findings, and why "skipped" is not "clean" + +A detector returns `Finding`s carrying `severity`, `subject`, `detail`, structured +`evidence`, and a `note` explaining what it means. Any CRITICAL fails the run. + +A detector whose evidence is **absent** must raise `SkipDetector` rather than return nothing. +Silence makes "could not check" and "checked, all clean" the same output, which is how a +broken check passes a broken run for weeks. Skips are reported separately, by name. + +### Component lifecycle + +Every hook is optional; override only what applies. + +``` +setup() resolve targets, create helpers +start() begin sampling / following / driving +tick() periodic, cheap, must not block +stop() stop doing the thing +collect() gather artifacts into the run directory +teardown() release cluster resources — runs even when the run failed +``` + +`collect` is separate from `stop` for exactly the case that motivated the framework: a +streaming collector has nothing to collect at the end because it has been writing all along, +while a post-run collector does all its work there — and the runner must be able to run +either, or both, without either knowing. + +Set `required = True` on a component that *is* the run (a workload, a migration driver): a +failure in its `setup` aborts, because continuing would produce a green result for a test +that never happened. Collectors leave it False, so losing one evidence stream degrades the +run instead of ending it — and the failure is recorded as a WARNING finding. + +## Configuration + +```yaml +components: + logs.stream: + containers: [spdk-container] + ttl_s: 21600 + logs.collect: true + ana.sample: + interval_s: 2.0 + +detectors: + ana.freeze-count: + # More than one freeze in a migration meant lost writes on 5 of 5 archived runs. + max_freezes: 1 + fio.checksum: + verify_lag_s: 45 + fio.throughput-outlier: false +``` + +Two rules make this predictable: + +* A name absent from the config is **off for components** and **on for detectors**. + Collecting is a cost you opt into; judging is not something you should have to remember to + switch on. +* An unknown name or option is an **error at startup**, with the valid list. A threshold that + looks set but is not is worse than one that is obviously missing. + +Bundled suites live in `sbtest/suites/` (`migration-full`, `migration-soak`, +`corruption-hunt`, `analyze-only`; `make suites` lists them) and are selected with +`--suite `, which also takes a path to your own file. A `.json` suite still loads, for +anything generating them programmatically. CLI `--enable-detector` / `--disable-component` +layer on top, and disable wins over enable. + +## Detector catalogue + +Every one of these came from a real defect. Defaults encode what the runs measured. + +| detector | what it catches | +|---|---| +| `ana.freeze-count` | **A migration that froze the volume more than once.** Exact predictor of silent write loss so far: 4/4 corrupting vs 0/42 clean. A migration takes the cutover pause once; the rest are retries, and each retry replays a non-idempotent transfer against a source that has been serving writes. | +| `ana.cutover-pause` | An all-paths-inaccessible window longer than the design pause (~2s). Complements the count: catches one window that overran, which the count cannot see. | +| `ana.split-brain` | Source and target both `optimized` at the same instant — two writers, silent corruption by construction. | +| `ana.unserved-after-cutover` | A Completed migration whose live target controller serves only some of the subsystem's namespaces — the half-moved case. | +| `ana.path-churn` | More distinct path addresses per host than the topology should produce. Informational: healthy counts are topology-dependent. | +| `fio.checksum` | **fio read back data it never wrote.** Reads succeeded, so nothing else notices. Attributes to a migration through a verify lag (see below). | +| `fio.job-error` | An fio job ended with a non-zero errno, with the errno's meaning — 121/EREMOTEIO points straight at the ANA detectors. | +| `fio.outage` | A pod's I/O stopped for longer than a cutover should cost. | +| `fio.throughput-outlier` | A pod far below the run's median IOPS. Weak alone; strong next to an ANA finding on the same subject. | +| `logs.pattern` | **User-definable regex checks over any collected log.** Ships a catalogue: undrained transfer, migration sub-task failure, host-not-allowed reconnect storm, write-to-RO-range, path-validation failure, stuck migration group, kernel reconnect loop. | +| `migration.outcomes` | Completion rate and phase breakdown. | +| `migration.errors` | Distinct migration errors, grouped by shape — 16 identical failures are one defect. | +| `nvme.stale-controllers` | Controllers that are live with no namespace (blocks every later migration of that subsystem) or stuck connecting. | +| `nvme.loss-timeout` | A `ctrl_loss_tmo` long enough that a leaked path outlives the run that made it. | +| `kernel.path-loss` | **How far the kernel got up the path-loss ladder** (see below). Stronger than ANA sampling for the same event: it is what the kernel did, not what a sampler caught, so it cannot miss a window shorter than the interval. | +| `kernel.filesystem-shutdown` | XFS/ext4 shut down or went read-only after failed log I/O — the volume needs unmount and repair. | +| `nvme.foreign-cluster` | **A controller retrying a subsystem whose cluster no longer exists.** No threshold, no topology: an NQN names its cluster. Hygiene only — it cannot affect the live cluster's migrations. | +| `nvme.dirty-start` | The fabric already held blocking debris **for the live cluster** at setup, so the run's results cannot be trusted. The one pre-existing CRITICAL. | +| `nvme.controller-churn` | Controllers created vs removed — "they never disappear", counted — plus controllers retrying without ever succeeding. | +| `kernel.fabric-errors` | Connect/reset/timeout errors grouped by kind. Texture around a failure rather than a verdict. | +| `control.node-flap` | A node marked down and back within seconds — a liveness check that depended on something other than the node. The shape behind a 9.5h outage. | +| `control.volume-health` | A volume or node whose health went false during the run and never returned. | +| `control.task-stuck` | Tasks created and never resolved — "the control plane stopped finishing things". | +| `control.retry-storm` | One operation attempted far more often than it should be; each retry re-does what the last half-did. | +| `control.node-agent` | The node-side agent returning errors, or **gaps in the liveness polling** — the upstream half of a false offline, visible nowhere else. | +| `evidence.log-coverage` | **A collected log that does not span the run**, bounding what every other log-based finding may claim. | +| `evidence.blind-spot` | A migration no log covers, so it cannot be post-mortemed whatever it did. | +| `evidence.inventory` | What evidence the run produced (INFO). | +| `security.secret-exposure` | Credential-shaped strings in collected logs. Reports the location, never the value. | + +Two defaults are load-bearing and worth knowing: + +* **`fio.checksum` verify lag (45s).** fio detects a lost write when it next *reads* that + block, 3–34s later in practice. Without the lag a migration's own losses are filed under + "no migration was running" — which is how a `Completed`-but-corrupting migration hid. +* **`ana.freeze-count` over `ana.cutover-pause`.** The count is more sensitive (two 3s + freezes look like one healthy pause on any longest-window measure), more specific (one + migration's single 5–6s pause lost nothing), and robust to sampling granularity. Keep both; + they catch different shapes, and `tests/test_detectors.py` pins exactly that. + +## Component catalogue + +| component | what it does | +|---|---| +| `logs.stream` | Follows chosen container logs for the whole run, surviving kubelet rotation, container restarts, and pod recreation. | +| `logs.collect` | Grabs container logs from each host's `/var/log/pods` at the end. Skips whatever `logs.stream` followed. | +| `host.dmesg` | `dmesg -T` from each storage worker. | +| `cluster.events` | `sbctl cluster get-logs` → `cluster-events.json`. | +| `nvme.snapshot` | Fabric snapshot before *and* after the run — "did the last run leave a mess?" is a real question, because a leaked controller breaks the *next* run. | +| `ana.sample` | Per-namespace ANA state on every consuming node, on an interval, written per migration in the layout `ArchiveEvidence` reads. Needs a driver to tell it which migration is in flight. | +| `workload.fio` | Provisions volumes from two StorageClasses (single-namespace and packed) and drives continuous md5-verified fio against them. `required`. | +| `migration.driver` | Creates `VolumeMigration` CRs in a loop, one at a time, and records what each one did. `required`. | + +### What gets collected + +| artifact | source | why per-what | +|---|---|---| +| `spdk-.txt` | storage-node SPDK container | **must be streamed.** Measured on vm04: a rotation every ~2 min, so the whole 50 MiB budget bought about **10 minutes** of retention. A migration was unrecoverable 6 seconds after it ended | +| `spdk--proxy.txt` | the SPDK JSON-RPC proxy | streamed too, but far lower volume and survives hours — so it is the fallback when the SPDK log is gone. It carries the RPC-level narrative (which calls, in what order, with what arguments) without SPDK's internal errors | +| `snode-api-.txt` | storage-node DaemonSet | per node — it starts and probes SPDK, so it is on the causal path of every node-offline decision | +| `csi-node-.txt` | CSI node plugin | per node — the plugin reconciles per host, so "which node" is the first question about anything it did | +| `.txt` | tasks pod, csi-controller | **per container.** The tasks pod runs seventeen independent runners; merging them gives a 50 MiB file that is not in time order, so its time span is meaningless and a pattern cannot be scoped to one runner | +| `operator.txt`, `webappapi.txt` | control plane | single-container, so per pod is fine | +| `dmesg-.txt` | each storage worker | ISO-timestamped, see the attribution section | +| `cluster-events.json` | `sbctl cluster get-logs` | the control plane's own account | + +### Why both log components exist + +The kubelet keeps only `containerLogMaxSize × containerLogMaxFiles` per container — 10Mi × 5 += **50 MiB** on the clusters this runs against. Measured SPDK write rates were 0.28–1.23 +MiB/min, so 50 MiB buys 41–176 minutes. A post-run grab of a four-hour run therefore returns +its tail and silently drops the rest; that is how several runs' worth of early evidence was +lost. So the high-volume logs are followed live and everything else is grabbed at the end, +where a one-shot grab is still complete. + +Worth knowing which of the two to reach for: when the SPDK log has aged out, the proxy log on +the *same node* usually still has the RPC sequence — on one run it recovered the full +freeze/revert/retry pattern of a corrupting migration whose SPDK log was already gone, and its +count of the retries was more accurate than the ANA sampler's. + +`logs.stream` handles three separate ways a log moves out from under a follower, because any +one of them leaves the stream *silent rather than failed*: rotation (same filename reopened — +`tail -F` handles it), a container restart (`.log` beside the old one — the target is +re-resolved on a timer), and a pod recreation (a whole new UID directory — same re-resolve). + +## Adding a detector + +```python +from sbtest.core import Detector, SkipDetector, critical, detector + +@detector +class MyCheck(Detector): + name = "mine.my-check" # dotted, stable; config keys off it + summary = "one line for `sbtest detectors`" + + def defaults(self) -> dict: + return {"threshold": 3} # also the option allow-list + + def detect(self, ev): + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") # not the same as clean + for m in migs: + if len(m.members) > self.opt("threshold"): + yield critical(self.name, title="subsystem too wide", subject=m.name, + evidence={"members": len(m.members)}) +``` + +Import it from `sbtest/detectors/__init__.py` and it appears in `sbtest detectors`, is +configurable by name, and runs against every archived run. Unit-test it against +`FakeEvidence` in `tests/test_detectors.py` — no cluster. + +## Adding a component + +```python +from sbtest.core import Component, component + +@component +class MyCollector(Component): + name = "mine.collector" + summary = "one line for `sbtest components`" + required = False # True only if this component *is* the run + + def defaults(self) -> dict: + return {"namespace": "default"} + + def setup(self, ctx): ... # resolve targets + def collect(self, ctx): + with open(ctx.path("my-artifact.txt"), "w") as fh: + fh.write("...") + def teardown(self, ctx): ... # always runs +``` + +Write artifacts through `ctx.path(...)` so they land in the run directory in the layout +`ArchiveEvidence` expects — that is what makes them replayable. Record observations with +`ctx.timeline.record("kind", subject=..., **data)` so detectors can read them back without +knowing which component produced them. + +## Layout + +``` +test/framework/ + sbtest/ + core/ evidence, findings, plugin registry, config, context, runner + components/ logs (stream + collect), nvme (sampler + snapshot), events, kube + detectors/ ana, control, fio, kernel, logs, meta, migration, nvme, security + adapters/ archive (finished run dir), live (run in progress) + suites/ migration-full, migration-soak, corruption-hunt, analyze-only (YAML) + cli.py + tests/ detector and core tests — 108, no cluster required + Makefile bootstraps .venv; every target depends on it + pyproject.toml packaging, ruff and mypy configuration +``` + +Note on lint config: `PTH` (pathlib-over-`os.path`) is deliberately **not** selected. +`operator/test/fio_migration_test.py` and everything around it use `os.path`, and one module +in a different idiom is worse than a consistent old one. + +## Driving a run + +`workload.fio` and `migration.driver` are the two components that *are* the run rather than +observers of it, and `migration-full` is the suite that wires them into the configuration the +corruption work used: + +```bash +make run SUITE=migration-full DURATION=7200 KEEP=1 +``` + +Both declare `required = True`, so a failure in their setup aborts the run instead of being +recorded as a warning. That distinction is the whole reason the flag exists: a run whose +workload never came up would otherwise sail through every detector — nothing to find, nothing +found — and report a clean pass for a test that never happened. + +Three things the driver reads from the backend instead of assuming, each because assuming it +produced a wrong answer that *looked* right: + +* **Where the volume is now.** The cluster's own rebalancer moves volumes with no Kubernetes + object changing, so a source cached at setup is stale by the tenth migration — and a stale + source means picking a target the volume already lives on, which the operator rejects and + which reads as a product bug. +* **Which volumes move together.** A migration moves the whole NVMe subsystem, so its blast + radius is the volume's *group*. The control plane decides the packing and a previous + migration can change it, so the group is re-read before each one. +* **What the migration actually did.** `status.sourceNodeUUID` is the operator's own resolved + answer, so it overwrites the driver's guess in the record. + +The driver also feeds `ana.sample`: `begin()` when a migration starts, `set_phase()` on each +phase transition, `end()` when it finishes. Sampling is worthless unattributed — a freeze +window is per migration — and the phase stamp is what makes a transition readable as "all +paths went inaccessible during Cutover" instead of "at 09:31:02". The handshake goes through +`ctx.shared`, so the sampler stays optional: with it disabled the driver still migrates and +the ANA detectors report themselves *skipped* rather than clean. + +Target selection is deliberate rather than random. The `target_policy` controls whether the +target node *also* hosts a pod consuming the subsystem being moved — the materially harder +case, since that host has to join a subsystem on the very node becoming its target. +`alternate` puts `consumer` first, so a run cut short still exercised it. When the policy +cannot be honoured the driver falls back to any other node and records the policy as +`consumer(unmet)`: the migration is still evidence, but the run's own record must not claim a +case it never reached. + +`workload.fio` enables md5 verification only when it can be trusted. fio's verify races +itself when two in-flight I/Os touch the same block; with one job that is fixable +(`--serialize_overlap`), across processes it is not without `io_submit_mode=offload`. So +`numjobs > 1` turns verification off and says so loudly — a run that measures throughput is +not a run that would have noticed data loss, and the report must not imply otherwise. + +## Trying it against a cluster + +`sbtest collect --suite migration-soak --duration 90 --judge` exercises the +observation half of the lifecycle on its own: grabber pods, live following, pre/post fabric +snapshots, control-plane events, dmesg, then judging. Useful for checking that collection +works before committing a long run to it, and for watching a cluster someone else is loading. + +Doing that for the first time found two bugs no amount of archive replay could have: + +* **Two components wanting a grabber on one node collided.** A Pod is immutable, so the second + to apply the same name failed on a field it may not change — and `logs.collect` silently + produced empty files for every node `logs.stream` had already claimed. The pod name now + carries the component. +* **A live run recorded no window**, so nothing could be dated, so every ring-buffer finding + was `unknown` — which counts — and the run failed on twenty-nine filesystem shutdowns from + hours earlier. The runner now writes `run.json`, which `ArchiveEvidence` prefers over + inferring a window from whatever happened to get logged. + +Both have regression tests. The lesson generalises: the detectors were verifiable offline, the +components were not, and only the components had these bugs. + +## Not yet here + +Deliberate gaps, so nobody looks for them: + +* **No snapshots during the run.** `operator/test/fio_migration_test.py` can take a + `VolumeSnapshot` before a migration and re-check afterwards that the backend snapshot still + resolves (`--snapshot-chance`). Not ported: it is a second dimension on top of migration, and + the corruption work ran with it at 0. +* **The driver migrates one volume at a time, in round-robin order.** Deliberate — concurrent + migrations of different subsystems make every host-side observation ambiguous about which one + caused it — but it does mean the concurrent case is untested by this framework. +* **No fault injection.** Killing a node mid-cutover, partitioning the fabric, or filling a + volume are the obvious next scenarios, and nothing here does them yet. +* **No role labelling in `ana.sample` output.** `ana.split-brain` and + `ana.unserved-after-cutover` need source/target roles per listener and skip without them. + Left open on purpose rather than half-done: the driver now knows the source and target node + UUIDs, and `sbctl storage-node list` gives each node's management IP, so labelling by IP is + easy — and wrong. One host serves both sides of a migration on different ports, so an IP + names a *node* while only `ip:port` names a *path*. The existing harness labelled by IP and + read a third-party replica's listener as the source's, which inverted a cutover/revert + conclusion in one review. What is actually missing is the port → lvstore mapping; until that + is available, these two detectors correctly skip instead of guessing. +* **The phase stamp is best-effort.** `set_phase` records the last phase the driver *saw*, + polling every few seconds, so a phase shorter than the poll interval never lands on a sample. + The ANA transitions themselves are sampled independently and are unaffected; only the label + is coarse. \ No newline at end of file diff --git a/test/framework/pyproject.toml b/test/framework/pyproject.toml new file mode 100644 index 000000000..18f9755c1 --- /dev/null +++ b/test/framework/pyproject.toml @@ -0,0 +1,67 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "sbtest" +version = "0.1.0" +description = "Component/detector framework for large-scale simplyblock test runs" +requires-python = ">=3.11" +# Suites are YAML, because a suite is a document to be read and argued with — every threshold +# in one wants a comment saying why it is that number, and JSON cannot carry one. Everything +# runs out of a venv (`make` builds it), so a single dependency costs nothing. +dependencies = ["pyyaml>=6"] + +[project.optional-dependencies] +dev = ["ruff>=0.6", "mypy>=1.11"] + +[project.scripts] +sbtest = "sbtest.cli:main" + +[tool.setuptools.packages.find] +include = ["sbtest*"] + +[tool.setuptools.package-data] +sbtest = ["suites/*.yaml"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4", "RET"] +ignore = [ + "E501", # long lines are the prose in docstrings, not code + "B008", +] + +# PTH (pathlib-over-os.path) is deliberately not selected: operator/test/fio_migration_test.py +# and everything around it use os.path, and one module in a different idiom is worse than a +# consistent old one. + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["E402"] # sys.path juggling before imports is the point in tests + +[tool.mypy] +python_version = "3.11" +files = ["sbtest", "tests"] +# Strict where it buys something. The framework's whole contract is that a detector reads +# Evidence and returns Findings, so that is exactly the boundary worth type-checking. +warn_unused_configs = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_return_any = true +no_implicit_optional = true +strict_equality = true +check_untyped_defs = true +disallow_untyped_defs = false # lifecycle hooks and tests read better untyped +disallow_incomplete_defs = true + +[[tool.mypy.overrides]] +module = "yaml" +ignore_missing_imports = true # PyYAML ships no stubs + +[[tool.mypy.overrides]] +# Tests build deliberately-wrong plugins to prove the runner survives them. +module = ["test_core", "test_detectors"] +disallow_incomplete_defs = false diff --git a/test/framework/requirements-dev.txt b/test/framework/requirements-dev.txt new file mode 100644 index 000000000..e74637bdf --- /dev/null +++ b/test/framework/requirements-dev.txt @@ -0,0 +1,3 @@ +# Optional: only needed to author YAML suites and to lint. +pyyaml>=6 +ruff>=0.6 diff --git a/test/framework/sbtest/__init__.py b/test/framework/sbtest/__init__.py new file mode 100644 index 000000000..1f4f7377f --- /dev/null +++ b/test/framework/sbtest/__init__.py @@ -0,0 +1,17 @@ +"""sbtest — a component-and-detector framework for large-scale simplyblock test runs. + +Two extension points: **components** do the side-effecting work of a run (drive a workload, +sample paths, follow logs) and can each be enabled or disabled independently; **detectors** +judge the evidence a run produced and can be added without touching collection. + +Importing this package registers the bundled components and detectors, so +`known_detectors()` and `known_components()` are populated after `import sbtest`. +""" + +from . import components as _components # noqa: F401 (registers via decorators) +from . import detectors as _detectors # noqa: F401 +from .core import * # noqa: F401,F403 +from .core import __all__ as _core_all + +__all__ = list(_core_all) +__version__ = "0.1.0" diff --git a/test/framework/sbtest/__main__.py b/test/framework/sbtest/__main__.py new file mode 100644 index 000000000..dd8a8c90e --- /dev/null +++ b/test/framework/sbtest/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/test/framework/sbtest/adapters/__init__.py b/test/framework/sbtest/adapters/__init__.py new file mode 100644 index 000000000..20ec71a38 --- /dev/null +++ b/test/framework/sbtest/adapters/__init__.py @@ -0,0 +1,11 @@ +"""Evidence adapters — where a detector's input comes from. + +`ArchiveEvidence` reads a finished run directory, which is what lets the detector set be +re-run against any past run. A live run uses `LiveEvidence`, assembled by the components +that collected it. +""" + +from .archive import ArchiveEvidence +from .live import LiveEvidence + +__all__ = ["ArchiveEvidence", "LiveEvidence"] diff --git a/test/framework/sbtest/adapters/archive.py b/test/framework/sbtest/adapters/archive.py new file mode 100644 index 000000000..b221b7a22 --- /dev/null +++ b/test/framework/sbtest/adapters/archive.py @@ -0,0 +1,432 @@ +"""ArchiveEvidence — read a finished run directory as Evidence. + +This is what makes a check testable against the run that motivated it. The layout it reads +is the one `operator/test/fio_migration_test.py` writes, so every archived fio-mig-* run +becomes a fixture for the whole detector set: + + / + state.json run bookkeeping (migrations, pods, pv/nqn maps) + test.log event log; the fallback when state.json is absent + ana/-mig-N.csv host ANA samples per migration + -fio-N/result.json fio's own summary + -fio-N/fio.log the pod's container log (verify failures live here) + -fio-N/timeseries.csv per-second IOPS + spdk-[-proxy].txt host-sourced container logs + operator.txt / webappapi.txt likewise + dmesg-.txt kernel ring buffer per storage worker + nvme-controllers.json fabric snapshot, when the run collected one + +Nothing here is required. A missing file makes the corresponding accessor return nothing, +which makes the detectors that need it report themselves skipped rather than clean — the +distinction the whole findings model is built around. +""" + +from __future__ import annotations + +import csv +import glob +import json +import os +import re +from collections.abc import Iterator +from datetime import UTC, datetime + +from ..core import ( + AnaSample, + ControlEvent, + FioJob, + IopsSample, + LogSpan, + Migration, + NvmeController, +) + + +def _dt(v: object) -> datetime | None: + if not v: + return None + if isinstance(v, datetime): + return v if v.tzinfo else v.replace(tzinfo=UTC) + try: + d = datetime.fromisoformat(str(v).replace("Z", "+00:00")) + except ValueError: + return None + return d if d.tzinfo else d.replace(tzinfo=UTC) + + +_TS_PATTERNS = ( + # CRI container log: 2026-08-19T22:23:18.994807954Z stderr F + (re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})"), "%Y-%m-%dT%H:%M:%S"), + # dmesg -T: [Thu Aug 20 05:46:57 2026] + (re.compile(r"^\[(\w{3} \w{3}\s+\d+ \d{2}:\d{2}:\d{2} \d{4})\]"), + "%a %b %d %H:%M:%S %Y"), +) + + +def _log_line_ts(raw: str) -> datetime | None: + """The timestamp of a collected log line, whichever of the known formats it uses.""" + for rx, fmt in _TS_PATTERNS: + m = rx.match(raw) + if m: + try: + return datetime.strptime(m.group(1), fmt).replace(tzinfo=UTC) + except ValueError: + return None + return None + + +class ArchiveEvidence: + """Evidence backed by a finished run directory.""" + + def __init__(self, outdir: str) -> None: + self.outdir = os.path.abspath(outdir) + if not os.path.isdir(self.outdir): + raise FileNotFoundError(self.outdir) + self._state = self._load_state() + self.run_id = self._state.get("run_id") or os.path.basename(self.outdir).replace( + "fio-mig-", "fiomig-") + self._ana_cache: dict[str, list[AnaSample]] = {} + + # ── bookkeeping ──────────────────────────────────────────────────────────────── + + def _load_state(self) -> dict: + p = os.path.join(self.outdir, "state.json") + if os.path.exists(p): + try: + with open(p) as fh: + loaded = json.load(fh) + if isinstance(loaded, dict): + return loaded + except (OSError, json.JSONDecodeError): + pass + return self._state_from_test_log() + + def _state_from_test_log(self) -> dict: + """Rebuild just enough from test.log when state.json is missing. + + Only the migration timeline is recovered, because that is what attribution needs and + it is the one thing the event log records unambiguously. + """ + p = os.path.join(self.outdir, "test.log") + if not os.path.exists(p): + return {} + migs: dict[str, dict] = {} + start_re = re.compile( + r"^(\S+) .*MIGRATION START\s+(\S+)\s+.*?(?:pod=(\S+))?\s*(?:pv=(\S+))?") + stop_re = re.compile(r"^(\S+) .*MIGRATION STOP\s+(\S+)\s+phase=(\S+)") + with open(p, errors="ignore") as fh: + for line in fh: + m = start_re.match(line) + if m and "MIGRATION START" in line: + name = m.group(2) + rec = migs.setdefault(name, {"name": name}) + rec["start"] = m.group(1) + for key, pat in (("pod", r"pod=(\S+)"), ("pv", r"pv=(\S+)"), + ("source", r"source=(\S+)"), ("target", r"target=(\S+)")): + mm = re.search(pat, line) + if mm: + rec[key] = mm.group(1) + mm = re.search(r"moves along: ([^)]*)\)", line) + rec["group_pvs"] = ([x.strip() for x in mm.group(1).split(",")] + [rec.get("pv", "")] + if mm else [rec.get("pv", "")]) + continue + m = stop_re.match(line) + if m: + rec = migs.setdefault(m.group(2), {"name": m.group(2)}) + rec["end"] = m.group(1) + rec["phase"] = m.group(3) + mm = re.search(r"error='([^']*)'", line) + if mm: + rec["error"] = mm.group(1) + return {"migrations": list(migs.values())} + + # ── Evidence protocol ────────────────────────────────────────────────────────── + + def migrations(self) -> list[Migration]: + # The framework's own driver writes migrations.json; the older harness writes them + # inside state.json. Prefer whichever is present, so one reader serves both. + p = os.path.join(self.outdir, "migrations.json") + if os.path.exists(p) and not self._state.get("migrations"): + from ..components.migration import migrations_from_file + try: + return migrations_from_file(p) + except (OSError, json.JSONDecodeError, ValueError): + pass + out = [] + for m in self._state.get("migrations", []): + start = _dt(m.get("start")) + if not start: + continue + cut = {} + for node, ts in (m.get("ana_cutover") or {}).items(): + d = _dt(ts) + if d: + cut[node] = d + out.append(Migration( + name=m.get("name", ""), start=start, end=_dt(m.get("end")), + phase=m.get("phase", ""), source=m.get("source", ""), + target=m.get("target", ""), pv=m.get("pv", ""), pod=m.get("pod", ""), + members=[x for x in (m.get("group_pvs") or []) if x], + error=m.get("error", "") or "", cutover=cut)) + out.sort(key=lambda x: x.start) + return out + + def ana_samples(self, migration: str) -> list[AnaSample]: + if migration in self._ana_cache: + return self._ana_cache[migration] + # state.json records an absolute path from the original host, which may not exist + # here; resolve by name inside this directory instead. + cands = [os.path.join(self.outdir, "ana", f"{migration}.csv")] + cands += glob.glob(os.path.join(self.outdir, "ana", f"*{migration}.csv")) + path = next((c for c in cands if os.path.exists(c)), None) + samples: list[AnaSample] = [] + if path: + grouped: dict[tuple, AnaSample] = {} + try: + with open(path, newline="") as fh: + for r in csv.DictReader(fh): + ts = _dt(r.get("ts")) + if not ts: + continue + key = (ts, r.get("node", ""), r.get("address", "")) + s = grouped.get(key) + if s is None: + s = AnaSample(ts=ts, node=r.get("node", ""), + address=r.get("address", ""), + state=r.get("ctrl_state", ""), + ana={}, phase=r.get("phase", ""), + role=r.get("role", "")) + grouped[key] = s + if r.get("nsid"): + s.ana[int(r["nsid"])] = r.get("ana_state", "") + except OSError: + pass + samples = [grouped[k] for k in sorted(grouped)] + self._ana_cache[migration] = samples + return samples + + def pods(self) -> list[str]: + pods = self._state.get("pods") + if pods: + return list(pods) + return sorted( + d for d in os.listdir(self.outdir) + if os.path.isdir(os.path.join(self.outdir, d)) and "-fio-" in d) + + def fio_jobs(self) -> list[FioJob]: + out = [] + for pod in self.pods(): + p = os.path.join(self.outdir, pod, "result.json") + if not os.path.exists(p): + continue + try: + with open(p) as fh: + res = json.load(fh) + except (OSError, json.JSONDecodeError): + continue + for job in res.get("jobs", []): + rd, wr = job.get("read", {}), job.get("write", {}) + out.append(FioJob( + pod=pod, error=int(job.get("error", 0) or 0), + read_iops=float(rd.get("iops", 0) or 0), + write_iops=float(wr.get("iops", 0) or 0), + total_iops=float(rd.get("iops", 0) or 0) + float(wr.get("iops", 0) or 0))) + return out + + def fio_timeseries(self, pod: str) -> list[IopsSample]: + p = os.path.join(self.outdir, pod, "timeseries.csv") + if not os.path.exists(p): + return [] + out = [] + # The offset column is `second` in the harness's own CSVs and `t`/`offset_s` in + # fio's raw logs. Reading only the latter silently placed every sample at offset 0, + # which does not look like a parse failure — the IOPS column still parsed, so the + # series was the right length with the whole run collapsed onto one instant. Anything + # that locates an outage in time was reading a flatline. + def _first(r: dict[str, str], *keys: str) -> str: + for k in keys: + v = r.get(k) + if v not in (None, ""): + return v + return "" + + try: + with open(p, newline="") as fh: + for r in csv.DictReader(fh): + try: + off = int(float(_first(r, "second", "t", "offset_s") or 0)) + tot = float(_first(r, "total_iops") or 0) + except (TypeError, ValueError): + continue + out.append(IopsSample(offset_s=off, + wall=_dt(_first(r, "wall", "wall_clock")), + total_iops=tot)) + except OSError: + return [] + out.sort(key=lambda s: s.offset_s) + return out + + def fio_log(self, pod: str) -> Iterator[str]: + p = os.path.join(self.outdir, pod, "fio.log") + if not os.path.exists(p): + return iter(()) + return self._lines(p) + + def container_logs(self) -> list[str]: + names = [] + for p in sorted(glob.glob(os.path.join(self.outdir, "*.txt"))): + base = os.path.basename(p)[:-4] + if base in ("test", "REVIEW"): + continue + names.append(base) + return names + + def container_log(self, name: str) -> Iterator[str]: + p = os.path.join(self.outdir, f"{name}.txt") + if not os.path.exists(p): + return iter(()) + return self._lines(p) + + def run_window(self) -> tuple[datetime | None, datetime | None]: + """From the event log's first and last stamps, falling back to the migrations. + + test.log is preferred because it brackets the whole run including setup and + collection, while the migrations only cover the part that was migrating. + """ + # run.json is authoritative when present: the run wrote it, rather than it being + # inferred from whatever happened to get logged. + p = os.path.join(self.outdir, "run.json") + if os.path.exists(p): + try: + with open(p) as fh: + rec = json.load(fh) + start, end = _dt(rec.get("start")), _dt(rec.get("end")) + if start: + return start, end + except (OSError, json.JSONDecodeError): + pass + + p = os.path.join(self.outdir, "test.log") + if os.path.exists(p): + first = last = None + try: + with open(p, errors="ignore") as fh: + for line in fh: + ts = _dt(line[:20].strip()) + if ts: + first = first or ts + last = ts + except OSError: + pass + if first: + return first, last + migs = self.migrations() + if migs: + ends = [m.end for m in migs if m.end] + return migs[0].start, (max(ends) if ends else None) + return None, None + + def control_events(self) -> list[ControlEvent]: + p = os.path.join(self.outdir, "cluster-events.json") + if not os.path.exists(p): + return [] + try: + with open(p) as fh: + raw = json.load(fh) + except (OSError, json.JSONDecodeError): + return [] + out = [] + for e in raw if isinstance(raw, list) else []: + ts = _dt(str(e.get("Date", "")).replace(" ", "T")) + if not ts: + continue + out.append(ControlEvent( + ts=ts, level=str(e.get("Level", "")), kind=str(e.get("Event", "")), + message=str(e.get("Message", "")), + subject=str(e.get("NodeId") or e.get("Storage_ID") or ""))) + out.sort(key=lambda x: x.ts) + return out + + def log_spans(self) -> list[LogSpan]: + """First and last timestamp of every collected log, however it stamps its lines. + + Three formats appear in one artifact directory — CRI (`2026-...Z stderr F`), dmesg + ctime (`[Thu Aug 20 ...]`) and dmesg ISO — so each is tried per line until one sticks. + + The bounds are the **minimum and maximum** timestamp, not the first and last line's. + That distinction is not pedantry: an artifact holding several containers is written one + container after another, so it is not in time order, and reading the last line gives + whatever the last *container* happened to say. Doing that made a 52 MiB log covering + two hours look like thirteen seconds. + """ + spans = [] + for name in self.container_logs(): + lo = hi = None + lines = 0 + for raw in self.container_log(name): + lines += 1 + ts = _log_line_ts(raw) + if ts is None: + continue + if lo is None or ts < lo: + lo = ts + if hi is None or ts > hi: + hi = ts + spans.append(LogSpan(name=name, first=lo, last=hi, lines=lines)) + return spans + + def cluster_uuid(self) -> str: + """The run's cluster, from the NQNs it recorded, else from the event log.""" + for nqn in (self._state.get("nqn_of") or {}).values(): + m = re.search(r"simplyblock:([0-9a-f-]{36}):", str(nqn)) + if m: + return m.group(1) + p = os.path.join(self.outdir, "test.log") + if os.path.exists(p): + try: + with open(p, errors="ignore") as fh: + for line in fh: + m = re.search(r"live cluster.*=\s*([0-9a-f-]{36})", line) + if m: + return m.group(1) + except OSError: + pass + return "" + + def nvme_controllers_pre(self) -> list[NvmeController]: + """The fabric as it was *before* the run — the state the run inherited. + + Optional, and not part of the Evidence protocol: only nvme.dirty-start wants it, and + a detector that needs it can ask with getattr and skip when it is absent. + """ + return self._controllers_from("nvme-controllers-pre.json") + + def nvme_controllers(self) -> list[NvmeController]: + return self._controllers_from("nvme-controllers-post.json") or \ + self._controllers_from("nvme-controllers.json") + + def _controllers_from(self, filename: str) -> list[NvmeController]: + p = os.path.join(self.outdir, filename) + if not os.path.exists(p): + return [] + try: + with open(p) as fh: + raw = json.load(fh) + except (OSError, json.JSONDecodeError): + return [] + out = [] + for c in raw if isinstance(raw, list) else raw.get("controllers", []): + out.append(NvmeController( + node=c.get("node", ""), name=c.get("name", ""), nqn=c.get("nqn", ""), + address=c.get("address", ""), state=c.get("state", ""), + namespaces={int(k): v for k, v in (c.get("namespaces") or {}).items()}, + ctrl_loss_tmo=c.get("ctrl_loss_tmo"))) + return out + + # ── helpers ──────────────────────────────────────────────────────────────────── + + @staticmethod + def _lines(path: str) -> Iterator[str]: + """Stream a log line by line. These files reach tens of megabytes.""" + with open(path, errors="ignore") as fh: + yield from fh diff --git a/test/framework/sbtest/adapters/live.py b/test/framework/sbtest/adapters/live.py new file mode 100644 index 000000000..1f45a2de8 --- /dev/null +++ b/test/framework/sbtest/adapters/live.py @@ -0,0 +1,37 @@ +"""LiveEvidence — evidence assembled from a run that just happened. + +A thin thing on purpose. Components write their artifacts into the run directory in the +same layout an archive uses, and record what they observed on the timeline; so a live run's +evidence is the archive reader pointed at the directory being written, overlaid with the +timeline for the parts that are not files yet. + +Keeping the two paths this close is deliberate: if live evidence and archived evidence +diverge, a check that passes during a run can fail on replay, and then nobody trusts either. +""" + +from __future__ import annotations + +from ..core import AnaSample, Migration, NvmeController, RunContext +from .archive import ArchiveEvidence + + +class LiveEvidence(ArchiveEvidence): + def __init__(self, ctx: RunContext) -> None: + super().__init__(ctx.outdir) + self.ctx = ctx + self.run_id = ctx.run_id + self._live_ana: dict[str, list[AnaSample]] = ctx.shared.get("ana.samples", {}) + self._live_migs: list[Migration] = ctx.shared.get("migrations", []) + self._live_ctrls: list[NvmeController] = ctx.shared.get("nvme.controllers", []) + + def migrations(self) -> list[Migration]: + return self._live_migs or super().migrations() + + def ana_samples(self, migration: str) -> list[AnaSample]: + return self._live_ana.get(migration) or super().ana_samples(migration) + + def nvme_controllers(self) -> list[NvmeController]: + return self._live_ctrls or super().nvme_controllers() + + def cluster_uuid(self) -> str: + return str(self.ctx.shared.get("cluster.uuid") or super().cluster_uuid()) diff --git a/test/framework/sbtest/cli.py b/test/framework/sbtest/cli.py new file mode 100644 index 000000000..a83a1af1c --- /dev/null +++ b/test/framework/sbtest/cli.py @@ -0,0 +1,241 @@ +"""sbtest command line. + + sbtest detectors what can judge a run, and with which options + sbtest components what can run during a run + sbtest analyze [...] judge a finished run directory + sbtest collect [...] observe a cluster, then collect the evidence + sbtest run --suite [...] drive a full test run, then judge it + +`analyze` is the one to reach for first. It runs the whole detector set over an artifact +directory — including every archived fio-mig-* run — so a check can be written or fixed and +tried immediately against the run that motivated it, instead of against the next four-hour +run. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +from . import __version__ +from .adapters import ArchiveEvidence +from .core import ( + Config, + Logger, + RunContext, + Runner, + apply_cli_toggles, + known_components, + known_detectors, + load, + now_utc, + suite_path, +) +from .detectors.ana import freeze_summary + + +def _cfg(args: argparse.Namespace, known_c: list[str], known_d: list[str]) -> Config: + path = suite_path(args.suite) if args.suite else None + if args.suite and not path: + raise SystemExit(f"suite not found: {args.suite}") + cfg = load(path, known_c, known_d) + apply_cli_toggles(cfg.components, args.enable_component, args.disable_component) + apply_cli_toggles(cfg.detectors, args.enable_detector, args.disable_detector) + return cfg + + +def _first_doc_line(cls: type) -> str: + """The first line of a class docstring, or "" — a class may have none at all.""" + doc = (cls.__doc__ or "").strip() + return doc.splitlines()[0] if doc else "" + + +def cmd_list(args: argparse.Namespace) -> int: + if args.what == "detectors": + for name, dcls in known_detectors().items(): + d = dcls() + print(name) + print(f" {d.summary or _first_doc_line(dcls)}") + for k, v in sorted(d.defaults().items()): + print(f" {k} = {v!r}") + else: + for name, ccls in known_components().items(): + c = ccls() + print(name) + print(f" {c.summary or _first_doc_line(ccls)}") + for k, v in sorted(c.defaults().items()): + print(f" {k} = {v!r}") + return 0 + + +def cmd_analyze(args: argparse.Namespace) -> int: + ev = ArchiveEvidence(args.rundir) + outdir = args.outdir or args.rundir + log = Logger(os.path.join(outdir, "sbtest-analyze.log") if args.write_log else None, + verbose=args.verbose) + ctx = RunContext(run_id=ev.run_id, outdir=outdir, log=log) + + cfg = _cfg(args, list(known_components()), list(known_detectors())) + # analyze never runs components; make that explicit rather than silently ignoring them. + if cfg.components.enabled: + log.info("analyze: components are not run (" + + ", ".join(sorted(cfg.components.enabled)) + ")") + cfg.components.enabled = {} + + log.info(f"analyzing {ev.outdir} (run {ev.run_id})") + runner = Runner(cfg, ctx).build() + runner.judge(ev) + + if args.freeze_table: + rows = freeze_summary(ev) + if rows: + log.info("cutover freezes per migration (>1 has predicted write loss exactly):") + for name, n, worst in sorted(rows, key=lambda r: (-r[1], r[0])): + mark = " <== re-froze" if n > 1 else "" + log.info(f" {name:28} freezes={n:<3} worst={worst:.0f}s{mark}") + + report = runner.emit(json_name=args.json_name) + from .core.runner import findings_by_subject_table + correlated = findings_by_subject_table(report) + if correlated: + log.info("subjects with more than one finding (the strongest signal these runs give):") + for line in correlated: + log.info(f" {line}") + return 1 if report.failed else 0 + + +def cmd_collect(args: argparse.Namespace) -> int: + os.makedirs(args.outdir, exist_ok=True) + run_id = args.run_id or f"sbtest-{int(now_utc().timestamp())}" + log = Logger(os.path.join(args.outdir, "sbtest.log"), verbose=args.verbose) + ctx = RunContext(run_id=run_id, outdir=args.outdir, log=log) + cfg = _cfg(args, list(known_components()), list(known_detectors())) + if not cfg.components.enabled: + raise SystemExit("collect: no components enabled — pass --enable-component or a suite") + + runner = Runner(cfg, ctx).build() + log.info(f"collecting into {args.outdir} for {args.duration}s") + try: + runner.setup() + runner.start() + if args.duration: + runner.run_for(float(args.duration)) + runner.stop() + runner.collect() + finally: + runner.teardown() + + if args.judge: + runner.judge(ArchiveEvidence(args.outdir)) + report = runner.emit(json_name=args.json_name) + return 1 if report.failed else 0 + log.info(f"collected into {args.outdir}; judge it with: sbtest analyze {args.outdir}") + return 0 + + +def cmd_run(args: argparse.Namespace) -> int: + """A driven run: components create the load, then the same detectors judge it. + + The difference from `collect` is only what the components do — one of them drives + migrations instead of watching them. Everything after that is identical, which is the + point: a live run and an archived one are judged by the same code, so a verdict from + either means the same thing. + """ + cfg = _cfg(args, list(known_components()), list(known_detectors())) + run_id = args.run_id or f"{cfg.run_id_prefix}-{int(now_utc().timestamp())}" + outdir = args.outdir or os.path.join(cfg.outdir, run_id) + os.makedirs(outdir, exist_ok=True) + log = Logger(os.path.join(outdir, "test.log"), verbose=args.verbose) + ctx = RunContext(run_id=run_id, outdir=outdir, log=log) + # Read by the components that create cluster objects. A kept run leaves the volumes and + # the CRs behind for inspection — the reason most post-mortems are possible at all. + ctx.shared["keep"] = bool(args.keep) + + if not cfg.components.enabled: + raise SystemExit("run: no components enabled — pass --enable-component or a suite") + drivers = [n for n in cfg.components.enabled if n.endswith(".driver")] + if not drivers: + log.warn("no driver component is enabled, so nothing will create load — this run " + "will only observe. Use `sbtest collect` if that is what you meant") + + runner = Runner(cfg, ctx).build() + log.info(f"run {run_id} -> {outdir} (duration {args.duration or 0:.0f}s, " + f"keep={bool(args.keep)})") + try: + runner.setup() + runner.start() + if args.duration: + runner.run_for(float(args.duration)) + runner.stop() + runner.collect() + finally: + runner.teardown() + + # Judged from the artifact directory rather than from memory, so the verdict comes from + # exactly the evidence someone else would re-analyse later. A run that reports PASS from + # in-memory state it never wrote down is not reproducible. + runner.judge(ArchiveEvidence(outdir)) + report = runner.emit(json_name=args.json_name) + return 1 if report.failed else 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="sbtest", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--version", action="version", version=f"sbtest {__version__}") + sub = p.add_subparsers(dest="cmd", required=True) + + def common(sp: argparse.ArgumentParser) -> None: + sp.add_argument("--suite", help="suite file, or a bare name under sbtest/suites/") + sp.add_argument("--enable-detector", action="append", default=[], metavar="NAME") + sp.add_argument("--disable-detector", action="append", default=[], metavar="NAME") + sp.add_argument("--enable-component", action="append", default=[], metavar="NAME") + sp.add_argument("--disable-component", action="append", default=[], metavar="NAME") + sp.add_argument("--json-name", default="findings.json", + help="where to write findings inside the run directory") + sp.add_argument("-v", "--verbose", action="store_true") + + for what in ("detectors", "components"): + sp = sub.add_parser(what, help=f"list the available {what}") + sp.set_defaults(func=cmd_list, what=what) + + sp = sub.add_parser("analyze", help="judge a finished run directory") + sp.add_argument("rundir") + sp.add_argument("--outdir", help="where to write findings (default: the run directory)") + sp.add_argument("--write-log", action="store_true", + help="also write sbtest-analyze.log into the output directory") + sp.add_argument("--freeze-table", action="store_true", + help="print freezes per migration alongside the findings") + common(sp) + sp.set_defaults(func=cmd_analyze) + + sp = sub.add_parser("collect", help="run the collection components against a cluster") + sp.add_argument("outdir") + sp.add_argument("--duration", type=float, default=0.0, + help="seconds to keep components running before collecting") + sp.add_argument("--run-id") + sp.add_argument("--judge", action="store_true", help="judge the result when done") + common(sp) + sp.set_defaults(func=cmd_collect) + + sp = sub.add_parser("run", help="drive a full test run against a cluster, then judge it") + sp.add_argument("--duration", type=float, default=0.0, + help="seconds to run before stopping the drivers and collecting") + sp.add_argument("--outdir", help="where to write artifacts " + "(default: /)") + sp.add_argument("--run-id") + sp.add_argument("--keep", action="store_true", + help="leave the volumes, pods and CRs behind for inspection") + common(sp) + sp.set_defaults(func=cmd_run) + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + return int(args.func(args)) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/test/framework/sbtest/components/__init__.py b/test/framework/sbtest/components/__init__.py new file mode 100644 index 000000000..b472f9819 --- /dev/null +++ b/test/framework/sbtest/components/__init__.py @@ -0,0 +1,9 @@ +"""Bundled components. Importing this module registers them all. + +Grouped by what they touch: `logs` collects container logs (live or post-run), `nvme` +observes the host fabric, `events` pulls the control plane's own event log. +""" + +from . import events, logs, migration, nvme, workload # noqa: F401 + +__all__ = ["events", "logs", "migration", "nvme", "workload"] diff --git a/test/framework/sbtest/components/events.py b/test/framework/sbtest/components/events.py new file mode 100644 index 000000000..2e4600033 --- /dev/null +++ b/test/framework/sbtest/components/events.py @@ -0,0 +1,60 @@ +"""Control-plane event log collection. + +Small, cheap, and disproportionately useful: it is the only source that says what the +control plane *thought* it was doing, which is what turns a host-side symptom into a +diagnosis. +""" + +from __future__ import annotations + +import json +from typing import Any + +from ..core import Component, RunContext, component +from . import kube + + +@component +class ClusterEvents(Component): + """Dump the simplyblock cluster event log via sbctl inside a webappapi pod.""" + + name = "cluster.events" + summary = "sbctl cluster get-logs -> cluster-events.json" + + def defaults(self) -> dict[str, Any]: + return {"namespace": "simplyblock", "pod_prefix": "simplyblock-webappapi", + "limit": 50000, "cluster_uuid": None} + + def collect(self, ctx: RunContext) -> None: + pods = kube.list_pods(self.opt("namespace"), [self.opt("pod_prefix")]) + if not pods: + ctx.log.warn(f"{self.name}: no {self.opt('pod_prefix')} pod; skipping") + return + pod = pods[0].name + cluster = self.opt("cluster_uuid") or ctx.shared.get("cluster.uuid") + if not cluster: + out = kube.exec_sh(self.opt("namespace"), pod, + "sbctl cluster list --json 2>/dev/null || true", timeout=60) + try: + items = json.loads(out) + cluster = (items[0].get("UUID") or items[0].get("uuid")) if items else None + except (json.JSONDecodeError, AttributeError, IndexError): + cluster = None + if not cluster: + ctx.log.warn(f"{self.name}: cannot resolve the cluster uuid; skipping") + return + + out = kube.exec_sh( + self.opt("namespace"), pod, + f"sbctl cluster get-logs {cluster} --json --limit={int(self.opt('limit'))}", + timeout=180) + if not out.strip(): + ctx.log.warn(f"{self.name}: sbctl returned nothing") + return + path = ctx.path("cluster-events.json") + with open(path, "w") as fh: + fh.write(out) + try: + ctx.log.info(f"{self.name}: {len(json.loads(out))} entries -> {path}") + except json.JSONDecodeError: + ctx.log.info(f"{self.name}: -> {path} (not valid JSON)") diff --git a/test/framework/sbtest/components/kube.py b/test/framework/sbtest/components/kube.py new file mode 100644 index 000000000..3c3e83166 --- /dev/null +++ b/test/framework/sbtest/components/kube.py @@ -0,0 +1,68 @@ +"""Minimal kubectl plumbing shared by the cluster-touching components. + +Deliberately subprocess-over-kubectl rather than a client library: it keeps the framework +stdlib-only, it works with whatever kubeconfig and context the operator already uses, and +every call it makes is one a human can paste into a terminal when a run misbehaves — which +is most of the debugging value. +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass + + +class KubectlError(RuntimeError): + pass + + +def run(args: list[str], timeout: int = 60, check: bool = True, + stdin: str | None = None) -> subprocess.CompletedProcess[str]: + cp = subprocess.run(["kubectl", *args], input=stdin, capture_output=True, + text=True, timeout=timeout, check=False) + if check and cp.returncode != 0: + raise KubectlError(f"kubectl {' '.join(args)}: {cp.stderr.strip() or cp.returncode}") + return cp + + +def run_bytes(args: list[str], timeout: int = 300) -> bytes: + cp = subprocess.run(["kubectl", *args], capture_output=True, timeout=timeout, check=False) + return cp.stdout + + +@dataclass(frozen=True) +class Pod: + name: str + namespace: str + node: str + containers: tuple[str, ...] + phase: str = "" + + +def list_pods(namespace: str, name_contains: list[str] | None = None) -> list[Pod]: + cp = run(["-n", namespace, "get", "pods", "-o", "json"]) + out = [] + for it in json.loads(cp.stdout).get("items", []): + name = it["metadata"]["name"] + if name_contains and not any(s in name for s in name_contains): + continue + out.append(Pod( + name=name, namespace=namespace, + node=it.get("spec", {}).get("nodeName", "") or "", + containers=tuple(c["name"] for c in it.get("spec", {}).get("containers", [])), + phase=it.get("status", {}).get("phase", "") or "")) + return out + + +def exec_sh(namespace: str, pod: str, script: str, container: str | None = None, + timeout: int = 300) -> str: + args = ["-n", namespace, "exec", pod] + if container: + args += ["-c", container] + args += ["--", "sh", "-c", script] + return run(args, timeout=timeout, check=False).stdout + + +def short(node: str) -> str: + return node.split(".")[0] diff --git a/test/framework/sbtest/components/logs.py b/test/framework/sbtest/components/logs.py new file mode 100644 index 000000000..4650cb310 --- /dev/null +++ b/test/framework/sbtest/components/logs.py @@ -0,0 +1,477 @@ +"""Container-log collection: a live follower and a post-run grab. + +Two components rather than one flag, because they are genuinely different mechanisms with +different failure modes, and the reason to have both is worth stating. + +The kubelet keeps only `containerLogMaxSize x containerLogMaxFiles` per container — 10Mi x 5 +on the clusters this runs against — and a busy SPDK container writes that in well under an +hour. A post-run grab of a four-hour run therefore returns its last forty minutes and +silently drops the rest, which is how several runs' worth of early evidence was lost. + +So: `logs.stream` follows the high-volume logs for the whole run, and `logs.collect` grabs +everything else at the end. Enabling both is the normal configuration; the stream declares +which containers it owns so the grab skips them. +""" + +from __future__ import annotations + +import contextlib +import os +import subprocess +from typing import Any + +from ..core import Component, RunContext, component +from . import kube + +#: Where the host keeps CRI container logs. Mounted read-only into the grabber. +HOST_POD_LOGS = "/var/log/pods" + +#: A busybox-ish image with sh, tail and gzip. The fio image already satisfies this and is +#: guaranteed to be pullable wherever these tests run. +DEFAULT_GRABBER_IMAGE = "quay.io/simplyblock-io/fio:latest" + + +def _log_dir(namespace: str, pod: str, container: str, if_missing: str) -> str: + """Resolve the container's CRI log directory into $d. + + One directory per pod *UID*, so a pod recreated under the same name leaves two behind. + Take the most recently written rather than letting the glob expand to several words — + that would make every quoted use of $d a path that does not exist, which silently + empties the grab for exactly the pods that have been restarted. + """ + return (f'd=$(ls -1dt {HOST_POD_LOGS}/{namespace}_{pod}_*/{container}/ 2>/dev/null | head -1); ' + f'[ -n "$d" ] || {{ {if_missing}; }}; ') + + +#: The live file is ".log"; rotation renames it to ".log.[.gz]" and +#: opens a new one under the same name. Pick it by name, not mtime: a .gz written by the +#: rotation that just happened is briefly the newest file in the directory. +_CURRENT = 'cur=$(ls -1 "$d" 2>/dev/null | grep -E "^[0-9]+\\.log$" | sort -n | tail -1); ' + +#: `gzip -cd`, not `zcat`: the latter is a .Z-only alias on some hosts, which would silently +#: drop every gzipped — i.e. every older — segment. +_CAT_ONE = 'case "$f" in *.gz) gzip -cd "$d$f" 2>/dev/null;; *) cat "$d$f" 2>/dev/null;; esac; ' + + +def dump_script(namespace: str, pod: str, container: str) -> str: + """Everything the host still retains for one container, oldest segment first.""" + return (_log_dir(namespace, pod, container, "exit 0") + + 'for f in $(ls -1tr "$d" 2>/dev/null); do ' + _CAT_ONE + 'done') + + +def stream_script(namespace: str, pod: str, container: str, poll_s: int = 5) -> str: + """Retained segments, then follow the live one for as long as this runs. + + The dump and the follow are one command so there is no gap between them: rotated + segments are catted oldest-first and the live file is handed to `tail -F` from its first + line instead of being catted. + + Three things move the log out from under a follower and all three are handled, because + any one of them leaves the stream silent rather than failing: + + * **rotation** renames the live file and reopens the same name — `tail -F` follows by + name and reopens it itself. + * **a container restart** opens `.log` beside the old one. `tail -F` would wait + forever on a file that still exists and never grows, so the target is re-resolved on + a timer and the old follower killed. + * **a pod recreation** makes a whole new UID directory, which the same re-resolve picks + up. + + Only the first target emits the rotated history; on a later switch the history is the + file that was already being followed. A new target is read from its first line, so a + restart costs latency, not data. + """ + return ( + 'prev=""; tpid=""; ' + 'while :; do ' + + _log_dir(namespace, pod, container, f"sleep {poll_s}; continue") + + _CURRENT + + f'[ -n "$cur" ] || {{ sleep {poll_s}; continue; }}; ' + 'if [ "$d$cur" != "$prev" ]; then ' + ' [ -n "$tpid" ] && kill "$tpid" 2>/dev/null; ' + ' if [ -z "$prev" ]; then ' + ' for f in $(ls -1tr "$d" 2>/dev/null); do ' + ' [ "$f" = "$cur" ] && continue; ' + _CAT_ONE + + ' done; ' + ' fi; ' + ' tail -F -n +1 "$d$cur" 2>/dev/null & tpid=$!; ' + ' prev="$d$cur"; ' + 'fi; ' + f'sleep {poll_s}; ' + 'done') + + +class _GrabberBase(Component): + """Shared management of the privileged pod that reads the host's /var/log/pods. + + A Component rather than a bare mixin: it uses `opt` and `name`, so inheriting states + that dependency instead of leaving it to whatever it happens to be mixed into. + """ + + def _grabber_manifest(self, ctx: RunContext, node: str, name: str, ttl_s: int) -> str: + import json as _json + return _json.dumps({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": name, "namespace": self.opt("namespace"), + "labels": {"sbtest-run": ctx.run_id, "sbtest": "loggrab"}}, + "spec": { + "nodeName": node, "restartPolicy": "Never", + "tolerations": [{"operator": "Exists"}], + "containers": [{ + "name": "grab", "image": self.opt("image"), + "imagePullPolicy": "IfNotPresent", + "command": ["sh", "-c", f"sleep {ttl_s}"], + # privileged + runAsUser 0 is required to read the host's + # /var/log/pods under SELinux: without it the container runs as + # container_t and gets EACCES even as root, and every grab + # silently produces empty files. + "securityContext": {"privileged": True, "runAsUser": 0}, + "volumeMounts": [{"name": "pods", "mountPath": HOST_POD_LOGS, + "readOnly": True}], + }], + "volumes": [{"name": "pods", "hostPath": {"path": HOST_POD_LOGS}}], + }, + }) + + def _start_grabbers(self, ctx: RunContext, nodes: list[str], ttl_s: int) -> dict[str, str]: + # The component name is in the pod name deliberately. Two components can both want a + # grabber on the same node — the follower needs one for the whole run, the post-run + # grab needs one at the end — and a Pod is immutable, so sharing a name means the + # second one to apply fails on a field it is not allowed to change. Which it did: + # logs.collect silently produced empty files for every node logs.stream had claimed. + out: dict[str, str] = {} + slug = self.name.replace(".", "-") + for node in sorted(nodes): + name = f"sbtest-{slug}-{kube.short(node)}-{ctx.run_id}"[:63] + try: + kube.run(["apply", "-f", "-"], + stdin=self._grabber_manifest(ctx, node, name, ttl_s)) + except kube.KubectlError as e: + ctx.log.warn(f"{self.name}: cannot start grabber on {node}: {e}") + continue + out[node] = name + if out: + kube.run(["-n", self.opt("namespace"), "wait", "--for=condition=Ready", + *[f"pod/{n}" for n in out.values()], "--timeout=120s"], + timeout=140, check=False) + ready = {} + for node, name in out.items(): + cp = kube.run(["-n", self.opt("namespace"), "get", "pod", name, + "-o", "jsonpath={.status.phase}"], check=False, timeout=30) + if cp.stdout.strip() == "Running": + ready[node] = name + else: + ctx.log.warn(f"{self.name}: grabber on {node} is not Running; " + "logs from that node will be missing") + return ready + + def _delete_grabbers(self, ctx: RunContext, names: list[str]) -> None: + if not names: + return + kube.run(["-n", self.opt("namespace"), "delete", "pod", *names, + "--ignore-not-found", "--wait=false"], check=False) + + +@component +class LogStream(_GrabberBase): + """Follow the high-volume container logs for the whole run. + + Enable this for anything that outruns kubelet rotation — in practice the SPDK containers + and their proxies. Everything it follows is registered in + `ctx.shared["logs.streamed"]` so `logs.collect` does not overwrite a full-run stream + with the tail the kubelet happens to still have. + """ + + name = "logs.stream" + summary = "follow chosen container logs live, surviving kubelet rotation and restarts" + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + "image": DEFAULT_GRABBER_IMAGE, + # Pods to follow, matched by substring, and the containers within them. + "pods_matching": ["snode-spdk"], + "containers": ["spdk-container", "spdk-proxy-container"], + # Artifact name: "spdk-" and "spdk--proxy" for the SPDK pods. + "name_from": "snode-port", + "ttl_s": 6 * 3600, + "poll_s": 5, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._grabbers: dict[str, str] = {} + self._streams: list[dict] = [] + self._parts: dict[tuple, int] = {} + + # -- naming --------------------------------------------------------------------- + @staticmethod + def _snode_port(pod: str) -> str: + parts = pod.split("-") # snode-spdk-pod-- + return parts[3] if len(parts) > 3 and parts[3].isdigit() else pod + + def _artifact_name(self, pod: str, container: str) -> str: + if self.opt("name_from") == "snode-port": + suffix = "-proxy" if "proxy" in container else "" + return f"spdk-{self._snode_port(pod)}{suffix}" + return f"{pod}-{container}" + + # -- lifecycle ------------------------------------------------------------------ + def setup(self, ctx: RunContext) -> None: + self._pods = kube.list_pods(self.opt("namespace"), self.opt("pods_matching")) + if not self._pods: + ctx.log.warn(f"{self.name}: no pods matching {self.opt('pods_matching')}; " + "nothing to follow") + return + nodes = {p.node for p in self._pods if p.node} + self._grabbers = self._start_grabbers(ctx, sorted(nodes), int(self.opt("ttl_s"))) + # Published so logs.collect can reuse these instead of starting a second privileged + # pod per node. Its TTL already covers the whole run, and by the time collection runs + # the followers have stopped, so the pod is idle and free to be exec'd into again. + ctx.shared.setdefault("logs.grabbers", {}).update(self._grabbers) + + def start(self, ctx: RunContext) -> None: + streamed = ctx.shared.setdefault("logs.streamed", set()) + for p in getattr(self, "_pods", []): + grab = self._grabbers.get(p.node) + if not grab: + continue + for container in self.opt("containers"): + if container not in p.containers: + continue + self._attach(ctx, grab, p, container) + streamed.add((p.name, container)) + if self._streams: + ctx.log.info(f"{self.name}: following {len(self._streams)} container log(s) " + "for the whole run") + + def _attach(self, ctx: RunContext, grabber: str, pod: kube.Pod, container: str) -> None: + base = self._artifact_name(pod.name, container) + key = (pod.name, container) + part = self._parts.get(key, 0) + # A re-attach goes to a new file rather than appending: it re-reads whatever is + # still retained, which would duplicate a stretch of the previous part in the + # middle of the file. Separate parts stay internally ordered, which is what makes + # them readable. + name = f"{base}.txt" if part == 0 else f"{base}.part{part}.txt" + path = ctx.path(name) + try: + fh = open(path, "ab") # noqa: SIM115 + proc = subprocess.Popen( # noqa: S603 + ["kubectl", "-n", self.opt("namespace"), "exec", grabber, "--", "sh", "-c", + stream_script(pod.namespace, pod.name, container, int(self.opt("poll_s")))], + stdout=fh, stderr=subprocess.DEVNULL) + except Exception as e: # noqa: BLE001 + ctx.log.warn(f"{self.name}: cannot follow {pod.name}/{container}: {e}") + return + self._parts[key] = part + 1 + self._streams.append({"proc": proc, "fh": fh, "path": path, "grabber": grabber, + "pod": pod, "container": container}) + + def tick(self, ctx: RunContext) -> None: + """Re-attach followers that died. Cheap enough to run on every tick.""" + for st in list(self._streams): + if st["proc"].poll() is None: + continue + self._streams.remove(st) + with contextlib.suppress(Exception): + st["fh"].close() + size = os.path.getsize(st["path"]) if os.path.exists(st["path"]) else 0 + ctx.log.warn(f"{self.name}: follower for {st['pod'].name}/{st['container']} " + f"ended early (rc={st['proc'].returncode}, {size / 1048576:.1f} MiB); " + "re-attaching into a new part") + ctx.timeline.record("logs.stream.reattach", subject=st["pod"].name, + container=st["container"]) + self._attach(ctx, st["grabber"], st["pod"], st["container"]) + + def stop(self, ctx: RunContext) -> None: + total = 0 + for st in self._streams: + proc = st["proc"] + try: + proc.terminate() + proc.wait(timeout=15) + except Exception: # noqa: BLE001 + with contextlib.suppress(Exception): + proc.kill() + with contextlib.suppress(Exception): + st["fh"].flush() + st["fh"].close() + if os.path.exists(st["path"]): + total += os.path.getsize(st["path"]) + if self._streams: + ctx.log.info(f"{self.name}: stopped {len(self._streams)} follower(s); " + f"{total / 1048576:.1f} MiB captured live") + self._streams = [] + + def teardown(self, ctx: RunContext) -> None: + self.stop(ctx) # idempotent; covers a run that failed before stop + for node in self._grabbers: + ctx.shared.get("logs.grabbers", {}).pop(node, None) + self._delete_grabbers(ctx, sorted(self._grabbers.values())) + self._grabbers = {} + + +@component +class LogCollect(_GrabberBase): + """Grab container logs from the hosts at the end of the run. + + Correct for anything that fits inside what the kubelet retains, which is everything + except the SPDK containers on a long run. Skips whatever `logs.stream` followed. + """ + + name = "logs.collect" + summary = "grab container logs from each host's /var/log/pods after the run" + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + "control_plane_namespace": "simplyblock", + "image": DEFAULT_GRABBER_IMAGE, + #: [{pods: [substr], containers: [name]|"all", name_from: ..., namespace: ...}] + "targets": [ + {"pods": ["snode-spdk"], "containers": ["spdk-container", "spdk-proxy-container"], + "name_from": "snode-port"}, + {"pods": ["operator", "webappapi"], "containers": "all", + "namespace": "simplyblock", "name_from": "pod-key"}, + # One artifact per container, not per pod. The tasks pod runs seventeen + # independent runners, so merging them produces a 50 MiB file that is not in + # time order — which makes its time span meaningless and stops a pattern being + # scoped to the one runner you care about. The container names are already + # unique and descriptive, so they are the artifact names. + {"pods": ["tasks"], "containers": "all", + "namespace": "simplyblock", "name_from": "container"}, + # The CSI driver is where the connects, the path reconcilers and + # NodeStage/NodePublish actually happen, so it is the log that says what the + # *host side* did and why. Kept per node rather than merged: the node plugin + # reconciles per host, so "which node" is the first question about anything + # it did, and a merged file loses it. + {"pods": ["simplyblock-csi-node"], "containers": ["csi-node"], + "namespace": "simplyblock", "name_from": "pod-node", "name": "csi-node"}, + # The node-side agent the control plane talks to on each host. It is what + # starts and probes the SPDK process, which puts it on the causal path of + # every "the node went offline" event — including the liveness check that + # concluded SPDK was dead because a Kubernetes API call blipped. + {"pods": ["simplyblock-storage-node-ds"], "containers": "all", + "namespace": "default", "name_from": "pod-node", "name": "snode-api"}, + {"pods": ["simplyblock-csi-controller"], "containers": "all", + "namespace": "simplyblock", "name_from": "container"}, + ], + "ttl_s": 1800, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._grabbers: dict[str, str] = {} + #: Only the ones this component created — the rest belong to whoever published them + #: and are that component's to remove. + self._own: dict[str, str] = {} + + def collect(self, ctx: RunContext) -> None: + streamed = ctx.shared.get("logs.streamed", set()) + plan: list[tuple[kube.Pod, str, str]] = [] + for target in self.opt("targets"): + ns = target.get("namespace", self.opt("namespace")) + for p in kube.list_pods(ns, target["pods"]): + wanted = (list(p.containers) if target.get("containers") == "all" + else [c for c in target["containers"] if c in p.containers]) + for c in wanted: + if (p.name, c) in streamed: + continue + plan.append((p, c, self._name_for(target, p, c))) + if not plan: + return + + nodes = {p.node for p, _c, _n in plan if p.node} + self._grabbers = self._start_grabbers(ctx, sorted(nodes), int(self.opt("ttl_s"))) + + grouped: dict[str, list[tuple[kube.Pod, str]]] = {} + for p, c, artifact in plan: + grouped.setdefault(artifact, []).append((p, c)) + + for artifact, items in sorted(grouped.items()): + path = ctx.path(f"{artifact}.txt") + with open(path, "wb") as fh: + for p, c in items: + grab = self._grabbers.get(p.node) + if not grab: + continue + if len(items) > 1: # several containers share one artifact; header them + fh.write(f"==================== {p.name} / {c} " + f"({kube.short(p.node)}) ====================\n".encode()) + data = kube.run_bytes( + ["-n", self.opt("namespace"), "exec", grab, "--", "sh", "-c", + dump_script(p.namespace, p.name, c)]) + fh.write(data) + if not data: + # The dump script swallows read errors, so an empty grab is + # otherwise indistinguishable from "this container logged nothing". + ctx.log.warn(f"{self.name}: empty grab for {p.name}/{c}") + ctx.log.info(f"{self.name}: {artifact}.txt " + f"({os.path.getsize(path) / 1048576:.1f} MiB)") + + @staticmethod + def _name_for(target: dict, pod: kube.Pod, container: str) -> str: + mode = str(target.get("name_from", "pod-container")) + if mode == "snode-port": + parts = pod.name.split("-") + port = parts[3] if len(parts) > 3 and parts[3].isdigit() else pod.name + return f"spdk-{port}{'-proxy' if 'proxy' in container else ''}" + if mode == "pod-key": + for key in target["pods"]: + if key in pod.name: + return str(key) + if mode == "container": + return container + if mode == "pod-node": + # "-": one artifact per node, which is how a DaemonSet's behaviour is + # actually reasoned about. `name` overrides the matched substring, because the + # generated pod names are long and carry nothing a reader wants. + key = target.get("name") or next( + (k for k in target["pods"] if k in pod.name), pod.name) + return f"{key}-{kube.short(pod.node)}" if pod.node else str(key) + return f"{pod.name}-{container}" + + def teardown(self, ctx: RunContext) -> None: + # Only what this component created. Deleting a grabber it merely borrowed would pull + # the pod out from under the component that owns it. + self._delete_grabbers(ctx, sorted(self._own.values())) + self._grabbers = {} + self._own = {} + + +@component +class Dmesg(Component): + """Kernel ring buffer from each storage worker. + + Rotation-limited in a different way from a container log: the ring buffer is a fixed + size, so a node that is logging heavily keeps only minutes of it — while a quiet one + keeps hours, which is the real hazard. A buffer spanning ten hours contains the previous + runs' damage, and counting that against this run is how a green build inherits its + predecessor's mess. + + That is why the timestamp format matters here more than it looks. `dmesg -T` renders + *local* time with no offset, so an event cannot be placed against a run window recorded + in UTC without assuming the two agree. `--time-format=iso` emits an offset, which makes + the comparison sound; it is preferred, with `-T` kept as a fallback for older util-linux. + """ + + name = "host.dmesg" + summary = "dmesg from each storage worker, ISO-timestamped so events can be placed in time" + + def defaults(self) -> dict[str, Any]: + return {"namespace": "default", "pods_matching": ["snode-spdk"], + "container": "spdk-container"} + + def collect(self, ctx: RunContext) -> None: + for p in kube.list_pods(self.opt("namespace"), self.opt("pods_matching")): + if not p.node: + continue + data = kube.run_bytes( + ["-n", p.namespace, "exec", p.name, "-c", self.opt("container"), "--", + "sh", "-c", "dmesg --time-format=iso 2>/dev/null || dmesg -T"], + timeout=120) + if not data: + ctx.log.warn(f"{self.name}: empty dmesg from {kube.short(p.node)}") + with open(ctx.path(f"dmesg-{kube.short(p.node)}.txt"), "wb") as fh: + fh.write(data) diff --git a/test/framework/sbtest/components/migration.py b/test/framework/sbtest/components/migration.py new file mode 100644 index 000000000..540a590eb --- /dev/null +++ b/test/framework/sbtest/components/migration.py @@ -0,0 +1,445 @@ +"""The volume-migration driver: the component that makes migrations happen. + +This is the first component that *is* the run rather than an observer of it, and that changes +two things about it. + +It is `required`, so a failure in its setup aborts instead of being recorded — a run whose +driver never started would otherwise report a clean pass for a test that never happened. + +And it drives the ANA sampler. Sampling only means something when the samples are attributed +to a migration, so the driver tells the sampler when each one begins, what phase it is in, and +when it ends. That is the one genuine dependency between components here, and it goes through +`ctx.shared` so the sampler stays optional: with sampling disabled the driver still migrates, +and the ANA detectors report themselves skipped rather than clean. + +Three things it reads from the backend rather than assuming: + +**Where the volume is now.** The cluster's own rebalancer moves volumes without any Kubernetes +object changing, so a source cached at setup is wrong by the tenth migration — and a wrong +source means picking a target the volume already lives on, which the operator rejects and +which reads as a product bug. + +**Which volumes move together.** A migration moves the whole NVMe subsystem, so what it +affects is the volume's *group*, not the volume. The control plane decides the packing; the +driver re-reads it before every migration, because a previous migration may have changed it. + +**What the migration actually did.** `status.sourceNodeUUID` is the operator's own resolved +answer and beats the driver's guess, so it is written back over it. + +What it deliberately does not do is judge anything. Whether a migration was acceptable is the +detectors' business; this records what happened and moves on. That separation is why the same +run can be re-judged later against a check that did not exist when it ran. +""" + +from __future__ import annotations + +import contextlib +import json +import random +import threading +import time +from datetime import datetime +from typing import Any + +from ..core import Component, Migration, RunContext, component, iso, now_utc +from . import kube +from .sbctl import Sbctl, SbctlError + +#: Phases the operator will not move on from. +TERMINAL = ("Completed", "Failed", "Aborted") + + +@component +class MigrationDriver(Component): + """Drive VolumeMigration CRs in a loop, one at a time, recording each outcome. + + One at a time on purpose. Concurrent migrations of the same subsystem are rejected by the + control plane, and concurrent migrations of *different* subsystems make every host-side + observation ambiguous about which one caused it. A test that cannot attribute what it saw + produces anecdotes. + """ + + name = "migration.driver" + summary = "create VolumeMigration CRs in a loop and record what each one did" + required = True + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + "api_group": "storage.simplyblock.io/v1alpha1", + # alternate | consumer | no-consumer | random. The policy is about whether the + # *target* also hosts a pod consuming this subsystem — the materially harder case, + # so which one a migration exercised is chosen rather than left to chance. + "target_policy": "alternate", + "gap_s": 30.0, # settle time between migrations + "timeout_s": 600.0, # per migration, before it is called a TIMEOUT + "poll_s": 3.0, + "max_migrations": 0, # 0 = for as long as the run lasts + # Volumes to migrate. Normally whatever the workload published; set explicitly to + # drive migrations against volumes this framework did not create. + "pvs": None, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._thread: threading.Thread | None = None + self._stop = threading.Event() + self._records: list[Migration] = [] + self._sb: Sbctl | None = None + self._nodes: list[str] = [] + self._node_host: dict[str, str] = {} + self._placement: dict[str, str] = {} # pv -> storage node uuid + self._volume_of: dict[str, str] = {} # pv -> lvol uuid + self._nqn_of: dict[str, str] = {} # pv -> subsystem nqn + self._groups: dict[str, list[str]] = {} + self._pvs: list[str] = [] + self._idx = 0 + + # ── setup ────────────────────────────────────────────────────────────────────── + + def setup(self, ctx: RunContext) -> None: + self._nodes, self._node_host = self._storage_nodes() + if len(self._nodes) < 2: + raise RuntimeError("need at least two online storage nodes to migrate between, " + f"found {len(self._nodes)}") + + self._pvs = list(self.opt("pvs") or ctx.shared.get("workload.pvs") or []) + if not self._pvs: + raise RuntimeError( + "no volumes to migrate: enable a workload component, or set this component's " + "`pvs` option to migrate volumes the run did not create") + + # Reuse the workload's sbctl handle when there is one, so the webappapi pod is + # resolved once per run rather than once per component. + self._sb = ctx.shared.get("workload.sbctl") or Sbctl() + self._volume_of = dict(ctx.shared.get("workload.volume_of", {})) + self._nqn_of = dict(ctx.shared.get("workload.nqn_of", {})) + self._groups = self._regroup() + self._placement = dict(ctx.shared.get("workload.placement", {})) + ctx.shared["migrations"] = self._records + ctx.log.info(f"{self.name}: {len(self._pvs)} volume(s) across {len(self._nodes)} " + f"storage node(s), policy={self.opt('target_policy')}") + + def _storage_nodes(self) -> tuple[list[str], dict[str, str]]: + """Online storage nodes as (uuids, uuid -> k8s hostname). + + Only online ones: migrating *to* an offline node is a rejected request, not a test. + """ + cp = kube.run(["get", "storagenodes", "-A", "-o", "json"], check=False) + try: + items = json.loads(cp.stdout).get("items", []) if cp.stdout else [] + except json.JSONDecodeError: + items = [] + uuids, hosts = [], {} + for it in items: + st = it.get("status", {}) + uuid = st.get("uuid") or "" + if not uuid or st.get("status") != "online": + continue + uuids.append(uuid) + hosts[uuid] = it.get("spec", {}).get("workerNode", "") or st.get("workerNode", "") + return uuids, hosts + + # ── the loop ─────────────────────────────────────────────────────────────────── + + def start(self, ctx: RunContext) -> None: + def loop() -> None: + gap = float(self.opt("gap_s")) + cap = int(self.opt("max_migrations")) + while not self._stop.is_set() and not ctx.stopping.is_set(): + if cap and self._idx >= cap: + ctx.log.info(f"{self.name}: reached max_migrations={cap}") + return + self._idx += 1 + try: + self._one(ctx, self._idx) + except Exception as e: # noqa: BLE001 + # A migration that blows up is a data point, not the end of the run: the + # next one may behave differently, and the detectors want the whole set. + ctx.log.error(f"{self.name}: migration {self._idx} errored: {e}") + if self._stop.wait(gap): + return + + self._thread = threading.Thread(target=loop, name="migration-driver", daemon=True) + self._thread.start() + + def _one(self, ctx: RunContext, idx: int) -> None: + pv = self._pvs[(idx - 1) % len(self._pvs)] + self._reread_subsystem(ctx, pv) + group = self._group_of(pv) + self._refresh_placement(group) + source = self._placement.get(pv, "") + target, policy, consumers = self._pick_target(ctx, group, idx, source) + + name = f"{ctx.run_id}-mig-{idx}" + rec = Migration(name=name, start=now_utc(), source=source, target=target, pv=pv, + pod=ctx.shared.get("workload.pod_of", {}).get(pv, ""), + members=list(group)) + self._records.append(rec) + + sampler = ctx.shared.get("ana.sampler") + if sampler is not None: + sampler.begin(name) + + ctx.timeline.record("migration.start", subject=name, pv=pv, source=source, + target=target, policy=policy, members=len(group), + nqn=self._nqn_of.get(pv, "")) + ctx.log.event( + f"MIGRATION START {name} pv={pv} members={len(group)} " + f"source={source[:8] or '?'}({self._node_host.get(source, '?')}) " + f"target={target[:8]}({self._node_host.get(target, '?')}) policy={policy} " + + (f"target hosts consumer(s): {', '.join(consumers)}" if consumers + else "target hosts no consumer")) + # A group whose members are not all on the source is inconsistent *before* the + # migration starts, and every later placement check would be judging that instead. + off = [p for p in group if self._placement.get(p, source) != source] + if off: + ctx.log.warn(f"{self.name}: {name}: subsystem members are not all on the source " + f"node: {', '.join(sorted(off))}") + + try: + kube.run(["apply", "-f", "-"], stdin=self._manifest(name, pv, target)) + except kube.KubectlError as e: + rec.phase, rec.error, rec.end = "Failed", f"create: {e}", now_utc() + ctx.log.error(f"{self.name}: cannot create {name}: {e}") + if sampler is not None: + sampler.end(ctx, name) + return + + self._await_terminal(ctx, rec, name, sampler) + + if sampler is not None: + sampler.end(ctx, name) + # Where the group ended up — for the next target pick, and for the placement checks. + self._refresh_placement(group) + ctx.timeline.record("migration.stop", subject=name, phase=rec.phase, error=rec.error, + landed=self._placement.get(pv, "")) + secs = f"{(rec.end - rec.start).total_seconds():.0f}s" if rec.end else "?" + ctx.log.event(f"MIGRATION STOP {name} phase={rec.phase} duration={secs}" + + (f" error={rec.error!r}" if rec.error else "")) + + def _await_terminal(self, ctx: RunContext, rec: Migration, name: str, + sampler: Any) -> None: + deadline = time.time() + float(self.opt("timeout_s")) + poll = float(self.opt("poll_s")) + seen = "" + while time.time() < deadline and not self._stop.is_set(): + cp = kube.run(["-n", self.opt("namespace"), "get", "volumemigration", name, + "-o", "json"], check=False, timeout=30) + if cp.returncode == 0 and cp.stdout: + try: + st = json.loads(cp.stdout).get("status", {}) + except json.JSONDecodeError: + st = {} + # The operator resolved the source from the control plane; ours came from a + # listing that may already be stale, so its answer wins. + if st.get("sourceNodeUUID"): + rec.source = str(st["sourceNodeUUID"]) + self._placement[rec.pv] = rec.source + phase = str(st.get("phase", "")) + if phase and phase != seen: + seen = phase + # The sampler stamps this on every sample it takes, which is what makes an + # ANA transition readable as "during cutover" rather than "at 09:31:02". + if sampler is not None and hasattr(sampler, "set_phase"): + sampler.set_phase(phase) + ctx.timeline.record("migration.phase", subject=name, phase=phase) + ctx.log.info(f"{self.name}: {name} -> {phase}") + if phase in TERMINAL: + rec.phase = phase + rec.error = str(st.get("errorMessage", "") or "") + rec.end = now_utc() + return + self._stop.wait(poll) + # Distinct from Failed on purpose: a migration the control plane rejected and one that + # never finished are different defects, and only one of them has an error to read. + rec.phase = "TIMEOUT" + rec.end = now_utc() + + # ── target selection ─────────────────────────────────────────────────────────── + + def _pick_target(self, ctx: RunContext, group: list[str], idx: int, + source: str) -> tuple[str, str, list[str]]: + """Pick the node to migrate to, honouring the policy. + + `alternate` starts with `consumer` on the first migration, so if a run is cut short the + harder case is the one that got exercised. + + When the policy cannot be honoured — no candidate matches — it falls back to any other + node and records that it did, rather than skipping the migration. A migration that ran + under the other condition is still evidence; one that did not run is not. + """ + candidates = [n for n in self._nodes if n != source] or list(self._nodes) + policy = str(self.opt("target_policy")) + if policy == "alternate": + policy = "consumer" if idx % 2 == 1 else "no-consumer" + + consuming = self._consumer_nodes(ctx, group) # k8s host -> consuming pods + want = None if policy == "random" else (policy == "consumer") + + pool = candidates + if want is not None: + matching = [n for n in candidates + if (self._node_host.get(n, "") in consuming) == want] + if matching: + pool = matching + else: + ctx.log.warn( + f"{self.name}: target policy {policy!r} cannot be honoured: no candidate " + f"node {'hosts' if want else 'is free of'} a consumer of this subsystem " + f"(consumers on {', '.join(sorted(consuming)) or 'none'}); picking any " + "other node") + policy = f"{policy}(unmet)" + target = random.choice(pool) # noqa: S311 (test placement, not cryptography) + return target, policy, sorted(consuming.get(self._node_host.get(target, ""), [])) + + def _consumer_nodes(self, ctx: RunContext, pvs: list[str]) -> dict[str, list[str]]: + """k8s node -> pods on it consuming any volume in this subsystem. + + The whole group, not just the volume being named: every pod holding *any* namespace of + the subsystem has its paths moved, so each one is a consumer for this purpose. + """ + pod_of = ctx.shared.get("workload.pod_of", {}) + node_of = ctx.shared.get("workload.node_of", {}) + out: dict[str, list[str]] = {} + for pv in pvs: + pod = pod_of.get(pv) + node = node_of.get(pod) if pod else None + if pod and node: + out.setdefault(node, []).append(pod) + return out + + # ── backend state ────────────────────────────────────────────────────────────── + + def _refresh_placement(self, pvs: list[str]) -> None: + """Re-read where these volumes are, from one `sbctl volume list`. + + One listing for the whole group: its members are compared against each other, and + per-volume lookups taken seconds apart could straddle a move and make a consistent + subsystem look split. + """ + if not self._sb: + return + lvols = {pv: self._volume_of[pv] for pv in pvs if pv in self._volume_of} + if not lvols: + return + # A failure here leaves the last known placement in place: a stale source beats no + # source, because "unknown" would make every target pick arbitrary. + with contextlib.suppress(SbctlError): + self._placement.update(self._sb.nodes_of(lvols)) + + def _reread_subsystem(self, ctx: RunContext, pv: str) -> None: + """Re-read this volume's subsystem before migrating it. + + Not cached from setup, because a previous migration can have changed the packing. The + group is what a migration moves, so a stale group means sampling the wrong nodes and + verifying the wrong volumes. + """ + if not self._sb: + return + lvol = self._volume_of.get(pv, "") + if not lvol: + return + nqn, _ = self._sb.subsystem_of(lvol) + if nqn and nqn != self._nqn_of.get(pv): + ctx.log.info(f"{self.name}: {pv} changed subsystem: " + f"{self._nqn_of.get(pv, '?')} -> {nqn}; regrouping") + self._nqn_of[pv] = nqn + self._groups = self._regroup() + + def _regroup(self) -> dict[str, list[str]]: + groups: dict[str, list[str]] = {} + for pv, nqn in self._nqn_of.items(): + groups.setdefault(nqn, []).append(pv) + return {n: sorted(m) for n, m in groups.items()} + + def _group_of(self, pv: str) -> list[str]: + """Everything a migration of `pv` moves: the volumes sharing its subsystem.""" + nqn = self._nqn_of.get(pv, "") + return list(self._groups.get(nqn, [pv])) if nqn else [pv] + + def _manifest(self, name: str, pv: str, target: str) -> str: + return json.dumps({ + "apiVersion": self.opt("api_group"), + "kind": "VolumeMigration", + "metadata": {"name": name, "namespace": self.opt("namespace"), + "labels": {"sbtest-run": "true", "sbtest": name}}, + "spec": {"pvName": pv, "targetNodeUUID": target}, + }) + + # ── lifecycle tail ───────────────────────────────────────────────────────────── + + def stop(self, ctx: RunContext) -> None: + self._stop.set() + if self._thread: + # Long enough for a migration already in flight to reach its own timeout: cutting + # it short would record a TIMEOUT the product never caused. + self._thread.join(timeout=float(self.opt("timeout_s")) + 30) + self._thread = None + done = [r for r in self._records if r.end] + ctx.log.info(f"{self.name}: {len(done)}/{len(self._records)} migration(s) reached a " + "terminal state") + + def collect(self, ctx: RunContext) -> None: + """Write the migration timeline where the analyser can read it back.""" + ctx.save_json("migrations.json", [{ + "name": r.name, + "start": iso(r.start), + "end": iso(r.end) if r.end else None, + "phase": r.phase, + "source": r.source, + "target": r.target, + "pv": r.pv, + "pod": r.pod, + "group_pvs": r.members, + "nqn": self._nqn_of.get(r.pv, ""), + "landed": self._placement.get(r.pv, ""), + "error": r.error, + } for r in self._records]) + by_phase: dict[str, int] = {} + for r in self._records: + by_phase[r.phase or "?"] = by_phase.get(r.phase or "?", 0) + 1 + ctx.log.info(f"{self.name}: " + + ", ".join(f"{k}={v}" for k, v in sorted(by_phase.items(), + key=lambda kv: -kv[1]))) + + def teardown(self, ctx: RunContext) -> None: + """Remove the CRs this run created, unless asked to keep them. + + By label rather than by name, so a CR whose creation succeeded but whose name never + made it into the records still goes. + """ + if ctx.shared.get("keep"): + ctx.log.info(f"{self.name}: keep set; leaving {len(self._records)} CR(s) in place") + return + kube.run(["-n", self.opt("namespace"), "delete", "volumemigration", + "-l", "sbtest-run=true", "--ignore-not-found", "--wait=false"], + check=False) + + +def migrations_from_file(path: str) -> list[Migration]: + """Read back what the driver wrote. Used by the archive adapter.""" + with open(path) as fh: + raw = json.load(fh) + + def _dt(v: object) -> datetime | None: + if not v: + return None + try: + return datetime.fromisoformat(str(v).replace("Z", "+00:00")) + except ValueError: + return None + + out = [] + for m in raw: + start = _dt(m.get("start")) + if not start: + continue + out.append(Migration( + name=m.get("name", ""), start=start, end=_dt(m.get("end")), + phase=m.get("phase", ""), source=m.get("source", ""), + target=m.get("target", ""), pv=m.get("pv", ""), pod=m.get("pod", ""), + members=[x for x in (m.get("group_pvs") or []) if x], + error=m.get("error", "") or "")) + out.sort(key=lambda x: x.start) + return out diff --git a/test/framework/sbtest/components/nvme.py b/test/framework/sbtest/components/nvme.py new file mode 100644 index 000000000..b41cc1ce9 --- /dev/null +++ b/test/framework/sbtest/components/nvme.py @@ -0,0 +1,247 @@ +"""Host NVMe observation: periodic ANA sampling and an end-of-run fabric snapshot. + +Both read the host's sysfs through a pod that already has it — the CSI node plugin — rather +than starting anything, because they need to run on every consuming node and the plugin is +already there on all of them. +""" + +from __future__ import annotations + +import threading +from typing import Any + +from ..core import AnaSample, Component, NvmeController, RunContext, component, now_utc +from . import kube + +#: Read every lvol controller's state, its address and its per-namespace ANA states. +#: One line per controller: name|state|address|ctrl_loss_tmo|nsid=ana,nsid=ana,... +_SNAPSHOT_SH = r''' +for c in /sys/class/nvme/nvme*; do + [ -f "$c/subsysnqn" ] || continue + nqn=$(cat "$c/subsysnqn" 2>/dev/null) + case "$nqn" in *lvol:*) ;; *) continue;; esac + ns="" + for n in "$c"/nvme*n*; do + [ -d "$n" ] || continue + id=$(cat "$n/nsid" 2>/dev/null) || continue + a=$(cat "$n/ana_state" 2>/dev/null) + ns="$ns$id=$a," + done + printf '%s|%s|%s|%s|%s|%s\n' \ + "$(basename "$c")" "$(cat "$c/state" 2>/dev/null)" \ + "$(cat "$c/address" 2>/dev/null | tr -d '\n')" \ + "$(cat "$c/ctrl_loss_tmo" 2>/dev/null)" "$nqn" "$ns" +done +''' + + +def _parse_snapshot(node: str, out: str) -> list[NvmeController]: + ctrls = [] + for line in out.splitlines(): + parts = line.split("|") + if len(parts) < 6: + continue + name, state, address, clt, nqn, ns = parts[:6] + traddr = trsvcid = "" + for kv in address.split(","): + if kv.startswith("traddr="): + traddr = kv[7:] + elif kv.startswith("trsvcid="): + trsvcid = kv[8:] + namespaces = {} + for item in ns.split(","): + if "=" in item: + k, _, v = item.partition("=") + try: + namespaces[int(k)] = v + except ValueError: + continue + try: + loss = int(clt) if clt.strip() else None + except ValueError: + loss = None + ctrls.append(NvmeController( + node=node, name=name, nqn=nqn, address=f"{traddr}:{trsvcid}", + state=state, namespaces=namespaces, ctrl_loss_tmo=loss)) + return ctrls + + +class _CsiNodeBase(Component): + """Shared lookup of the CSI node plugin, the window onto each host's sysfs. + + A Component for the same reason as _GrabberBase: it uses `opt` and `name`. + """ + + def _csi_node_pods(self, ctx: RunContext) -> dict[str, str]: + """node -> CSI node-plugin pod, the window onto each host's sysfs.""" + out = {} + for p in kube.list_pods(self.opt("csi_namespace"), [self.opt("csi_pod_prefix")]): + if p.node: + out[p.node] = p.name + if not out: + ctx.log.warn(f"{self.name}: no CSI node pods matching " + f"{self.opt('csi_pod_prefix')!r} in {self.opt('csi_namespace')}; " + "host NVMe state cannot be read") + return out + + +@component +class FabricSnapshot(_CsiNodeBase): + """Record every lvol NVMe controller on every node, at setup and at the end. + + Two snapshots on purpose. The one at setup answers "did the last run leave a mess?" — + which is a real question, because a leaked controller breaks the *next* run rather than + the one that created it. The one at the end answers "did this run leave a mess?". + """ + + name = "nvme.snapshot" + summary = "snapshot host NVMe controllers before and after the run (leak detection)" + + def defaults(self) -> dict[str, Any]: + return {"csi_namespace": "simplyblock", "csi_pod_prefix": "simplyblock-csi-node", + "container": "csi-node", "at_setup": True, "at_collect": True} + + def _snapshot(self, ctx: RunContext, label: str) -> list[NvmeController]: + ctrls: list[NvmeController] = [] + for node, pod in sorted(self._csi_node_pods(ctx).items()): + out = kube.exec_sh(self.opt("csi_namespace"), pod, _SNAPSHOT_SH, + container=self.opt("container"), timeout=120) + ctrls.extend(_parse_snapshot(kube.short(node), out)) + ctx.save_json(f"nvme-controllers-{label}.json", + [{"node": c.node, "name": c.name, "nqn": c.nqn, "address": c.address, + "state": c.state, "namespaces": {str(k): v for k, v in c.namespaces.items()}, + "ctrl_loss_tmo": c.ctrl_loss_tmo} for c in ctrls]) + stuck = [c for c in ctrls if c.state == "connecting"] + empty = [c for c in ctrls if c.state == "live" and c.serves_nothing] + ctx.log.info(f"{self.name} ({label}): {len(ctrls)} lvol controller(s), " + f"{len(stuck)} connecting, {len(empty)} live-with-no-namespace") + return ctrls + + def setup(self, ctx: RunContext) -> None: + if not self.opt("at_setup"): + return + pre = self._snapshot(ctx, "pre") + ctx.shared["nvme.controllers.pre"] = pre + + def collect(self, ctx: RunContext) -> None: + if not self.opt("at_collect"): + return + post = self._snapshot(ctx, "post") + ctx.shared["nvme.controllers"] = post + # The detectors read the canonical name; keep it pointing at the post-run state. + ctx.save_json("nvme-controllers.json", + [{"node": c.node, "name": c.name, "nqn": c.nqn, "address": c.address, + "state": c.state, "namespaces": {str(k): v for k, v in c.namespaces.items()}, + "ctrl_loss_tmo": c.ctrl_loss_tmo} for c in post]) + + +@component +class AnaSampler(_CsiNodeBase): + """Sample per-namespace ANA state on every consuming node, on an interval. + + The evidence behind every ANA detector, and the reason the sampling interval is a + first-class option: the freeze *count* is robust to it, but a freeze shorter than one + interval can be missed entirely, so a suite hunting short pauses should lower it. + + Samples are written per migration into ana/.csv in the same layout the + archive reader expects, so a run's ANA evidence is replayable. + """ + + name = "ana.sample" + summary = "sample host ANA state per namespace on an interval, per migration" + + def defaults(self) -> dict[str, Any]: + return {"csi_namespace": "simplyblock", "csi_pod_prefix": "simplyblock-csi-node", + "container": "csi-node", "interval_s": 2.0, "nodes": None} + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._thread: threading.Thread | None = None + self._stop = threading.Event() + self._current: str | None = None + self._phase = "" + self._samples: dict[str, list[AnaSample]] = {} + self._pods: dict[str, str] = {} + + def setup(self, ctx: RunContext) -> None: + self._pods = self._csi_node_pods(ctx) + ctx.shared["ana.samples"] = self._samples + # Published so a driver can attribute samples to the migration in flight. Samples + # that belong to no migration are worth nothing — freeze_windows is per migration — + # so without a driver calling begin/end this component collects nothing, and the ANA + # detectors correctly report themselves skipped rather than clean. + ctx.shared["ana.sampler"] = self + + # -- the scenario drives these two ---------------------------------------------- + def begin(self, migration: str) -> None: + """Start attributing samples to `migration`. Called by whatever drives migrations.""" + self._current = migration + self._phase = "" + self._samples.setdefault(migration, []) + + def set_phase(self, phase: str) -> None: + """Stamp subsequent samples with the migration's current phase. + + This is what turns a wall-clock ANA transition into a readable one: "all paths went + inaccessible during Cutover" is a finding, "all paths went inaccessible at 09:31:02" + is a timestamp. Optional — the driver calls it if it has a phase to report, and a + sampler that is never told stamps the empty string. + """ + self._phase = phase + + def end(self, ctx: RunContext, migration: str) -> str | None: + """Stop attributing and write the CSV for `migration`.""" + self._current = None + self._phase = "" + samples = self._samples.get(migration) or [] + if not samples: + return None + path = ctx.path(f"ana/{migration}.csv") + with open(path, "w") as fh: + fh.write("ts,node,phase,address,role,ctrl_state,nsid,ana_state\n") + for s in sorted(samples, key=lambda x: (x.ts, x.node, x.address)): + stamp = s.ts.strftime("%Y-%m-%dT%H:%M:%SZ") + if not s.ana: + # A controller with no namespace at all is itself the finding, so it + # gets a row rather than being skipped. + fh.write(f"{stamp},{s.node},{s.phase},{s.address},{s.role},{s.state},,\n") + for nsid, ana in sorted(s.ana.items()): + fh.write(f"{stamp},{s.node},{s.phase},{s.address},{s.role}," + f"{s.state},{nsid},{ana}\n") + return path + + def start(self, ctx: RunContext) -> None: + if not self._pods: + return + interval = float(self.opt("interval_s")) + if interval <= 0: + ctx.log.info(f"{self.name}: disabled (interval_s <= 0)") + return + + def loop() -> None: + while not self._stop.is_set() and not ctx.stopping.is_set(): + mig = self._current + if mig: + for node, pod in self._pods.items(): + try: + out = kube.exec_sh(self.opt("csi_namespace"), pod, _SNAPSHOT_SH, + container=self.opt("container"), timeout=30) + except Exception: # noqa: BLE001 + continue + ts = now_utc() + for c in _parse_snapshot(kube.short(node), out): + self._samples.setdefault(mig, []).append(AnaSample( + ts=ts, node=c.node, address=c.address, state=c.state, + ana=dict(c.namespaces), phase=self._phase)) + self._stop.wait(interval) + + self._thread = threading.Thread(target=loop, name="ana-sampler", daemon=True) + self._thread.start() + ctx.log.info(f"{self.name}: sampling every {interval}s on " + f"{len(self._pods)} node(s)") + + def stop(self, ctx: RunContext) -> None: + self._stop.set() + if self._thread: + self._thread.join(timeout=15) + self._thread = None diff --git a/test/framework/sbtest/components/sbctl.py b/test/framework/sbtest/components/sbctl.py new file mode 100644 index 000000000..835ac6d88 --- /dev/null +++ b/test/framework/sbtest/components/sbctl.py @@ -0,0 +1,194 @@ +"""A thin `sbctl` client — the control plane's own view of the cluster. + +Kubernetes objects say what was *asked for*; sbctl says what the storage cluster actually +did. The two disagree often enough that mixing them up is its own class of bug: a +StorageClass can carry a `cluster_id` from an installation that no longer exists, and a +volume's placement drifts under the cluster's own rebalancer without any Kubernetes object +changing. So placement, subsystem grouping and the live cluster id all come from here. + +Not a Component: it holds no lifecycle and produces no artifacts. Components construct one +and ask it questions. +""" + +from __future__ import annotations + +import json +from typing import Any + +from . import kube + + +class SbctlError(RuntimeError): + pass + + +class Sbctl: + """Runs `sbctl ... --json` inside a webappapi pod. + + The pod is resolved once and cached. If it is replaced mid-run the next call re-resolves + rather than failing the run: a restarted webappapi is normal, and losing placement + resolution would turn every later verification into "skipped" for no good reason. + """ + + def __init__(self, namespace: str = "simplyblock", pod_match: str = "webappapi") -> None: + self.namespace = namespace + self.pod_match = pod_match + self._pod = "" + self._cluster_uuid = "" + + # ── plumbing ─────────────────────────────────────────────────────────────────── + + def pod(self, refresh: bool = False) -> str: + if self._pod and not refresh: + return self._pod + for p in kube.list_pods(self.namespace, [self.pod_match]): + if p.phase == "Running": + self._pod = p.name + return p.name + raise SbctlError(f"no running '*{self.pod_match}*' pod in namespace {self.namespace}") + + def _json(self, *argv: str, timeout: int = 60) -> Any: + for attempt in (0, 1): + pod = self.pod(refresh=bool(attempt)) + cp = kube.run(["-n", self.namespace, "exec", pod, "--", "sbctl", *argv, "--json"], + check=False, timeout=timeout) + if cp.returncode == 0: + break + if attempt: + raise SbctlError(f"sbctl {' '.join(argv)} failed: " + f"{(cp.stderr or cp.stdout).strip()}") + try: + return json.loads(cp.stdout) + except json.JSONDecodeError as e: + raise SbctlError(f"sbctl {' '.join(argv)} returned unparseable output: {e}") from e + + # ── queries ──────────────────────────────────────────────────────────────────── + + def cluster_uuid(self) -> str: + """The live cluster's UUID. + + Authoritative after a reinstall, which is the whole reason it is read at all: a + StorageClass cloned from the pool's own SC can still carry a dead cluster id, and + every volume provisioned from it would target a cluster that no longer exists. + """ + if self._cluster_uuid: + return self._cluster_uuid + clusters = self._json("cluster", "list") + if not clusters: + raise SbctlError("sbctl cluster list returned no clusters") + active = [c for c in clusters if str(c.get("Status", "")).upper() == "ACTIVE"] + chosen = active or clusters + if len(chosen) != 1: + desc = ", ".join(f"{c.get('Name')}={c.get('UUID')}({c.get('Status')})" + for c in chosen) + raise SbctlError(f"expected exactly one active cluster, found " + f"{len(chosen)}: {desc}") + uuid = str(chosen[0].get("UUID") or "") + if not uuid: + raise SbctlError("the active cluster has no UUID in sbctl output") + self._cluster_uuid = uuid + return uuid + + def volume_list(self) -> list[dict]: + out = self._json("volume", "list") + return out if isinstance(out, list) else [] + + def volume_get(self, lvol: str) -> dict: + out = self._json("volume", "get", lvol) + return out if isinstance(out, dict) else {} + + def storage_node_list(self) -> list[dict]: + out = self._json("storage-node", "list") + return out if isinstance(out, list) else [] + + def snapshot_list(self) -> list[dict]: + out = self._json("snapshot", "list") + return out if isinstance(out, list) else [] + + # ── derived views ────────────────────────────────────────────────────────────── + + def host_map(self) -> tuple[dict[str, str], dict[str, str]]: + """(sbctl hostname -> node uuid, node uuid -> management IP). + + The translation exists because a volume's `Hostname` is the storage node's own name + (short host + RPC port, e.g. `vm04_4424`) while everything on the Kubernetes side + speaks node UUIDs. The management IP is the transport address the node's subsystems + listen on, which is the only way a sampled path can be attributed to a role. + """ + hosts: dict[str, str] = {} + ips: dict[str, str] = {} + for n in self.storage_node_list(): + host, uuid = str(n.get("Hostname") or ""), str(n.get("UUID") or "") + if host and uuid: + hosts[host] = uuid + if uuid and n.get("Management IP"): + ips[uuid] = str(n["Management IP"]) + if not hosts: + raise SbctlError("sbctl storage-node list reported no Hostname/UUID pairs") + return hosts, ips + + def subsystem_of(self, lvol: str) -> tuple[str, int]: + """(NQN, ns_id) of one volume. ('', 0) when unresolvable. + + Read per volume rather than derived from the StorageClass, because the control plane + decides how it packs namespaced volumes into subsystems. What a migration has to move + follows from the real grouping, not the requested one. + """ + if not lvol: + return "", 0 + try: + data = self.volume_get(lvol) + except SbctlError: + return "", 0 + nqn = str(data.get("nqn") or data.get("NQN") or "") + try: + ns_id = int(data.get("ns_id") or data.get("NS ID") or 0) + except (TypeError, ValueError): + ns_id = 0 + return nqn, ns_id + + def nodes_of(self, lvols: dict[str, str]) -> dict[str, str]: + """{key: node uuid} for `{key: lvol uuid}`, from a *single* volume listing. + + One listing for the whole set on purpose: the members of a shared subsystem are + compared against each other, and per-volume listings taken seconds apart could + straddle a move and make a consistent subsystem look split. + """ + try: + vols = self.volume_list() + hosts, _ = self.host_map() + except SbctlError: + return {} + by_lvol: dict[str, dict] = {} + for v in vols: + for k in (v.get("Id"), v.get("LVolUUID")): + if k: + by_lvol[str(k)] = v + + out: dict[str, str] = {} + missed: set[str] = set() + for key, lvol in lvols.items(): + vol = by_lvol.get(lvol) + if not vol: + continue + node = hosts.get(str(vol.get("Hostname") or "")) + if node: + out[key] = node + else: + missed.add(str(vol.get("Hostname") or "")) + # An unknown hostname is not proof a volume is unplaceable — a node may have joined + # since the map was built. Rebuild once before giving up, so a stale map cannot + # quietly turn every placement check into "skipped". + if missed: + try: + hosts, _ = self.host_map() + except SbctlError: + return out + for key, lvol in lvols.items(): + if key in out: + continue + vol = by_lvol.get(lvol) + node = hosts.get(str(vol.get("Hostname") or "")) if vol else "" + if node: + out[key] = node + return out diff --git a/test/framework/sbtest/components/workload.py b/test/framework/sbtest/components/workload.py new file mode 100644 index 000000000..8131fc764 --- /dev/null +++ b/test/framework/sbtest/components/workload.py @@ -0,0 +1,534 @@ +"""The fio workload: volumes with continuous, verified I/O on them. + +A migration test needs I/O in flight to be a test at all — a volume nobody is writing to +migrates cleanly whatever the code does, because nothing is there to lose. So this component +provisions volumes, drives fio against them for the length of the run, and collects what fio +saw. + +Two things here are less obvious than they look. + +**Verification is only enabled when it can be trusted.** fio's md5 verify races itself when +two in-flight I/Os touch the same block, and reports corruption that never happened. With one +job that is solvable (`--serialize_overlap`); across processes it is not, without +`io_submit_mode=offload`. So `numjobs > 1` turns verification *off* and says so loudly, +because a run that measures throughput is not a run that would have noticed data loss — and +the report must not imply otherwise. + +**Two StorageClasses, not one.** Batch migration only exists when several volumes share an +NVMe subsystem, and whether they do is the control plane's decision, not the test's. So the +namespaced volumes get their own class and the real grouping is *read back* from the backend +afterwards. If the volumes each ended up in their own subsystem, the batch half of the run is +vacuous — which is worth knowing at minute one rather than during the analysis. +""" + +from __future__ import annotations + +import json +import time +from typing import Any + +from ..core import Component, RunContext, component +from . import kube +from .sbctl import Sbctl, SbctlError + +FIO_IMAGE = "alpine:3.20" +PARAM_MAX_NS = "max_namespace_per_subsys" + + +@component +class FioWorkload(Component): + """Provision volumes, run verified fio on them, collect what fio saw. + + `required`, because a run whose workload never came up is not a passing run — it is no run + at all, and the detectors have nothing to judge. + """ + + name = "workload.fio" + summary = "provision volumes and drive continuous verified fio I/O against them" + required = True + + def defaults(self) -> dict[str, Any]: + return { + "namespace": "default", + # The operator's own pool SC, named simplyblock---. Cloned + # rather than hand-written so the run inherits whatever the pool really uses. + "source_storageclass": "simplyblock-default-simplyblock-cluster-pool1", + "pods": 0, # single-namespace volumes + "ns_pods": 6, # namespaced volumes (share subsystems -> batch migration) + "ns_per_subsys": 6, + "volume_size_gb": 10, + "file_size_gb": 1, + "fstype": "xfs", + "runtime_s": 3600, + "iodepth": 8, + "numjobs": 1, # >1 disables verification; see the module docstring + "verify_fatal": False, + "bs": "4k", + "rwmixread": 70, + "ioengine": "libaio", + "ready_timeout_s": 420, + "image": FIO_IMAGE, + } + + def __init__(self, **options: Any) -> None: + super().__init__(**options) + self._sb = Sbctl() + self._pods: list[str] = [] + self._pvcs: list[str] = [] + self._pvc_of: dict[str, str] = {} + self._kind_of: dict[str, str] = {} + self._pv_of: dict[str, str] = {} # pod -> pv + self._pod_of: dict[str, str] = {} # pv -> pod + self._volume_of: dict[str, str] = {} # pv -> lvol uuid + self._nqn_of: dict[str, str] = {} # pv -> subsystem nqn + self._nsid_of: dict[str, int] = {} + self._groups: dict[str, list[str]] = {} # nqn -> pvs, ns_id ordered + self._node_host: dict[str, str] = {} # storage node uuid -> k8s host + self._sc_single = "" + self._sc_ns = "" + + # ── setup: classes, volumes, pods, then read back what really happened ───────── + + def setup(self, ctx: RunContext) -> None: + self._node_host = self._storage_node_hosts() + self._ensure_storageclasses(ctx) + self._create(ctx) + self._wait_running(ctx) + self._resolve_pvs(ctx) + self._resolve_subsystems(ctx) + self._publish(ctx) + + def _storage_node_hosts(self) -> dict[str, str]: + cp = kube.run(["get", "storagenodes", "-A", "-o", "json"], check=False) + try: + items = json.loads(cp.stdout).get("items", []) if cp.stdout else [] + except json.JSONDecodeError: + return {} + out = {} + for it in items: + st = it.get("status", {}) + uuid = st.get("uuid") or "" + if uuid: + out[uuid] = it.get("spec", {}).get("workerNode", "") or st.get("workerNode", "") + return out + + def _ensure_storageclasses(self, ctx: RunContext) -> None: + """(Re)create the run's StorageClasses from the live pool SC. + + Always delete-then-create, so a class left behind by a previous run can never be + silently reused with different parameters. And `cluster_id` is forced to the live + cluster rather than copied: the source SC can still carry a dead one after a + reinstall, and every volume from it would target a cluster that no longer exists. + """ + src_name = self.opt("source_storageclass") + cp = kube.run(["get", "sc", src_name, "-o", "json"], check=False) + if cp.returncode != 0 or not cp.stdout: + # The name embeds the namespace, cluster and pool, so it differs per install. + # Naming the candidates turns a config error into a one-line fix instead of a + # trip to kubectl. + raise RuntimeError( + f"source StorageClass {src_name} not found. Either the name is wrong for this " + f"install — candidates: {', '.join(self._candidate_sources()) or 'none'} — or " + "the operator has not created it yet, which usually means the Pool is not " + "Active (its create is stuck retrying on the backend). Check: kubectl -n " + f"{self.opt('namespace')} get pool -o jsonpath='{{.items[*].status}}'") + src = json.loads(cp.stdout) + + cluster = self._sb.cluster_uuid() + base = dict(src.get("parameters", {})) + stale = base.get("cluster_id") + if stale and stale != cluster: + ctx.log.warn(f"{self.name}: source SC {src_name} carries stale cluster_id " + f"{stale}; overriding with the live cluster {cluster}") + base["cluster_id"] = cluster + base["csi.storage.k8s.io/fstype"] = str(self.opt("fstype")) + # A single-namespace class must say so explicitly rather than inherit whatever the + # source SC or the CSI default carries — otherwise the "single" half of the run could + # quietly be namespaced too, and the comparison would be with itself. + base.pop(PARAM_MAX_NS, None) + + self._sc_single = f"sbtest-{ctx.run_id}-single" + self._apply_sc(src, self._sc_single, dict(base, **{PARAM_MAX_NS: "1"})) + ctx.log.info(f"{self.name}: StorageClass {self._sc_single} " + f"(fstype={self.opt('fstype')}, {PARAM_MAX_NS}=1, cluster_id={cluster})") + if int(self.opt("ns_pods")) > 0: + self._sc_ns = f"sbtest-{ctx.run_id}-ns" + self._apply_sc(src, self._sc_ns, + dict(base, **{PARAM_MAX_NS: str(self.opt("ns_per_subsys"))})) + ctx.log.info(f"{self.name}: StorageClass {self._sc_ns} " + f"({PARAM_MAX_NS}={self.opt('ns_per_subsys')})") + + @staticmethod + def _candidate_sources() -> list[str]: + """simplyblock StorageClasses that are not themselves a test artifact. + + Anything carrying `sbtest-` is one of ours from an earlier run and would be a + confusing suggestion — cloning a clone inherits its overrides. + """ + cp = kube.run(["get", "sc", "-o", "json"], check=False) + if cp.returncode != 0 or not cp.stdout: + return [] + try: + items = json.loads(cp.stdout).get("items", []) + except json.JSONDecodeError: + return [] + return sorted( + it["metadata"]["name"] for it in items + if it.get("provisioner") == "csi.simplyblock.io" + and "sbtest-" not in it["metadata"]["name"]) + + @staticmethod + def _apply_sc(src: dict, name: str, params: dict) -> None: + kube.run(["delete", "sc", name, "--ignore-not-found"], check=False) + kube.run(["apply", "-f", "-"], stdin=json.dumps({ + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": name, "labels": {"sbtest-run": "true"}}, + "provisioner": src.get("provisioner", "csi.simplyblock.io"), + "parameters": params, + "reclaimPolicy": src.get("reclaimPolicy", "Delete"), + "volumeBindingMode": src.get("volumeBindingMode", "WaitForFirstConsumer"), + "allowVolumeExpansion": src.get("allowVolumeExpansion", True), + })) + + def _create(self, ctx: RunContext) -> None: + script = self._fio_script(ctx) + affinity = self._affinity(ctx) + docs: list[dict] = [] + # One continuous index across both kinds, so a single name filter matches every pod + # this component created. + plan = ([("single", self._sc_single)] * int(self.opt("pods")) + + [("namespaced", self._sc_ns)] * int(self.opt("ns_pods"))) + if not plan: + raise RuntimeError("workload.fio: both `pods` and `ns_pods` are 0, so the run " + "would have no I/O and could not detect anything") + for i, (kind, sc) in enumerate(plan): + pvc, pod = f"{ctx.run_id}-pvc-{i}", f"{ctx.run_id}-fio-{i}" + self._pvcs.append(pvc) + self._pods.append(pod) + self._pvc_of[pod] = pvc + self._kind_of[pod] = kind + labels = {"sbtest": ctx.run_id, "sbtest-run": "true", "volume-kind": kind} + docs.append({ + "apiVersion": "v1", "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc, "labels": labels}, + "spec": {"accessModes": ["ReadWriteOnce"], "storageClassName": sc, + "resources": {"requests": { + "storage": f"{self.opt('volume_size_gb')}Gi"}}}, + }) + docs.append({ + "apiVersion": "v1", "kind": "Pod", + "metadata": {"name": pod, "labels": dict(labels, app="fio")}, + "spec": { + "restartPolicy": "Never", + "terminationGracePeriodSeconds": 5, + "affinity": affinity, + "containers": [{ + "name": "fio", "image": str(self.opt("image")), + "imagePullPolicy": "IfNotPresent", + "command": ["sh", "-c", script], + "volumeMounts": [{"name": "data", "mountPath": "/data"}, + {"name": "logs", "mountPath": "/logs"}], + "resources": {"requests": {"cpu": "250m", "memory": "256Mi"}}, + }], + "volumes": [ + {"name": "data", "persistentVolumeClaim": {"claimName": pvc}}, + # fio's own logs live on an emptyDir, never on the volume under test: + # collecting the evidence must not depend on the health of the thing + # the evidence is about. + {"name": "logs", "emptyDir": {}}, + ], + }, + }) + kube.run(["-n", self.opt("namespace"), "apply", "-f", "-"], + stdin="\n---\n".join(json.dumps(d) for d in docs)) + ctx.log.info(f"{self.name}: created {len(plan)} PVC(s) + fio pod(s): " + f"{self.opt('pods')} single-namespace + {self.opt('ns_pods')} namespaced") + + def _affinity(self, ctx: RunContext) -> dict: + """Pin fio pods to the storage worker nodes and off the control plane. + + Off the control plane in particular: a consuming pod there would exercise a path the + product does not otherwise have, and a fio pod competing with the API server produces + latency findings about the test rather than the system. + """ + workers = sorted({h for h in self._node_host.values() if h}) + exprs: list[dict] = [{"key": "node-role.kubernetes.io/control-plane", + "operator": "DoesNotExist"}] + if workers: + exprs.insert(0, {"key": "kubernetes.io/hostname", "operator": "In", + "values": workers}) + ctx.log.info(f"{self.name}: pods restricted to {', '.join(workers)}") + else: + ctx.log.warn(f"{self.name}: no storage worker hostnames known; only excluding " + "control-plane nodes") + return {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [{"matchExpressions": exprs}]}}} + + def _fio_script(self, ctx: RunContext) -> str: + # fio always lays out a file-backed target before random I/O and cannot be told to + # skip it for a filesystem file. A large working set buys nothing here — the point is + # continuous I/O, not capacity — so keep the file small and the layout takes seconds. + file_gb = max(1, min(int(self.opt("file_size_gb")), + int(self.opt("volume_size_gb")) - 2)) + numjobs = int(self.opt("numjobs")) + args = [ + "fio", "--name=fiotest", "--filename=/data/fiotest", f"--size={file_gb}G", + f"--ioengine={self.opt('ioengine')}", "--direct=1", "--rw=randrw", + f"--rwmixread={self.opt('rwmixread')}", f"--bs={self.opt('bs')}", + f"--iodepth={self.opt('iodepth')}", f"--numjobs={numjobs}", + "--group_reporting", "--time_based", f"--runtime={self.opt('runtime_s')}", + "--continue_on_error=all", # record EIO, do not die on it + "--percentile_list=50:95:99:99.9", + "--write_iops_log=/logs/iops", "--write_lat_log=/logs/lat", + "--write_bw_log=/logs/bw", "--log_avg_msec=1000", + "--eta=always", "--eta-newline=30", + "--output=/logs/result.json", "--output-format=json", + ] + if numjobs == 1: + args += [ + # An md5 header per block, re-verified continuously during the run rather + # than only at the end — so a lost write surfaces while the volume is still + # migrating, close enough in time to attribute to a migration. + "--verify=md5", "--verify_backlog=4096", "--verify_backlog_batch=4096", + # Dump the mismatching block so its content can be read afterwards: a block + # holding its pre-write content is a lost write, which is a different defect + # from a block holding garbage. + "--verify_dump=1", + # Not fatal by default, so EVERY corrupted block gets reported instead of + # only the first. That turns the count into a measurement — with a known + # write rate it bounds how wide the window of lost writes was. Fatal stops at + # the first, making the count a lower bound of 1 that says nothing about size. + f"--verify_fatal={1 if self.opt('verify_fatal') else 0}", + ] + if int(self.opt("iodepth")) > 1: + args.append("--serialize_overlap=1") + else: + ctx.log.warn( + f"{self.name}: data-integrity verification DISABLED: numjobs={numjobs} (>1 " + "cannot serialize overlapping writes without io_submit_mode=offload, so " + "verify would report corruption that never happened). This run measures I/O " + "only, not integrity — use iodepth for concurrency instead") + return ( + "set -u\n" + 'echo "[pod] $(date -u +%FT%TZ) installing fio"\n' + "apk add --no-cache fio >/dev/null 2>&1 || " + '{ echo "[pod] apk add fio FAILED"; exit 90; }\n' + "mkdir -p /logs\n" + 'echo "[pod] $(date -u +%FT%TZ) starting fio"\n' + + " ".join(args) + "\n" + "rc=$?\n" + 'echo "$rc" > /logs/fio.rc\n' + 'echo "[pod] $(date -u +%FT%TZ) fio exited rc=$rc"\n' + # Stay alive after fio exits so the logs on the emptyDir can still be collected. + "sleep 100000\n" + ) + + def _wait_running(self, ctx: RunContext) -> None: + deadline = time.time() + float(self.opt("ready_timeout_s")) + ns = self.opt("namespace") + phases: dict[str, str] = {} + while time.time() < deadline and not ctx.stopping.is_set(): + cp = kube.run(["-n", ns, "get", "pods", "-l", f"sbtest={ctx.run_id}", + "-o", "json"], check=False) + phases = {} + if cp.returncode == 0 and cp.stdout: + for it in json.loads(cp.stdout).get("items", []): + phases[it["metadata"]["name"]] = it.get("status", {}).get("phase", "?") + bad = [f"{p}={ph}" for p, ph in phases.items() if ph in ("Failed", "Unknown")] + if bad: + raise RuntimeError(f"fio pod(s) failed during startup: {', '.join(bad)}") + running = [p for p, ph in phases.items() if ph == "Running"] + if len(running) == len(self._pods): + ctx.log.info(f"{self.name}: all {len(running)} fio pod(s) Running") + return + time.sleep(5) + # Name the pods that did not make it and what they are stuck at: "6 of 6 did not + # start" sends someone to kubectl for the one fact the message could have carried. + stuck = ", ".join(f"{p}={phases.get(p, 'absent')}" for p in self._pods + if phases.get(p) != "Running") + raise RuntimeError( + f"only {sum(1 for p in self._pods if phases.get(p) == 'Running')} of " + f"{len(self._pods)} fio pod(s) reached Running within " + f"{self.opt('ready_timeout_s')}s: {stuck}") + + def _resolve_pvs(self, ctx: RunContext) -> None: + ns = self.opt("namespace") + for pod in self._pods: + pvc = self._pvc_of[pod] + cp = kube.run(["-n", ns, "get", "pvc", pvc, "-o", "json"]) + pv = json.loads(cp.stdout).get("spec", {}).get("volumeName", "") + if not pv: + raise RuntimeError(f"PVC {pvc} has no bound PV") + cp = kube.run(["get", "pv", pv, "-o", "json"]) + handle = json.loads(cp.stdout).get("spec", {}).get("csi", {}).get( + "volumeHandle", "") + # "::" + parts = handle.split(":") + if len(parts) != 3 or not parts[2]: + raise RuntimeError(f"PV {pv} has an unexpected CSI volume handle {handle!r}") + self._pv_of[pod] = pv + self._pod_of[pv] = pod + self._volume_of[pv] = parts[2] + ctx.log.info(f"{self.name}: resolved PVC -> PV -> lvol:") + for pod in self._pods: + pv = self._pv_of[pod] + ctx.log.info(f" {pod} {self._pvc_of[pod]} -> {pv} " + f"(lvol {self._volume_of[pv]}, {self._kind_of[pod]})") + + def _resolve_subsystems(self, ctx: RunContext) -> None: + """Read each volume's real NVMe subsystem from the backend and group by it. + + Discovered, never assumed: the control plane decides the packing, and everything + downstream — which volumes a migration moves, which nodes must be sampled — follows + from the real grouping. + """ + for pv, lvol in self._volume_of.items(): + nqn, nsid = self._sb.subsystem_of(lvol) + if not nqn: + ctx.log.warn(f"{self.name}: cannot resolve the subsystem of {pv} " + f"(lvol {lvol}); it will migrate as a group of one") + continue + self._nqn_of[pv] = nqn + self._nsid_of[pv] = nsid + groups: dict[str, list[str]] = {} + for pv, nqn in self._nqn_of.items(): + groups.setdefault(nqn, []).append(pv) + self._groups = {n: sorted(m, key=lambda p: self._nsid_of.get(p, 0)) + for n, m in groups.items()} + + shared = {n: m for n, m in self._groups.items() if len(m) > 1} + ctx.log.info(f"{self.name}: {len(self._groups)} subsystem(s), {len(shared)} shared by " + "more than one volume") + for nqn, pvs in sorted(self._groups.items(), key=lambda kv: -len(kv[1])): + members = ", ".join(f"{self._pod_of.get(p, p)}(ns{self._nsid_of.get(p, '?')})" + for p in pvs) + ctx.log.info(f" {nqn} [{len(pvs)}] {members}") + + # Say this at minute one rather than leaving it for the analysis: if the namespaced + # volumes each got their own subsystem, no batch migration can be exercised at all, + # and the run is only testing the single-namespace path whatever its verdict says. + if int(self.opt("ns_pods")) >= 2 and not shared: + ctx.log.warn( + f"{self.name}: the {self.opt('ns_pods')} namespaced volume(s) each got their " + f"OWN subsystem ({PARAM_MAX_NS}={self.opt('ns_per_subsys')}), so no batch " + "migration can be exercised. Check that the CSI driver honours " + f"{PARAM_MAX_NS} and that the volumes landed on the same storage node — a " + "subsystem is shared per node") + + def _publish(self, ctx: RunContext) -> None: + """Hand the driver everything it needs to choose targets and attribute observations.""" + pvs = [self._pv_of[p] for p in self._pods if p in self._pv_of] + node_of: dict[str, str] = {} + cp = kube.run(["-n", self.opt("namespace"), "get", "pods", "-l", + f"sbtest={ctx.run_id}", "-o", "json"], check=False) + if cp.returncode == 0 and cp.stdout: + for it in json.loads(cp.stdout).get("items", []): + node_of[it["metadata"]["name"]] = it.get("spec", {}).get("nodeName", "") or "" + + ctx.shared.update({ + "workload.pvs": pvs, + "workload.pods": list(self._pods), + "workload.pod_of": dict(self._pod_of), + "workload.pv_of": dict(self._pv_of), + "workload.volume_of": dict(self._volume_of), + "workload.node_of": node_of, + "workload.nqn_of": dict(self._nqn_of), + "workload.subsystem_of": {pv: list(self._groups.get(self._nqn_of.get(pv, ""), + [pv])) + for pv in pvs}, + "workload.placement": self._placement(), + "workload.sbctl": self._sb, + }) + + def _placement(self) -> dict[str, str]: + try: + return self._sb.nodes_of(dict(self._volume_of)) + except SbctlError: + return {} + + # ── collection ───────────────────────────────────────────────────────────────── + + def collect(self, ctx: RunContext) -> None: + """Pull fio's own account out of each pod, into the layout the analyser reads. + + Read with `exec cat` rather than `kubectl cp`, which truncates large files without + reporting an error — a silently short result.json reads as a clean run. + """ + ns = self.opt("namespace") + migs = ctx.shared.get("migrations") or [] + for pod in self._pods: + d = ctx.dir(pod) + for remote, local in (("/logs/result.json", "result.json"), + ("/logs/fio.rc", "fio.rc")): + out = kube.exec_sh(ns, pod, f"cat {remote} 2>/dev/null", timeout=120) + if out: + with open(f"{d}/{local}", "w") as fh: + fh.write(out) + log = kube.run(["-n", ns, "logs", pod, "--tail=-1"], check=False, timeout=300) + if log.stdout: + with open(f"{d}/fio.log", "w") as fh: + fh.write(log.stdout) + self._write_timeseries(ctx, ns, pod, d, migs) + ctx.log.info(f"{self.name}: collected fio output for {len(self._pods)} pod(s)") + + def _write_timeseries(self, ctx: RunContext, ns: str, pod: str, d: str, + migs: list) -> None: + """Per-second IOPS from fio's iops log, with the migration in flight that second. + + The migration column is the point: correlating a throughput dip with the migration + that caused it is otherwise a manual join across two files, and the detectors need it + to attribute an outage to a specific migration rather than to the run. + + Column names match the older harness's CSVs (`second`, `wall_clock`) so a single + reader serves both. + """ + raw = kube.exec_sh(ns, pod, "cat /logs/iops.*log 2>/dev/null", timeout=180) + if not raw.strip(): + return + # fio: , , , , ... + per_sec: dict[int, dict[str, float]] = {} + for line in raw.splitlines(): + f = [x.strip() for x in line.split(",")] + if len(f) < 3: + continue + try: + sec = int(int(f[0]) / 1000) + val = float(f[1]) + rw = int(f[2]) + except ValueError: + continue + row = per_sec.setdefault(sec, {"read": 0.0, "write": 0.0}) + row["read" if rw == 0 else "write"] += val + if not per_sec: + return + + start, _ = ctx.window() + with open(f"{d}/timeseries.csv", "w") as fh: + fh.write("second,wall_clock,total_iops,read_iops,write_iops,active_migration\n") + for sec in sorted(per_sec): + row = per_sec[sec] + total = row["read"] + row["write"] + wall = "" + active = "" + if start: + from datetime import timedelta + ts = start + timedelta(seconds=sec) + wall = ts.strftime("%Y-%m-%dT%H:%M:%SZ") + active = next((m.name for m in migs if m.covers(ts)), "") + fh.write(f"{sec},{wall},{total},{row['read']},{row['write']},{active}\n") + + def teardown(self, ctx: RunContext) -> None: + if ctx.shared.get("keep"): + ctx.log.info(f"{self.name}: keep set; leaving {len(self._pods)} pod(s), " + f"{len(self._pvcs)} PVC(s) and the StorageClasses in place") + return + ns = self.opt("namespace") + kube.run(["-n", ns, "delete", "pod", "-l", "sbtest-run=true", + "--ignore-not-found", "--grace-period=5"], check=False, timeout=300) + kube.run(["-n", ns, "delete", "pvc", "-l", "sbtest-run=true", + "--ignore-not-found"], check=False, timeout=300) + for sc in (self._sc_single, self._sc_ns): + if sc: + kube.run(["delete", "sc", sc, "--ignore-not-found"], check=False) + ctx.log.info(f"{self.name}: removed pods, PVCs and StorageClasses") diff --git a/test/framework/sbtest/core/__init__.py b/test/framework/sbtest/core/__init__.py new file mode 100644 index 000000000..0ddf689d2 --- /dev/null +++ b/test/framework/sbtest/core/__init__.py @@ -0,0 +1,39 @@ +"""Framework core: evidence, findings, plugins, config, context, runner.""" + +from .config import Config, Selection, apply_cli_toggles, load, suite_path +from .context import Event, Logger, RunContext, Timeline, iso, now_utc +from .evidence import ( + AnaSample, + ControlEvent, + Evidence, + FioJob, + IopsSample, + LogSpan, + Migration, + NvmeController, + attribute, + freeze_windows, +) +from .findings import Attribution, Finding, Report, Severity, critical, info, warning +from .plugin import ( + Component, + Detector, + SkipDetector, + build_component, + build_detector, + component, + detector, + known_components, + known_detectors, +) +from .runner import Runner, findings_by_subject_table + +__all__ = [ + "AnaSample", "Attribution", "Component", "Config", "Detector", "Event", "Evidence", + "ControlEvent", "Finding", "LogSpan", + "FioJob", "IopsSample", "Logger", "Migration", "NvmeController", "Report", + "RunContext", "Runner", "Selection", "Severity", "SkipDetector", "Timeline", + "apply_cli_toggles", "attribute", "build_component", "build_detector", "component", + "critical", "detector", "findings_by_subject_table", "freeze_windows", "info", "iso", + "known_components", "known_detectors", "load", "now_utc", "suite_path", "warning", +] diff --git a/test/framework/sbtest/core/config.py b/test/framework/sbtest/core/config.py new file mode 100644 index 000000000..93d91b8cd --- /dev/null +++ b/test/framework/sbtest/core/config.py @@ -0,0 +1,162 @@ +"""Suite configuration — which components run, which detectors judge, and with what. + +The shape is deliberately flat and explicit: + + run: + id_prefix: fiomig + outdir: ./runs + components: + logs.stream: {enabled: true, containers: [spdk-container, spdk-proxy-container]} + logs.collect: {enabled: true} + ana.sample: {enabled: true, interval_s: 2.0} + detectors: + ana.freeze-count: {enabled: true, max_freezes: 1} + fio.checksum: {enabled: true, verify_lag_s: 45} + +Two rules make this predictable. A name absent from the config is **off** for components +and **on** for detectors — collecting is a cost you opt into, judging is not something you +should have to remember to switch on. And an unknown option is an error rather than a +silently ignored key, because a threshold that looks set but is not is worse than one that +is obviously missing. + +Suites are YAML. A suite is a document to be read and argued with — every threshold in one +wants a sentence saying why it is that number, and the numbers here were each paid for by a +run that went wrong. JSON cannot carry that sentence, so the format was the wrong one. A +suite written as `.json` still loads, for anything generating them programmatically. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from typing import Any + +import yaml + +DEFAULT_DETECTORS_ON = True +DEFAULT_COMPONENTS_ON = False + + +@dataclass +class Selection: + """A resolved name -> options mapping for one extension point.""" + + enabled: dict[str, dict[str, Any]] = field(default_factory=dict) + disabled: list[str] = field(default_factory=list) + + def __bool__(self) -> bool: + return bool(self.enabled) + + +@dataclass +class Config: + run: dict[str, Any] = field(default_factory=dict) + components: Selection = field(default_factory=Selection) + detectors: Selection = field(default_factory=Selection) + raw: dict[str, Any] = field(default_factory=dict) + + @property + def run_id_prefix(self) -> str: + return str(self.run.get("id_prefix", "sbtest")) + + @property + def outdir(self) -> str: + return str(self.run.get("outdir", ".")) + + +def _resolve(section: dict[str, Any], known: list[str], default_on: bool) -> Selection: + sel = Selection() + section = section or {} + + unknown = set(section) - set(known) + if unknown: + raise ValueError(f"unknown name(s) in config: {sorted(unknown)}; known: {sorted(known)}") + + for name in known: + spec = section.get(name) + if spec is None: + spec = default_on + if isinstance(spec, bool): + if spec: + sel.enabled.setdefault(name, {}) + else: + sel.disabled.append(name) + continue + if not isinstance(spec, dict): + raise ValueError(f"config for {name!r} must be a bool or a mapping, got {type(spec).__name__}") + opts = dict(spec) + if not opts.pop("enabled", True): + sel.disabled.append(name) + continue + sel.enabled[name] = opts + return sel + + +def load(path: str | None, known_components: list[str], known_detectors: list[str], + overrides: dict[str, Any] | None = None) -> Config: + """Read a suite file (or take defaults) and resolve it against what is registered. + + Resolving against the registry here rather than at use time means a typo in a name + fails at startup, with the list of valid names, instead of quietly running a suite that + is missing the check it was written for. + """ + raw: dict[str, Any] = {} + if path: + with open(path) as fh: + text = fh.read() + try: + # JSON keeps its own parser rather than riding YAML's superset handling, which + # differs on duplicate keys and tabs — a generated suite should fail on those + # rather than be quietly reinterpreted. + if path.endswith(".json"): + raw = json.loads(text) if text.strip() else {} + else: + raw = yaml.safe_load(text) or {} + except (yaml.YAMLError, json.JSONDecodeError) as e: + raise ValueError(f"cannot parse suite {path}: {e}") from e + if not isinstance(raw, dict): + raise ValueError(f"suite {path} must be a mapping at the top level, got " + f"{type(raw).__name__}") + for k, v in (overrides or {}).items(): + raw.setdefault(k, v) if not isinstance(v, dict) else raw.setdefault(k, {}).update(v) + + unknown_top = set(raw) - {"run", "components", "detectors", "description"} + if unknown_top: + raise ValueError(f"unknown top-level key(s) in {path}: {sorted(unknown_top)}") + + return Config( + run=raw.get("run", {}) or {}, + components=_resolve(raw.get("components", {}), known_components, DEFAULT_COMPONENTS_ON), + detectors=_resolve(raw.get("detectors", {}), known_detectors, DEFAULT_DETECTORS_ON), + raw=raw, + ) + + +def apply_cli_toggles(sel: Selection, enable: list[str], disable: list[str]) -> Selection: + """Apply `--enable x --disable y` on top of a resolved selection. + + Disable wins over enable when both name the same thing: a run someone explicitly narrowed + should not be widened by a default in a suite file they did not write. + """ + for name in enable: + if name in sel.disabled: + sel.disabled.remove(name) + sel.enabled.setdefault(name, {}) + for name in disable: + sel.enabled.pop(name, None) + if name not in sel.disabled: + sel.disabled.append(name) + return sel + + +def suite_path(name: str) -> str | None: + """Resolve a bare suite name against the bundled suites directory.""" + if os.path.sep in name or name.endswith((".yaml", ".yml", ".json")): + return name if os.path.exists(name) else None + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + for ext in (".yaml", ".yml", ".json"): + p = os.path.join(here, "suites", name + ext) + if os.path.exists(p): + return p + return None diff --git a/test/framework/sbtest/core/context.py b/test/framework/sbtest/core/context.py new file mode 100644 index 000000000..11a36c349 --- /dev/null +++ b/test/framework/sbtest/core/context.py @@ -0,0 +1,166 @@ +"""RunContext — the shared state a component is handed, and the run's event timeline. + +Components get exactly one object, so adding a component never changes a signature +elsewhere. The timeline is the part worth explaining: components record events on it, and +detectors read those events back as evidence. That is how a sampler's observation reaches a +check without the two knowing about each other. +""" + +from __future__ import annotations + +import json +import os +import sys +import threading +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any + + +def now_utc() -> datetime: + return datetime.now(UTC) + + +def iso(ts: datetime) -> str: + return ts.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class Logger: + """Timestamped console log, mirrored to a file in the artifact directory. + + A file mirror rather than console-only because the whole framework exists to make runs + re-readable afterwards, and a run whose log lives only in someone's terminal scrollback + is a run that cannot be reviewed. + """ + + LEVELS = ("DEBUG", "INFO", "EVENT", "WARN", "ERROR", "CRITICAL") + + def __init__(self, path: str | None = None, verbose: bool = False) -> None: + self._fh = open(path, "a", buffering=1) if path else None # noqa: SIM115 + self._verbose = verbose + self._lock = threading.Lock() + + def _emit(self, level: str, msg: str) -> None: + if level == "DEBUG" and not self._verbose: + return + line = f"{iso(now_utc())} [{level:8}] {msg}" + with self._lock: + stream = sys.stderr if level in ("ERROR", "CRITICAL") else sys.stdout + print(line, file=stream, flush=True) + if self._fh: + self._fh.write(line + "\n") + + def debug(self, m: str) -> None: self._emit("DEBUG", m) + def info(self, m: str) -> None: self._emit("INFO", m) + def event(self, m: str) -> None: self._emit("EVENT", m) + def warn(self, m: str) -> None: self._emit("WARN", m) + def error(self, m: str) -> None: self._emit("ERROR", m) + def crit(self, m: str) -> None: self._emit("CRITICAL", m) + + def close(self) -> None: + if self._fh: + self._fh.close() + self._fh = None + + +@dataclass +class Event: + """Something a component observed, timestamped and typed. + + `kind` is what detectors filter on and should be a stable dotted name + ("migration.start", "node.offline"), because a detector keying off a message string + breaks the first time someone improves the wording. + """ + + ts: datetime + kind: str + subject: str = "" + data: dict[str, Any] = field(default_factory=dict) + + +class Timeline: + """Thread-safe ordered record of what happened during a run.""" + + def __init__(self) -> None: + self._events: list[Event] = [] + self._lock = threading.Lock() + + def record(self, kind: str, subject: str = "", **data: Any) -> Event: + ev = Event(ts=now_utc(), kind=kind, subject=subject, data=data) + with self._lock: + self._events.append(ev) + return ev + + def of_kind(self, *kinds: str) -> list[Event]: + with self._lock: + return [e for e in self._events if e.kind in kinds] + + def all(self) -> list[Event]: + with self._lock: + return sorted(self._events, key=lambda e: e.ts) + + def to_list(self) -> list[dict]: + return [{"ts": iso(e.ts), "kind": e.kind, "subject": e.subject, "data": e.data} + for e in self.all()] + + +@dataclass +class RunContext: + """Everything a component may need, and the only thing it is given.""" + + run_id: str + outdir: str + log: Logger + timeline: Timeline = field(default_factory=Timeline) + #: Free-form scratch space shared between components, keyed by component name. Used for + #: the handful of genuine dependencies between them (a workload publishing the pods it + #: created, so a sampler knows which nodes to watch) — always read defensively, since + #: the producing component may be disabled. + shared: dict[str, Any] = field(default_factory=dict) + #: Set when the run is being torn down, so a component's background loop can exit. + stopping: threading.Event = field(default_factory=threading.Event) + + def path(self, *parts: str) -> str: + """A path inside the artifact directory, with parent directories created.""" + p = os.path.join(self.outdir, *parts) + os.makedirs(os.path.dirname(p) or self.outdir, exist_ok=True) + return p + + def dir(self, *parts: str) -> str: + """A *directory* inside the artifact directory, created. Unlike `path`, which creates + the parent of the thing you name, this creates the thing you name.""" + p = os.path.join(self.outdir, *parts) + os.makedirs(p, exist_ok=True) + return p + + def window(self) -> tuple[datetime | None, datetime | None]: + """The run's start and end as currently known — end is None while it is still running. + + Components need the start to place a relative offset (fio counts seconds from its own + launch) on the wall clock, which is the only way an observation from one component can + be lined up against another's. + """ + return getattr(self, "_window_start", None), getattr(self, "_window_end", None) + + def mark_window(self, start: datetime | None = None, end: datetime | None = None) -> None: + """Record when the run began and ended, into run.json. + + Without this nothing in the artifact directory says when the run was, so every + detector that reads a ring buffer has to mark what it finds Attribution.UNKNOWN — and + UNKNOWN counts against the run. A live collect therefore failed on twenty-nine + filesystem shutdowns that had happened hours before it started. + """ + self._window_start = getattr(self, "_window_start", None) or start or now_utc() + if end: + self._window_end = end + self.save_json("run.json", { + "run_id": self.run_id, + "start": iso(self._window_start), + "end": iso(end) if end else None, + }) + + def save_json(self, name: str, obj: Any) -> str: + p = self.path(name) + with open(p, "w") as fh: + json.dump(obj, fh, indent=2, default=str) + return p diff --git a/test/framework/sbtest/core/evidence.py b/test/framework/sbtest/core/evidence.py new file mode 100644 index 000000000..1185be9d9 --- /dev/null +++ b/test/framework/sbtest/core/evidence.py @@ -0,0 +1,277 @@ +"""Evidence — the only thing a detector is allowed to read. + +Detectors do not touch the cluster and do not know how anything was collected. They are +pure functions from Evidence to findings, and that is the whole point: the same detector +runs against a live run's in-memory state and against a four-hour archive on disk, so a +check can be fixed and re-tried against the run that motivated it instead of only against +the next one. Reproducing a verdict offline was the single biggest gap in the harness this +framework grew out of. + +Everything here is lazy. A run's SPDK logs are tens of megabytes per node and most +detectors never open them, so `container_log` yields lines and `ana_samples` is fetched per +migration rather than eagerly for all of them. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import Protocol, runtime_checkable + +# ── the value types detectors reason about ────────────────────────────────────────── + + +@dataclass(frozen=True) +class AnaSample: + """One controller's state on one consuming host at one instant.""" + + ts: datetime + node: str + address: str # ":" — an ip alone names a node, only ip:port names a path + state: str # controller state: live / connecting / resetting / ... + ana: dict[int, str] = field(default_factory=dict) # nsid -> ANA state + phase: str = "" # the migration's phase at that instant, when known + role: str = "" # source / target / other, when the collector knew it + + ACCESSIBLE = ("optimized", "non-optimized", "nonoptimized", "non_optimized") + + def accessible_nsids(self) -> set[int]: + return {n for n, a in self.ana.items() if a in self.ACCESSIBLE} + + @property + def ip(self) -> str: + return self.address.rsplit(":", 1)[0] + + @property + def port(self) -> str: + _, _, p = self.address.rpartition(":") + return p + + +@dataclass +class Migration: + """One migration attempt, as the timeline saw it.""" + + name: str + start: datetime + end: datetime | None = None + phase: str = "" # Completed / Failed / TIMEOUT / ... + source: str = "" # storage-node uuid + target: str = "" + pv: str = "" + pod: str = "" + members: list[str] = field(default_factory=list) # PVs moving together + error: str = "" + # Host-observed cutover instants per node, when the collector derived them. + cutover: dict[str, datetime] = field(default_factory=dict) + + @property + def batch(self) -> bool: + return len(self.members) > 1 + + def covers(self, ts: datetime, lag: timedelta = timedelta(0)) -> bool: + """Whether ts falls in this migration's window, optionally extended by `lag`. + + The lag exists for symptoms that are *detected* later than they happen — an fio + verify failure surfaces when fio next reads the block, seconds to tens of seconds + after the write was lost. Without it a migration's own losses get filed under "no + migration was running", which is exactly how a completed-but-corrupting migration + stayed invisible. + """ + if ts < self.start: + return False + end = self.end or self.start + return ts <= end + lag + + +@dataclass +class FioJob: + """One fio job's outcome for one pod.""" + + pod: str + error: int = 0 # errno fio ended with, 0 = clean + total_iops: float = 0.0 + read_iops: float = 0.0 + write_iops: float = 0.0 + + +@dataclass(frozen=True) +class IopsSample: + """One second of one pod's I/O, from the fio time series.""" + + offset_s: int # seconds since fio started + wall: datetime | None + total_iops: float + + +@dataclass(frozen=True) +class ControlEvent: + """One control-plane event — a status change, an object creation, a task update. + + The control plane's own account of what it thought was happening, which is the other half + of every host-side symptom: "the path went away" and "the control plane decided the node + was down" are the same incident told from two ends. + """ + + ts: datetime + level: str # Info / Warning / Error + kind: str # STATUS_CHANGE / OBJ_CREATED / ... + message: str + subject: str = "" # node / volume / task id, when the event names one + + +@dataclass(frozen=True) +class LogSpan: + """What time range a collected log actually covers. + + Exists because the answer is routinely "less than the run". A log-based verdict over a log + that only covers the last forty minutes of a four-hour run is not a verdict, and nothing + else in the evidence makes that visible. + """ + + name: str + first: datetime | None + last: datetime | None + lines: int = 0 + + +@dataclass(frozen=True) +class NvmeController: + """One NVMe controller on one host, as sysfs reports it. + + The shape that matters for leak detection: a controller can be `live` and serve no + namespace at all, which looks connected from every angle a connect checks. + """ + + node: str + name: str # "nvme7" + nqn: str + address: str # ":" + state: str # live / connecting / ... + namespaces: dict[int, str] = field(default_factory=dict) # nsid -> ana state + ctrl_loss_tmo: int | None = None + + @property + def serves_nothing(self) -> bool: + return not self.namespaces + + +# ── the contract ──────────────────────────────────────────────────────────────────── + + +@runtime_checkable +class Evidence(Protocol): + """What a detector may ask for. Every accessor may legitimately return nothing. + + A detector that needs evidence a run does not have must *say so* (see + `Detector.detect` and `Report.skip`) rather than return no findings — silence is how a + check that cannot run gets mistaken for a check that passed. + """ + + run_id: str + outdir: str + + def migrations(self) -> list[Migration]: ... + + def ana_samples(self, migration: str) -> list[AnaSample]: ... + + def fio_jobs(self) -> list[FioJob]: ... + + def fio_timeseries(self, pod: str) -> list[IopsSample]: ... + + def fio_log(self, pod: str) -> Iterator[str]: ... + + def container_logs(self) -> list[str]: + """Names of the container logs available, e.g. "spdk-4420", "operator".""" + ... + + def container_log(self, name: str) -> Iterator[str]: ... + + def nvme_controllers(self) -> list[NvmeController]: ... + + def pods(self) -> list[str]: ... + + def run_window(self) -> tuple[datetime | None, datetime | None]: + """When the run started and ended, or (None, None) if not known. + + Needed by any detector reading evidence that outlives the run — dmesg is a ring + buffer covering hours, so without a window a detector counts the previous runs' + damage as this one's. A detector that gets (None, None) must mark what it finds + Attribution.UNKNOWN rather than assume. + """ + ... + + def control_events(self) -> list[ControlEvent]: ... + + def log_spans(self) -> list[LogSpan]: + """The time range each collected log covers. Empty when not determinable.""" + ... + + def cluster_uuid(self) -> str: + """The cluster this run ran against, or "" when unknown. + + Present for one reason: an NQN names its cluster, so a controller whose NQN names a + *different* cluster is leaked beyond any doubt — no threshold, no topology, no + "might be a transient reconnect". See detectors/kernel.py::ForeignCluster. + """ + ... + + +# ── helpers shared by detectors ───────────────────────────────────────────────────── + + +def freeze_windows(samples: Iterable[AnaSample], + expected_nsids: set[int] | None = None) -> list[tuple[datetime, float]]: + """The windows in which some namespace had no accessible path on a node. + + Returns (start, seconds) per window, taken from the node that saw the most of them. + Per node rather than merged across nodes: every consuming host sees the same freeze, so + the count is how many times the volume froze, not how many hosts noticed. + + Zero-length windows are dropped. A window one sample wide began and ended between two + samples, so counting it would make the result depend on the sampling interval rather + than on what the volume did. + + This is the primitive behind the freeze-count detector, which is the sharpest predictor + of silent write loss found so far — see detectors/ana.py. + """ + by_node: dict[str, dict[datetime, set[int]]] = {} + for s in samples: + if not s.ana: + continue + per_ts = by_node.setdefault(s.node, {}) + per_ts.setdefault(s.ts, set()).update(s.accessible_nsids()) + + best: list[tuple[datetime, float]] = [] + for per_ts in by_node.values(): + times = sorted(per_ts) + if not times: + continue + want = expected_nsids or {n for acc in per_ts.values() for n in acc} + if not want: + continue + windows: list[tuple[datetime, float]] = [] + start: datetime | None = None + for t in times: + if want - per_ts[t]: + if start is None: + start = t + elif start is not None: + windows.append((start, (t - start).total_seconds())) + start = None + if start is not None: + windows.append((start, (times[-1] - start).total_seconds())) + windows = [w for w in windows if w[1] > 0] + if len(windows) > len(best): + best = windows + return best + + +def attribute(migrations: list[Migration], ts: datetime, + lag: timedelta = timedelta(0)) -> Migration | None: + """The migration a symptom at `ts` belongs to, allowing for detection lag.""" + for m in migrations: + if m.covers(ts, lag): + return m + return None diff --git a/test/framework/sbtest/core/findings.py b/test/framework/sbtest/core/findings.py new file mode 100644 index 000000000..7d6d78f3a --- /dev/null +++ b/test/framework/sbtest/core/findings.py @@ -0,0 +1,223 @@ +"""Findings — what a detector produces, and how a run is judged from them. + +A finding is deliberately more than a log line. The thing that made the migration +post-mortems expensive was never noticing that something was wrong; it was reconstructing, +hours later, *which* migration a symptom belonged to and *what evidence* said so. So a +finding carries its subject, the evidence that supports it, and where to look next — and +the report is assembled from findings rather than printed as it goes. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from enum import Enum, IntEnum + + +class Attribution(Enum): + """Whether a finding is something *this run* did. + + dmesg is a ring buffer and a cluster outlives its runs, so evidence routinely contains + damage and debris from before the run began. Counting that against the run is how a + green build gets blamed for its predecessor's mess — and, worse, how a genuinely broken + run hides inside inherited noise. + + The distinction changes the verdict rather than merely annotating it. A CRITICAL the run + caused is a FAIL. A CRITICAL it *inherited* means the environment was already damaged, so + the run cannot be trusted either way: that is INCONCLUSIVE, which is a different + conversation (fix the environment, run again) from a failure (fix the code). + """ + + RUN = "run" # happened inside the run's window + PRE_EXISTING = "pre-existing" # happened before the run started + UNKNOWN = "unknown" # no usable timestamp, or no known run window + + def __str__(self) -> str: # noqa: D105 + return self.value + + +class Severity(IntEnum): + """How much a finding matters. Ordered, so `max()` over findings is the verdict. + + The line that matters is CRITICAL vs the rest: CRITICAL fails the run. WARNING is for + something a human should read but that does not condemn the build — a threshold + approached, a cleanup that did not complete. INFO carries context that is only + interesting next to another finding. + """ + + INFO = 10 + WARNING = 20 + CRITICAL = 30 + + def __str__(self) -> str: # noqa: D105 + return self.name + + +@dataclass +class Finding: + """One defect, with everything needed to act on it without re-deriving it. + + `subject` is what the finding is *about* — a migration name, a pod, a node. It is what + makes findings joinable across detectors, which is where most of the diagnostic value + turned out to be: "the four migrations that re-froze" and "the four migrations that lost + writes" are only obviously the same set if both name their subjects the same way. + """ + + detector: str + severity: Severity + title: str + subject: str = "" + detail: str = "" + #: Whether the run caused this. Defaults to RUN: most detectors read evidence that only + #: exists because the run produced it, so anything they find is the run's. Detectors + #: reading a ring buffer or a long-lived cluster's state must say otherwise. + attribution: Attribution = Attribution.RUN + # Structured backing for the claim: counts, timestamps, thresholds. Goes to JSON + # verbatim, so keep it to primitives. + evidence: dict = field(default_factory=dict) + # Where a human should look — artifact paths, log offsets. + artifacts: list[str] = field(default_factory=list) + # Optional remediation or interpretation note. Used for findings whose meaning is not + # obvious from the title, which is most of the interesting ones. + note: str = "" + + def to_dict(self) -> dict: + d = asdict(self) + d["severity"] = str(self.severity) + d["attribution"] = str(self.attribution) + return d + + def one_line(self) -> str: + head = f"[{self.severity}] {self.detector}" + if self.subject: + head += f" {self.subject}" + if self.attribution is not Attribution.RUN: + head += f" ({self.attribution})" + return f"{head}: {self.title}" + + @property + def counts_against_the_run(self) -> bool: + """Whether this finding may fail the run. + + UNKNOWN counts: a detector that cannot place an event in time has found something + real, and treating "I am not sure when" as "not this run" is how a defect gets + excused. Only evidence positively dated before the run is excluded. + """ + return self.attribution is not Attribution.PRE_EXISTING + + +def _make(detector: str, severity: Severity, title: str, subject: str, detail: str, + evidence: dict | None, artifacts: list[str] | None, note: str, + attribution: Attribution) -> Finding: + return Finding(detector=detector, severity=severity, title=title, subject=subject, + detail=detail, evidence=evidence or {}, artifacts=artifacts or [], + note=note, attribution=attribution) + + +def critical(detector: str, title: str, subject: str = "", detail: str = "", + evidence: dict | None = None, artifacts: list[str] | None = None, + note: str = "", + attribution: Attribution = Attribution.RUN) -> Finding: + """A defect that fails the run.""" + return _make(detector, Severity.CRITICAL, title, subject, detail, evidence, artifacts, note, + attribution) + + +def warning(detector: str, title: str, subject: str = "", detail: str = "", + evidence: dict | None = None, artifacts: list[str] | None = None, + note: str = "", + attribution: Attribution = Attribution.RUN) -> Finding: + """Something a human should read that does not condemn the run.""" + return _make(detector, Severity.WARNING, title, subject, detail, evidence, artifacts, note, + attribution) + + +def info(detector: str, title: str, subject: str = "", detail: str = "", + evidence: dict | None = None, artifacts: list[str] | None = None, + note: str = "", + attribution: Attribution = Attribution.RUN) -> Finding: + """Context that is interesting next to another finding.""" + return _make(detector, Severity.INFO, title, subject, detail, evidence, artifacts, note, + attribution) + + +@dataclass +class Report: + """Every finding of a run, plus the verdict they add up to.""" + + run_id: str = "" + findings: list[Finding] = field(default_factory=list) + # Detectors that ran but could not decide, and why. A detector that silently returns + # nothing because its evidence was missing is indistinguishable from one that returned + # nothing because the run was clean — which is the failure mode that lets a broken + # check pass a broken run for weeks. + skipped: dict[str, str] = field(default_factory=dict) + + def add(self, *findings: Finding) -> None: + self.findings.extend(findings) + + def skip(self, detector: str, reason: str) -> None: + self.skipped[detector] = reason + + @property + def verdict(self) -> str: + """PASS, FAIL, or INCONCLUSIVE. + + INCONCLUSIVE is not a softer FAIL, it is a different problem. It means the run began + in a state that was already broken — a filesystem shut down before it started, a host + retrying controllers for a cluster that no longer exists — so neither a pass nor a + failure can be believed. The action is to clean the environment and run again, not to + go looking through the code. + """ + if self.failed: + return "FAIL" + if self.inherited_damage: + return "INCONCLUSIVE" + return "PASS" + + @property + def failed(self) -> bool: + """A CRITICAL this run is answerable for.""" + return any(f.severity >= Severity.CRITICAL and f.counts_against_the_run + for f in self.findings) + + @property + def inherited_damage(self) -> bool: + """A CRITICAL that pre-dates the run — the environment was already broken.""" + return any(f.severity >= Severity.CRITICAL and not f.counts_against_the_run + for f in self.findings) + + def attributed(self, attribution: Attribution) -> list[Finding]: + return [f for f in self.findings if f.attribution is attribution] + + def of_severity(self, sev: Severity, counting_only: bool = False) -> list[Finding]: + return [f for f in self.findings if f.severity == sev + and (f.counts_against_the_run or not counting_only)] + + def by_subject(self) -> dict[str, list[Finding]]: + """Findings grouped by what they are about, worst first within each subject. + + This is the join that turns separate detectors into a diagnosis: a subject carrying + both a freeze-count finding and a checksum finding is a much stronger statement than + either alone. + """ + out: dict[str, list[Finding]] = {} + for f in self.findings: + out.setdefault(f.subject or "-", []).append(f) + for v in out.values(): + v.sort(key=lambda f: -int(f.severity)) + return out + + def to_dict(self) -> dict: + return { + "run_id": self.run_id, + "verdict": self.verdict, + "counts": {str(s): len(self.of_severity(s)) for s in Severity}, + "counts_by_attribution": { + str(a): len(self.attributed(a)) for a in Attribution}, + "findings": [f.to_dict() for f in self.findings], + "skipped": self.skipped, + } + + def to_json(self, indent: int = 2) -> str: + return json.dumps(self.to_dict(), indent=indent, sort_keys=False) diff --git a/test/framework/sbtest/core/plugin.py b/test/framework/sbtest/core/plugin.py new file mode 100644 index 000000000..a49feff70 --- /dev/null +++ b/test/framework/sbtest/core/plugin.py @@ -0,0 +1,182 @@ +"""Detectors, components, and the registry that makes both selectable by name. + +Two extension points, deliberately different shapes: + +* A **Detector** is a pure judgement: Evidence in, findings out. No cluster access, no + ordering constraints, no state between runs. That is what makes them cheap to write, to + unit-test against a handful of synthetic samples, and to re-run against an archive. + +* A **Component** does the side-effecting work — starting pods, sampling, following logs, + driving a workload. It has a lifecycle, it can fail, and it may be enabled or disabled + independently of everything else. + +The split is the point. Every check worth having came from an incident, and none of them +should require touching collection code to add. Conversely, turning streaming log +collection on or off should not disturb a single check. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any + +from .findings import Finding + +if TYPE_CHECKING: # pragma: no cover + from .context import RunContext + from .evidence import Evidence + + +class SkipDetector(Exception): + """Raised by a detector whose evidence is absent, so the report can say so. + + The alternative — returning no findings — makes "could not check" and "checked, all + clean" the same output, which is how a broken check passes a broken run. + """ + + +class Detector: + """Base class for a defect detector. + + Subclasses set `name` and implement `detect`. Options arrive through `configure` from + the config file, so thresholds are never hard-coded at the call site — a detector's + default should encode what is known (see the ANA detectors, whose defaults come from + measured runs) while staying overridable per suite. + + Deliberately not a dataclass: a generated __init__ would assign the base class's empty + default over every subclass's `name`, so every finding and every skip would be reported + against an anonymous detector. + """ + + #: Dotted, stable — findings and config both key off it. + name: str = "" + #: One line, shown by `sbtest detectors`. Say what it catches, not how. + summary: str = "" + + def __init__(self) -> None: + self.options: dict[str, Any] = dict(self.defaults()) + + def configure(self, **options: Any) -> Detector: + unknown = set(options) - set(self.defaults()) + if unknown: + raise ValueError( + f"detector {self.name}: unknown option(s) {sorted(unknown)}; " + f"known: {sorted(self.defaults())}") + self.options = {**self.defaults(), **options} + return self + + def defaults(self) -> dict[str, Any]: + """Option names and their defaults. Also the allow-list for `configure`.""" + return {} + + def opt(self, key: str) -> Any: + return self.options.get(key, self.defaults().get(key)) + + def detect(self, ev: Evidence) -> Iterable[Finding]: # pragma: no cover + raise NotImplementedError + + +class Component: + """Base class for a lifecycle unit: collection, sampling, workload, driver. + + Every hook is optional; override only what applies. The runner calls them in this + order, and guarantees `stop`/`teardown` run for any component whose `setup` was + entered, so a component that starts pods is responsible for removing them and will be + given the chance to. + + setup() once, before anything runs. Resolve targets, create helpers. + start() begin doing the thing (sampling, following, driving). + tick() called periodically by the runner; cheap, must not block long. + stop() stop doing the thing. Data already written stays written. + collect() gather artifacts into the run directory. After stop, before detect. + teardown() release cluster resources. Runs even when the run failed. + + `collect` is separate from `stop` because the two differ for exactly the case that + motivated this framework: a streaming collector has nothing to collect at the end (it + has been writing all along) while a post-run collector does all its work there, and the + runner has to be able to run one, the other, or both without either knowing. + """ + + name: str = "" + #: One line, shown by `sbtest components`. + summary: str = "" + #: Whether a failure in `setup` should abort the run. + #: + #: False for collectors: a run that loses one evidence stream is still a run, and the + #: detectors that needed it will report themselves skipped. True for a component that + #: *is* the run — a workload or a migration driver — because continuing without it + #: produces a green result for a test that never happened, which is the worst outcome + #: available. + required: bool = False + + def __init__(self, **options: Any) -> None: + self.options = {**self.defaults(), **options} + unknown = set(options) - set(self.defaults()) + if unknown: + raise ValueError( + f"component {self.name}: unknown option(s) {sorted(unknown)}; " + f"known: {sorted(self.defaults())}") + + def defaults(self) -> dict[str, Any]: + return {} + + def opt(self, key: str) -> Any: + return self.options.get(key) + + # -- lifecycle, all optional ---------------------------------------------------- + def setup(self, ctx: RunContext) -> None: ... + def start(self, ctx: RunContext) -> None: ... + def tick(self, ctx: RunContext) -> None: ... + def stop(self, ctx: RunContext) -> None: ... + def collect(self, ctx: RunContext) -> None: ... + def teardown(self, ctx: RunContext) -> None: ... + + +# ── registry ──────────────────────────────────────────────────────────────────────── + +_DETECTORS: dict[str, Callable[[], Detector]] = {} +_COMPONENTS: dict[str, Callable[..., Component]] = {} + + +def detector(cls: type[Detector]) -> type[Detector]: + """Register a detector class under its `name`.""" + if not cls.name: + raise ValueError(f"{cls.__qualname__} must set `name`") + if cls.name in _DETECTORS: + raise ValueError(f"duplicate detector name {cls.name!r}") + _DETECTORS[cls.name] = cls + return cls + + +def component(cls: type[Component]) -> type[Component]: + """Register a component class under its `name`.""" + if not cls.name: + raise ValueError(f"{cls.__qualname__} must set `name`") + if cls.name in _COMPONENTS: + raise ValueError(f"duplicate component name {cls.name!r}") + _COMPONENTS[cls.name] = cls + return cls + + +def known_detectors() -> dict[str, type[Detector]]: + return dict(sorted(_DETECTORS.items())) # type: ignore[arg-type] + + +def known_components() -> dict[str, type[Component]]: + return dict(sorted(_COMPONENTS.items())) # type: ignore[arg-type] + + +def build_detector(name: str, **options: Any) -> Detector: + try: + cls = _DETECTORS[name] + except KeyError: + raise KeyError(f"unknown detector {name!r}; known: {sorted(_DETECTORS)}") from None + return cls().configure(**options) + + +def build_component(name: str, **options: Any) -> Component: + try: + cls = _COMPONENTS[name] + except KeyError: + raise KeyError(f"unknown component {name!r}; known: {sorted(_COMPONENTS)}") from None + return cls(**options) diff --git a/test/framework/sbtest/core/runner.py b/test/framework/sbtest/core/runner.py new file mode 100644 index 000000000..afb9a3c47 --- /dev/null +++ b/test/framework/sbtest/core/runner.py @@ -0,0 +1,234 @@ +"""The runner — drives component lifecycles, then runs detectors over the evidence. + +Two entry points, and the important thing is that they share the second half: + + Runner.execute() full run: components through their lifecycle, then judge. + Runner.judge(ev) judge alone, over evidence from anywhere — including an archive. + +Sharing the judging half is what makes a check fixable against the run that motivated it. +""" + +from __future__ import annotations + +import time +import traceback +from typing import Any + +from .config import Config +from .context import RunContext, now_utc +from .evidence import Evidence +from .findings import Report, Severity, warning +from .plugin import Component, Detector, SkipDetector, build_component, build_detector + + +class Runner: + def __init__(self, cfg: Config, ctx: RunContext) -> None: + self.cfg = cfg + self.ctx = ctx + self.components: list[Component] = [] + self.detectors: list[Detector] = [] + self.report = Report(run_id=ctx.run_id) + #: Components whose setup was entered, so teardown is owed to them. + self._entered: list[Component] = [] + + # ── wiring ───────────────────────────────────────────────────────────────────── + + def build(self) -> Runner: + for name, opts in self.cfg.components.enabled.items(): + try: + self.components.append(build_component(name, **opts)) + except Exception as e: # noqa: BLE001 + raise RuntimeError(f"cannot build component {name!r}: {e}") from e + for name, opts in self.cfg.detectors.enabled.items(): + try: + self.detectors.append(build_detector(name, **opts)) + except Exception as e: # noqa: BLE001 + raise RuntimeError(f"cannot build detector {name!r}: {e}") from e + + if self.cfg.components.disabled: + self.ctx.log.info("components disabled: " + ", ".join(sorted(self.cfg.components.disabled))) + if self.cfg.detectors.disabled: + self.ctx.log.info("detectors disabled: " + ", ".join(sorted(self.cfg.detectors.disabled))) + self.ctx.log.info(f"components enabled ({len(self.components)}): " + + (", ".join(c.name for c in self.components) or "-")) + self.ctx.log.info(f"detectors enabled ({len(self.detectors)}): " + + (", ".join(d.name for d in self.detectors) or "-")) + return self + + # ── lifecycle ────────────────────────────────────────────────────────────────── + + def _phase(self, hook: str, comps: list[Component], fatal: bool = False) -> None: + """Call one hook on each component, recording rather than raising on failure. + + A collector that fails must not take the run with it — the run's purpose is the + workload and the judgement, and partial evidence still judges; the failure becomes a + WARNING finding so the gap is on the record rather than invisible. + + `fatal` is passed for the setup of a component that declares itself `required`, + which is how a workload or a migration driver says that continuing without it would + produce a green result for a test that never ran. + """ + for c in comps: + try: + getattr(c, hook)(self.ctx) + except Exception as e: # noqa: BLE001 + self.ctx.log.error(f"component {c.name}.{hook} failed: {e}") + self.ctx.log.debug(traceback.format_exc()) + self.report.add(warning( + detector=f"component/{c.name}", + title=f"{hook} failed: {e}", + subject=c.name, + note="Evidence this component would have produced may be missing; any " + "detector that depends on it will report itself skipped.", + )) + if fatal: + raise + + def setup(self) -> None: + # Before any component runs: the window has to include setup, because a component + # that breaks something breaks it during setup as much as during the run. + self.ctx.mark_window() + for c in self.components: + # Appended before the hook runs, so a component that failed half-way through + # allocating still gets the teardown it is owed. + self._entered.append(c) + self._phase("setup", [c], fatal=c.required) + + def start(self) -> None: + self._phase("start", self.components) + + def tick(self) -> None: + self._phase("tick", self.components) + + def run_for(self, seconds: float, tick_every: float = 5.0) -> None: + """Tick components for `seconds`. The scenario's own work happens in a component.""" + deadline = time.time() + seconds + while time.time() < deadline and not self.ctx.stopping.is_set(): + self.tick() + time.sleep(min(tick_every, max(0.0, deadline - time.time()))) + + def stop(self) -> None: + self.ctx.stopping.set() + self._phase("stop", list(reversed(self.components))) + + def collect(self) -> None: + self._phase("collect", self.components) + # Closed after collection, so the window covers everything the artifacts contain. + self.ctx.mark_window(end=now_utc()) + + def teardown(self) -> None: + self._phase("teardown", list(reversed(self._entered))) + + # ── judging ──────────────────────────────────────────────────────────────────── + + def judge(self, ev: Evidence) -> Report: + """Run every enabled detector over `ev` and fold the results into the report.""" + for d in self.detectors: + try: + found = list(d.detect(ev)) + except SkipDetector as e: + self.report.skip(d.name, str(e) or "evidence not available") + continue + except Exception as e: # noqa: BLE001 + # A detector that throws is a bug in the detector, not a clean run. Say so + # loudly rather than letting it look like "nothing found". + self.ctx.log.error(f"detector {d.name} raised: {e}") + self.ctx.log.debug(traceback.format_exc()) + self.report.skip(d.name, f"raised {type(e).__name__}: {e}") + self.report.add(warning( + detector=d.name, title=f"detector raised {type(e).__name__}: {e}", + note="This is a detector bug: the run was not judged on this dimension.")) + continue + self.report.add(*found) + return self.report + + def execute(self, evidence_for: Any, duration_s: float = 0.0) -> Report: + """Full run: lifecycle, then judge. `evidence_for` builds Evidence from the ctx.""" + try: + self.setup() + self.start() + if duration_s: + self.run_for(duration_s) + self.stop() + self.collect() + finally: + self.teardown() + return self.judge(evidence_for(self.ctx)) + + # ── output ───────────────────────────────────────────────────────────────────── + + def emit(self, json_name: str = "findings.json") -> Report: + r = self.report + log = self.ctx.log + line = "=" * 78 + log.info(line) + log.info("FINDINGS") + log.info(line) + + if not r.findings: + log.info("no findings") + for sev in (Severity.CRITICAL, Severity.WARNING, Severity.INFO): + group = [f for f in r.of_severity(sev) if f.counts_against_the_run] + if not group: + continue + log.info(f"{sev} ({len(group)}):") + for f in group: + emit = log.crit if sev is Severity.CRITICAL else ( + log.warn if sev is Severity.WARNING else log.info) + emit(f" {f.subject or '-'}: {f.title}") + if f.detail: + for chunk in f.detail.splitlines(): + emit(f" {chunk}") + if f.note: + emit(f" note: {f.note}") + for a in f.artifacts: + emit(f" evidence: {a}") + + inherited = [f for f in r.findings if not f.counts_against_the_run] + if inherited: + log.info(line) + log.info(f"PRE-EXISTING ({len(inherited)}) — already true before this run began, " + "so not counted against it:") + for f in inherited: + emit = log.crit if f.severity is Severity.CRITICAL else log.warn + emit(f" [{f.severity}] {f.subject or '-'}: {f.title}") + if f.detail: + for chunk in f.detail.splitlines(): + emit(f" {chunk}") + if f.note: + emit(f" note: {f.note}") + + if r.skipped: + log.info(f"skipped ({len(r.skipped)}) — not judged on these dimensions:") + for name, why in sorted(r.skipped.items()): + log.warn(f" {name}: {why}") + + path = self.ctx.path(json_name) + with open(path, "w") as fh: + fh.write(r.to_json()) + log.info(line) + summary = (f"{len(r.of_severity(Severity.CRITICAL, counting_only=True))} critical, " + f"{len(r.of_severity(Severity.WARNING, counting_only=True))} warning") + if inherited: + summary += f", {len(inherited)} pre-existing" + log.info(f"RESULT: {r.verdict} ({summary}) -> {path}") + if r.verdict == "INCONCLUSIVE": + log.crit("The environment was already damaged before this run started, so neither " + "a pass nor a failure can be believed. Clean it up and run again.") + log.info(line) + return r + + +def findings_by_subject_table(report: Report) -> list[str]: + """Findings grouped by subject — the join that makes correlated defects visible. + + A subject carrying two independent findings is the strongest signal these runs produce: + "re-froze the volume" plus "lost writes" on the same migration is what turned a + correlation into a root cause. + """ + lines = [] + for subject, group in sorted(report.by_subject().items()): + if len(group) < 2: + continue + lines.append(f"{subject}: " + "; ".join(f"{f.detector}({f.severity})" for f in group)) + return lines diff --git a/test/framework/sbtest/detectors/__init__.py b/test/framework/sbtest/detectors/__init__.py new file mode 100644 index 000000000..8a44ddba9 --- /dev/null +++ b/test/framework/sbtest/detectors/__init__.py @@ -0,0 +1,23 @@ +"""Bundled detectors. Importing this module registers them all. + +Each module groups checks by the evidence they read, not by the subsystem they blame: +`ana` reads host path samples, `fio` reads the workload's own output, `nvme` reads a fabric +snapshot, `logs` reads collected container logs, `migration` reads the timeline, `kernel` reads +dmesg — the only source for what the *host* did about a fabric event — `control` reads the +control plane's own event log, `security` scans whatever was collected, and `meta` judges the +evidence itself rather than the system. +""" + +from . import ( # noqa: F401 + ana, + control, + fio, + kernel, + logs, + meta, + migration, + nvme, + security, +) + +__all__ = ["ana", "control", "fio", "kernel", "logs", "meta", "migration", "nvme", "security"] diff --git a/test/framework/sbtest/detectors/ana.py b/test/framework/sbtest/detectors/ana.py new file mode 100644 index 000000000..0a0ed519f --- /dev/null +++ b/test/framework/sbtest/detectors/ana.py @@ -0,0 +1,311 @@ +"""ANA / host-path detectors. + +These are the checks that came out of the migration corruption work, and the first one is +the reason this framework has a detector abstraction at all: it is a one-number check that +predicted silent data loss exactly, and it was not expressible in the harness without +editing the harness. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import ( + AnaSample, + Detector, + Evidence, + Finding, + SkipDetector, + critical, + detector, + freeze_windows, + info, + warning, +) + +#: How long the control plane deliberately holds every path inaccessible while it moves a +#: volume. Measured, not guessed: it is a `time.sleep(2)` in the batch migration barrier. +CUTOVER_PAUSE_DESIGN_S = 2.0 + + +@detector +class FreezeCount(Detector): + """More than one cutover freeze in a migration. + + On the run this came from (fiomig-1787171993, 46 migrations) this was **exact**: the + four migrations that froze the volume more than once are precisely the four that + silently lost writes, and none of the other 42 lost anything. + + The mechanism is why it works. A migration takes the pause once; a second window means + the control plane released the source back into the read/write path and retried the + transfer, and the retry replays a non-idempotent step against a source that has been + serving writes in between. The freeze count is the retry count. + + Prefer this over the pause *duration*, which the same run showed to be both less + sensitive and less specific: two 3s freezes look like one healthy pause on any + longest-window measure, and one migration's single 5-6s pause lost nothing. + """ + + name = "ana.freeze-count" + summary = ("a migration that froze the volume more than once — retried cutover, and an " + "exact predictor of silent write loss so far") + + def defaults(self) -> dict: + return { + "max_freezes": 1, + # A freeze shorter than this is treated as sampling noise rather than a window. + # 0 keeps every non-zero window, which is what the measured result used. + "min_window_s": 0.0, + } + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples: + continue + judged += 1 + windows = [w for w in freeze_windows(samples) + if w[1] >= float(self.opt("min_window_s"))] + if len(windows) <= int(self.opt("max_freezes")): + continue + spans = ", ".join(f"{s.strftime('%H:%M:%S')}+{d:.0f}s" for s, d in windows) + yield critical( + self.name, + title=f"volume froze {len(windows)} times during one migration", + subject=m.name, + detail=f"windows: {spans}", + evidence={"freezes": len(windows), "max_freezes": int(self.opt("max_freezes")), + "windows_s": [round(d, 1) for _, d in windows], + "phase": m.phase, "members": len(m.members)}, + note="A migration takes the cutover pause once; the rest are retries of a " + "step that did not take. Every migration observed to freeze more than " + "once has also silently lost writes — treat as corruption until the " + "checksums say otherwise, including when the phase is Completed.", + ) + if not judged: + raise SkipDetector("no ANA samples for any migration") + + +@detector +class CutoverPause(Detector): + """A cutover pause that ran longer than the design window allows. + + A bounded pause is expected — the control plane drives every path inaccessible for about + CUTOVER_PAUSE_DESIGN_S while it moves the volume — so this bounds that window rather + than forbidding it. The default allows the design window plus one sampling interval, + which is the widest a well-behaved pause can be *measured* as. + + Keep it alongside ana.freeze-count rather than instead of it: this catches a single + window that overran, which the count cannot see. + """ + + name = "ana.cutover-pause" + summary = "an all-paths-inaccessible window longer than the cutover pause allows" + + def defaults(self) -> dict: + return {"max_pause_s": CUTOVER_PAUSE_DESIGN_S + 3.0, + "design_pause_s": CUTOVER_PAUSE_DESIGN_S} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + limit = float(self.opt("max_pause_s")) + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples: + continue + judged += 1 + windows = freeze_windows(samples) + if not windows: + continue + worst = max(d for _, d in windows) + if worst <= limit: + continue + yield critical( + self.name, + title=f"every path to some namespace was inaccessible for {worst:.0f}s", + subject=m.name, + detail=f"limit {limit:.0f}s; the cutover pause is meant to last about " + f"{float(self.opt('design_pause_s')):.0f}s", + evidence={"worst_pause_s": round(worst, 1), "max_pause_s": limit, + "freezes": len(windows), "phase": m.phase}, + note="An application that sees an I/O error during a migration sees it " + "inside this window, so its length bounds the blast radius.", + ) + if not judged: + raise SkipDetector("no ANA samples for any migration") + + +@detector +class SplitBrain(Detector): + """Source and target both serving at the same instant. + + Two simultaneously optimized paths to two copies means a read can land on either, and + the one that is behind returns data that was never written there. This is the one ANA + defect that is silent corruption *by construction* rather than by race, so it is worth + checking even though it has not yet been observed. + """ + + name = "ana.split-brain" + summary = "source and target paths both optimized at the same instant (two writers)" + + def defaults(self) -> dict: + return {"optimized_states": ["optimized"]} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + opt = set(self.opt("optimized_states")) + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples or not any(s.role for s in samples): + continue + judged += 1 + per_ts: dict = {} + for s in samples: + if any(a in opt for a in s.ana.values()): + per_ts.setdefault(s.ts, {}).setdefault(s.role or "?", set()).add(s.address) + for ts, roles in sorted(per_ts.items()): + if "source" in roles and "target" in roles: + yield critical( + self.name, + title="source and target both optimized at the same instant", + subject=m.name, + detail=f"at {ts:%H:%M:%S}: source={sorted(roles['source'])} " + f"target={sorted(roles['target'])}", + evidence={"ts": str(ts), "source": sorted(roles["source"]), + "target": sorted(roles["target"])}, + note="Reads can be served by either copy; the one that is behind " + "returns data that was never written at that offset.", + ) + break # one finding per migration is enough to act on + if not judged: + raise SkipDetector("no role-labelled ANA samples (needs source/target roles)") + + +@detector +class UnservedAfterCutover(Detector): + """A completed migration whose target does not serve every namespace. + + The half-moved case seen from the host: the migration reports success, a live controller + exists at the target, and one of the subsystem's namespaces has no path over it. On a + shared subsystem that is one volume left stranded while its siblings moved. + """ + + name = "ana.unserved-after-cutover" + summary = "after a Completed migration, a live target controller serves only some namespaces" + + def defaults(self) -> dict: + return {"completed_phases": ["Completed"]} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = [m for m in ev.migrations() if m.phase in set(self.opt("completed_phases"))] + if not migs: + raise SkipDetector("no completed migrations in this run") + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + targets = [s for s in samples if s.role == "target"] + if not targets: + continue + judged += 1 + last_ts = max(s.ts for s in samples) + final = [s for s in targets if s.ts == last_ts] + live = [s for s in final if s.state == "live"] + served = {n for s in live for n in s.accessible_nsids()} + expected = {n for s in samples for n in s.ana} + missing = expected - served + if live and expected and missing: + yield critical( + self.name, + title=f"{len(missing)} namespace(s) unserved on the target after cutover", + subject=m.name, + detail=f"target serves {sorted(served) or '-'} of {sorted(expected)}; " + f"missing {sorted(missing)}", + evidence={"served": sorted(served), "expected": sorted(expected), + "missing": sorted(missing)}, + ) + elif not live: + yield warning( + self.name, + title="no live target controller after a completed migration", + subject=m.name, + detail=f"controllers at the target: " + f"{sorted({(s.address, s.state) for s in final})}", + note="Non-live controllers at the target address are normal while the " + "old instance is torn down; a persisting one is not.", + ) + if not judged: + raise SkipDetector("no role-labelled ANA samples for completed migrations") + + +@detector +class PathChurn(Detector): + """An unusual number of distinct path addresses for one subsystem on one host. + + A leak indicator that does not need a live cluster: a subsystem accumulating listeners + it never sheds shows up in the ANA samples as a growing address set. Informational by + default, because the healthy count is topology-dependent (primary plus HA replicas per + side) and a threshold that fits one cluster will not fit another. + """ + + name = "ana.path-churn" + summary = "more distinct path addresses per host than the topology should produce" + + def defaults(self) -> dict: + return {"max_addresses": 6, "severity": "info"} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + limit = int(self.opt("max_addresses")) + make = critical if self.opt("severity") == "critical" else ( + warning if self.opt("severity") == "warning" else info) + judged = 0 + for m in migs: + samples = ev.ana_samples(m.name) + if not samples: + continue + judged += 1 + per_node: dict[str, set[str]] = {} + for s in samples: + per_node.setdefault(s.node, set()).add(s.address) + for node, addrs in sorted(per_node.items()): + if len(addrs) > limit: + yield make( + self.name, + title=f"{len(addrs)} distinct path addresses on {node}", + subject=m.name, + detail=", ".join(sorted(addrs)), + evidence={"node": node, "addresses": sorted(addrs), "limit": limit}, + note="An ip names a node; only ip:port names a path. Extra " + "addresses on one node are usually leaked listeners from " + "earlier migrations.", + ) + if not judged: + raise SkipDetector("no ANA samples for any migration") + + +def freeze_summary(ev: Evidence) -> list[tuple[str, int, float]]: + """(migration, freezes, worst window) for every migration with samples. + + Exposed because it is the table that makes the freeze-count result legible at a glance, + and callers other than the detector want it — the CLI prints it and the README quotes it. + """ + out = [] + for m in ev.migrations(): + samples: list[AnaSample] = ev.ana_samples(m.name) + if not samples: + continue + w = freeze_windows(samples) + out.append((m.name, len(w), max((d for _, d in w), default=0.0))) + return out diff --git a/test/framework/sbtest/detectors/control.py b/test/framework/sbtest/detectors/control.py new file mode 100644 index 000000000..319d4dadc --- /dev/null +++ b/test/framework/sbtest/detectors/control.py @@ -0,0 +1,341 @@ +"""Control-plane detectors, from the cluster's own event log. + +The other half of every host-side symptom. "Every path went away" and "the control plane +decided the node was down" are the same incident told from two ends, and only one of those +ends explains *why*. + +Nothing here is migration-specific — these are the failure shapes a distributed control plane +has regardless of what operation is running, which is why they belong in a framework rather +than in a migration test. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Iterable +from datetime import UTC, datetime, timedelta + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector, info, warning + + +@detector +class NodeFlap(Detector): + """A node that went offline and came back inside one run. + + The shape behind a real 9.5-hour outage: a three-second kube-apiserver blip made the + liveness probe conclude SPDK was dead — the probe listed pods through the Kubernetes API — + so three storage nodes were marked offline at once, and nothing re-probed them afterwards. + + A flap is therefore worth reporting *even when it recovered*, because the recovery is + luck: the same trigger with a slower re-probe is an outage. A short flap is the signal, not + the noise — a node that is genuinely gone stays gone. + """ + + name = "control.node-flap" + summary = "a storage node marked offline and back inside one run — liveness, not liveliness" + + RE_STATUS = re.compile( + r"(?:Storage node|Management node) status changed from:?\s*(\S+)\s*to:?\s*(\S+)") + #: States that mean the control plane stopped trusting the node. + DOWN = {"down", "offline", "unreachable", "suspended"} + UP = {"online", "active"} + + def defaults(self) -> dict: + return {"max_flaps": 0, + # A transition back inside this window is a flap rather than a real outage + # followed by a real recovery. + "flap_within_s": 300.0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + window = timedelta(seconds=float(self.opt("flap_within_s"))) + + # subject -> [(ts, went_down)] + transitions: dict[str, list] = {} + for e in events: + m = self.RE_STATUS.search(e.message) + if not m: + continue + frm, to = m.group(1).strip(". "), m.group(2).strip(". ") + if to in self.DOWN: + transitions.setdefault(e.subject or "?", []).append((e.ts, True, frm, to)) + elif to in self.UP and frm in self.DOWN: + transitions.setdefault(e.subject or "?", []).append((e.ts, False, frm, to)) + + flaps = [] + for subject, seq in transitions.items(): + seq.sort() + for i, (ts, down, _f, to) in enumerate(seq): + if not down: + continue + back = next(((t2, f2) for t2, d2, f2, _t2 in seq[i + 1:] if not d2), None) + if back and back[0] - ts <= window: + flaps.append((subject, ts, (back[0] - ts).total_seconds(), to)) + + if len(flaps) <= int(self.opt("max_flaps")): + return + yield critical( + self.name, + title=f"{len(flaps)} node flap(s): marked down and back within " + f"{float(self.opt('flap_within_s')):.0f}s", + subject="control-plane", + detail="; ".join(f"{s[:8]} {t:%H:%M:%S} down->up in {d:.0f}s (as {to})" + for s, t, d, to in sorted(flaps, key=lambda x: x[1])[:8]), + evidence={"flaps": [{"subject": s, "at": str(t), "seconds": round(d), "state": to} + for s, t, d, to in flaps]}, + note="A node that recovers in seconds was probably never gone: the usual cause is " + "a liveness check that depends on something other than the node — an API " + "call, a lock, a lease. Every volume whose paths were on it took a hit for " + "the duration, and the recovery here was luck rather than design.", + ) + + +@detector +class TaskStuck(Detector): + """A control-plane task that never reached a terminal state. + + Generic on purpose. Whatever the task is — migration, rebalance, backup, node add — one + that is created and never resolved holds locks, blocks the next operation of its kind, and + makes the cluster's state un-diagnosable. This is the check that says "the control plane + stopped finishing things", which no per-operation test asks. + """ + + name = "control.task-stuck" + summary = "a task created during the run that never reached a terminal state" + + RE_CREATED = re.compile(r"[Tt]ask created") + RE_UPDATED = re.compile(r"[Tt]ask updated") + TERMINAL = ("done", "completed", "failed", "cancelled", "canceled", "suspended") + + def defaults(self) -> dict: + return {"max_stuck": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + + created = sum(1 for e in events if self.RE_CREATED.search(e.message)) + if not created: + raise SkipDetector("the event log records no task creations to follow") + terminal = sum(1 for e in events + if self.RE_UPDATED.search(e.message) + and any(t in e.message.lower() for t in self.TERMINAL)) + outstanding = created - terminal + if outstanding <= int(self.opt("max_stuck")): + yield info(self.name, title=f"{created} task(s) created, {terminal} resolved", + subject="control-plane", + evidence={"created": created, "terminal": terminal}) + return + yield warning( + self.name, + title=f"{outstanding} task(s) created but never seen to finish", + subject="control-plane", + detail=f"{created} created, {terminal} reached a terminal state", + evidence={"created": created, "terminal": terminal, "outstanding": outstanding}, + note="The event log may simply end before they finished — check it against the run " + "window before treating this as a leak. A task that is genuinely stuck holds " + "its lock and blocks the next operation of its kind.", + ) + + +@detector +class VolumeHealth(Detector): + """A volume whose health went false and never came back. + + Independent of what the run was doing: whatever the operation, a volume that ends the run + unhealthy is a volume someone has to go and look at. On one archived run five volumes went + unhealthy mid-migration and only three recovered — the other two stayed down for the rest + of the run, which is a much more useful statement than the migration's own phase. + """ + + name = "control.volume-health" + summary = "a volume or node whose health went false during the run and never recovered" + + RE_HEALTH = re.compile(r"(\S+) health check changed from:?\s*(\S+)\s*to:?\s*(\S+)") + + def defaults(self) -> dict: + return {"max_unrecovered": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + state: dict[str, bool] = {} + seen = False + for e in events: + m = self.RE_HEALTH.search(e.message) + if not m: + continue + seen = True + to = m.group(3).strip(". ").lower() + state[e.subject or m.group(1)] = to in ("true", "healthy", "online") + if not seen: + raise SkipDetector("the event log records no health transitions") + + bad = sorted(k for k, ok in state.items() if not ok) + if len(bad) <= int(self.opt("max_unrecovered")): + return + yield critical( + self.name, + title=f"{len(bad)} object(s) ended the run unhealthy", + subject="control-plane", + detail=", ".join(b[:12] for b in bad[:16]) + (" ..." if len(bad) > 16 else ""), + evidence={"unhealthy": bad}, + note="Health that goes false and does not return outlives the run: the next run " + "starts from here. Check it against nvme.dirty-start on the following run.", + ) + + +@detector +class RetryStorm(Detector): + """One operation attempted far more often than it should need to be. + + A retry loop is invisible in a pass/fail result and obvious in a count. It matters beyond + the operation retried: each attempt usually re-does whatever the previous one half-did, + which is how a retry turns a failure into damage — the batch-migration corruption is + exactly that shape. + """ + + name = "control.retry-storm" + summary = "the same control-plane operation attempted far more often than it should be" + + def defaults(self) -> dict: + return {"max_repeats": 5, "kinds": ["STATUS_CHANGE"]} + + @staticmethod + def _shape(msg: str) -> str: + s = re.sub(r"[0-9a-f]{8}-[0-9a-f-]{27}", "", msg) + s = re.sub(r"\d+", "N", s) + return s[:120] + + def detect(self, ev: Evidence) -> Iterable[Finding]: + events = ev.control_events() + if not events: + raise SkipDetector("no control-plane event log collected") + kinds = set(self.opt("kinds") or []) + counts: dict[tuple[str, str], int] = {} + for e in events: + if kinds and e.kind not in kinds: + continue + counts[(e.subject or "?", self._shape(e.message))] = \ + counts.get((e.subject or "?", self._shape(e.message)), 0) + 1 + + hot = {k: v for k, v in counts.items() if v > int(self.opt("max_repeats"))} + if not hot: + return + worst = sorted(hot.items(), key=lambda kv: -kv[1])[:8] + yield warning( + self.name, + title=f"{len(hot)} operation(s) repeated more than {self.opt('max_repeats')} times", + subject="control-plane", + detail="; ".join(f"{n}x {msg}" for (_subj, msg), n in worst), + evidence={"repeats": [{"subject": s, "shape": m, "count": n} + for (s, m), n in worst]}, + note="Each attempt usually re-does what the last one half-did, which is how a " + "retry turns a failure into damage rather than merely delaying it.", + ) + + +@detector +class NodeAgent(Detector): + """The node-side agent's own account: failed calls, and gaps in the liveness polling. + + The storage-node DaemonSet is what starts and probes the SPDK process, so it sits on the + causal path of every "the node went offline" decision. Its access log is mostly liveness + polling — `/snode/check`, `/snode/spdk_process_is_up`, `/snode/ping_ip`, thousands of each + per run — and two things in it are worth watching: + + * **a non-2xx response.** The control plane asked whether SPDK was alive and got an error + rather than an answer. Whatever it decided next, it decided on no information; this is + the upstream half of `control.node-flap`. + * **a gap in the polling.** Either the control plane stopped asking or the agent stopped + answering, and both precede a node being declared down. A poll that never happened + leaves no other trace, so this is the only place the stall is visible. + + Grounded rather than speculative: the outage this is built for was a liveness check that + concluded SPDK was dead because a Kubernetes API call it depended on blipped for three + seconds, and three nodes went offline at once. + """ + + name = "control.node-agent" + summary = "the node agent returning errors, or gaps in the liveness polling it answers" + + RE_ACCESS = re.compile(r'"(?:GET|POST|PUT|DELETE) (/snode/[a-z_]+)[^"]*" (\d{3})') + RE_TS = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})") + + def defaults(self) -> dict: + return {"logs": ["snode-api-*"], + # A liveness poll is expected every couple of seconds; this is where a gap + # stops being scheduling jitter and starts being a stall. + "max_poll_gap_s": 60.0, + "liveness_endpoints": ["/snode/check", "/snode/spdk_process_is_up"], + "max_errors": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + names = [n for n in ev.container_logs() + if any(fnmatch.fnmatch(n, g) for g in self.opt("logs"))] + if not names: + raise SkipDetector("no node-agent logs collected (enable the storage-node " + "DaemonSet target in logs.collect)") + + live = set(self.opt("liveness_endpoints") or []) + gap_limit = float(self.opt("max_poll_gap_s")) + errors: dict[str, dict[str, int]] = {} + gaps: list[tuple[str, str, float]] = [] + + for name in names: + last_poll: datetime | None = None + for raw in ev.container_log(name): + m = self.RE_ACCESS.search(raw) + if not m: + continue + endpoint, status = m.group(1), m.group(2) + if not status.startswith("2"): + key = f"{endpoint} {status}" + per = errors.setdefault(name, {}) + per[key] = per.get(key, 0) + 1 + if endpoint not in live: + continue + mt = self.RE_TS.match(raw) + if not mt: + continue + try: + ts = datetime.strptime(mt.group(1), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + except ValueError: + continue + if last_poll is not None and (ts - last_poll).total_seconds() > gap_limit: + gaps.append((name, f"{last_poll:%H:%M:%S}", (ts - last_poll).total_seconds())) + last_poll = ts + + total_errors = sum(sum(v.values()) for v in errors.values()) + if total_errors > int(self.opt("max_errors")): + yield critical( + self.name, + title=f"{total_errors} node-agent call(s) returned an error", + subject="control-plane", + detail="; ".join(f"{n}: " + ", ".join(f"{k} x{c}" for k, c in sorted(v.items())) + for n, v in sorted(errors.items())), + evidence={"errors": errors}, + note="The control plane asked and got an error rather than an answer, so " + "whatever it decided next it decided on no information. Read with " + "control.node-flap: this is the upstream half of a false offline.", + ) + + if gaps: + worst = sorted(gaps, key=lambda g: -g[2])[:6] + yield warning( + self.name, + title=f"{len(gaps)} gap(s) in the liveness polling, worst {worst[0][2]:.0f}s", + subject="control-plane", + detail="; ".join(f"{n} after {t} ({d:.0f}s)" for n, t, d in worst), + evidence={"gaps": [{"log": n, "after": t, "seconds": round(d)} + for n, t, d in gaps]}, + note="Either the control plane stopped asking or the agent stopped answering. " + "A poll that never happened leaves no other trace, so this is the only " + "place the stall is visible — and it is what precedes a node being " + "declared down.", + ) diff --git a/test/framework/sbtest/detectors/fio.py b/test/framework/sbtest/detectors/fio.py new file mode 100644 index 000000000..8f528e805 --- /dev/null +++ b/test/framework/sbtest/detectors/fio.py @@ -0,0 +1,288 @@ +"""fio-side detectors: I/O errors, silent corruption, and sustained outages. + +The checksum detector is the one that decides whether a run lost data, so its attribution +matters as much as its detection — see VERIFY_LAG_S. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from datetime import UTC, datetime, timedelta + +from ..core import Detector, Evidence, Finding, SkipDetector, attribute, critical, detector, warning + +#: How long after a migration ends one of its lost writes may still surface. +#: +#: fio notices a lost write when it next *reads* that block, not when the write was lost, +#: so detection trails the loss by however long the read pattern takes to come round — +#: measured at 3-34s across runs. Attribution has to allow for that: without it, a +#: migration's own losses get filed under "no migration was running", which is exactly how +#: a Completed-but-corrupting migration stayed invisible for a whole analysis pass. +VERIFY_LAG_S = 45.0 + +#: fio's verify failure lines. "bad magic header" is the signature seen in every event so +#: far: a block that never carried an fio header at all, i.e. a *first* write that was lost +#: rather than a stale or torn overwrite. +_VERIFY_RE = re.compile( + r"verify:? (?:bad magic header|header|crc|md5|pattern)|" + r"verify_(?:header|md5|crc)|" + r"got (?:crc|md5) .*expected", + re.IGNORECASE) + +_OFFSET_RE = re.compile(r"offset[= ](\d+)") +#: CRI log prefix: 2026-08-19T22:23:18.994807954Z stderr F +_CRI_TS_RE = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})") + + +def _cri_ts(line: str) -> datetime | None: + m = _CRI_TS_RE.match(line) + if not m: + return None + return datetime.strptime(m.group(1), "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + + +@detector +class JobError(Detector): + """An fio job that ended with an error. + + Reported per pod with the errno, because the errno is the diagnosis and they mean very + different things: 84 is a failed *verification* (the data was wrong), 121 is a path that + went away under the I/O (the data never arrived). + + The numbers below are **Linux** errno values, written out rather than looked up. The + analysis usually runs somewhere other than the host that produced the run, and the errno + table is platform-specific — 84 is EILSEQ on Linux and EOVERFLOW on macOS, so resolving it + locally silently mislabels the most important code in the set. + """ + + name = "fio.job-error" + summary = "an fio job ended with a non-zero errno" + + ERRNO_HINT = { + 84: "EILSEQ — fio's verify failed: a block read back did not match what was written. " + "This is silent data corruption, not an I/O failure; see fio.checksum for the " + "blocks and the migration they belong to", + 121: "EREMOTEIO — every path to the namespace was inaccessible under the I/O; " + "correlate with ana.cutover-pause / ana.freeze-count on the same window", + 5: "EIO — the target failed the command outright rather than losing the path", + 28: "ENOSPC — capacity, not connectivity", + 110: "ETIMEDOUT — the command was accepted and never completed", + 108: "ESHUTDOWN — the target went away mid-flight", + } + + def defaults(self) -> dict: + return {"ignore_errnos": []} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + jobs = ev.fio_jobs() + if not jobs: + raise SkipDetector("no fio job results") + ignore = {int(x) for x in self.opt("ignore_errnos")} + for j in jobs: + if not j.error or j.error in ignore: + continue + yield critical( + self.name, + title=f"fio job ended in error {j.error}", + subject=j.pod, + detail=self.ERRNO_HINT.get(j.error, ""), + evidence={"errno": j.error, "total_iops": j.total_iops}, + ) + + +@detector +class Checksum(Detector): + """fio read back a block that does not match what it wrote — silent corruption. + + The read *succeeded*, so nothing outside fio's own verification notices: no I/O error, + no kernel message, no control-plane event. This is the most serious thing a run can + find, and it is why the attribution window is generous rather than exact — a lost write + that cannot be tied to a migration is much harder to act on than one that can. + """ + + name = "fio.checksum" + summary = "fio read back data it never wrote (silent corruption), attributed to a migration" + + def defaults(self) -> dict: + return {"verify_lag_s": VERIFY_LAG_S} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + pods = ev.pods() + if not pods: + raise SkipDetector("no fio pods") + lag = timedelta(seconds=float(self.opt("verify_lag_s"))) + migs = ev.migrations() + seen_any_log = False + # subject -> list of (pod, ts, line) + per_subject: dict[str, list[tuple[str, datetime | None, str]]] = {} + + for pod in pods: + for line in ev.fio_log(pod): + seen_any_log = True + if not _VERIFY_RE.search(line): + continue + ts = _cri_ts(line) + m = attribute(migs, ts, lag) if ts else None + per_subject.setdefault(m.name if m else "outside-any-migration", []).append( + (pod, ts, line.strip())) + + if not seen_any_log: + raise SkipDetector("no fio pod logs available") + + for subject, hits in sorted(per_subject.items()): + pods_hit: dict[str, int] = {} + offsets = [] + for pod, _ts, line in hits: + pods_hit[pod] = pods_hit.get(pod, 0) + 1 + mo = _OFFSET_RE.search(line) + if mo: + offsets.append(int(mo.group(1))) + mig = next((m for m in migs if m.name == subject), None) + lagged = "" + if mig and mig.end: + late = [t for _p, t, _l in hits if t and t > mig.end] + if late: + worst = max((t - mig.end).total_seconds() for t in late) + lagged = (f"; {len(late)} of them detected up to {worst:.0f}s after the " + f"migration ended, inside fio's verify backlog") + yield critical( + self.name, + title=f"{len(hits)} block(s) read back with the wrong contents", + subject=subject, + detail=("pods: " + ", ".join(f"{p}={n}" for p, n in sorted(pods_hit.items())) + + (f"; phase={mig.phase}" if mig else "") + lagged), + evidence={"blocks": len(hits), "pods": pods_hit, + "offsets": sorted(offsets)[:32], + "phase": mig.phase if mig else "", + "verify_lag_s": float(self.opt("verify_lag_s"))}, + artifacts=[f"{p}/fio.log" for p in sorted(pods_hit)], + note="The reads succeeded, so this is silent: the volume served data that " + "was never written at those offsets. A Completed migration can do " + "this — phase is not a filter.", + ) + + +@detector +class Outage(Detector): + """A sustained window where a pod did no I/O at all. + + Distinct from a checksum failure and from a job error: the I/O neither failed nor + returned wrong data, it stopped. Short dips are normal during a cutover, so only runs + at or above `min_seconds` count. + + Reported per pod rather than per window. A pod that stalls repeatedly stalls *hundreds* of + times in a soak — one run produced 781 qualifying windows at a 10s threshold — and a + report with 781 entries for one symptom is a report nobody reads to the end. The worst + windows are named, the rest are counted, and the total downtime is what the finding leads + with, because "18% of the run did no I/O" is the fact that decides anything. + """ + + name = "fio.outage" + summary = "a pod's I/O stopped for longer than a cutover should cost" + + def defaults(self) -> dict: + return {"min_seconds": 30, "iops_floor": 0.0, "name_worst": 6} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + pods = ev.pods() + if not pods: + raise SkipDetector("no fio pods") + floor = float(self.opt("iops_floor")) + need = int(self.opt("min_seconds")) + show = int(self.opt("name_worst")) + migs = ev.migrations() + saw_series = False + + for pod in pods: + series = ev.fio_timeseries(pod) + if not series: + continue + saw_series = True + windows: list[tuple[int, int, int, str]] = [] # dur, from, to, migration + run_start: int | None = None + prev_off: int | None = None + # None is a sentinel that closes a run still open at the end of the series. + for s in [*series, None]: + down = s is not None and s.total_iops <= floor + if down: + assert s is not None # implied by `down`; stated for the type checker + if run_start is None: + run_start = s.offset_s + elif run_start is not None: + end = s.offset_s if s is not None else (prev_off or run_start) + dur = end - run_start + if dur >= need: + wall = next((x.wall for x in series if x.offset_s == run_start), None) + mig = attribute(migs, wall) if wall else None + windows.append((dur, run_start, end, mig.name if mig else "")) + run_start = None + if s is not None: + prev_off = s.offset_s + + if not windows: + continue + windows.sort(key=lambda w: -w[0]) + downtime = sum(w[0] for w in windows) + span = series[-1].offset_s - series[0].offset_s or 1 + named = ", ".join( + f"{d}s at +{a}s" + (f" ({m})" if m else " (no migration)") + for d, a, _, m in windows[:show]) + during = sorted({w[3] for w in windows if w[3]}) + yield critical( + self.name, + title=(f"I/O stopped {len(windows)}x for {downtime}s total " + f"({downtime / span:.0%} of the run), worst {windows[0][0]}s"), + subject=pod, + detail=named + (f", and {len(windows) - show} more" if len(windows) > show + else ""), + evidence={"windows": len(windows), "downtime_s": downtime, + "worst_s": windows[0][0], + "fraction_of_run": round(downtime / span, 4), + "migrations": during, + "worst_windows": [{"seconds": d, "from_offset_s": a, + "to_offset_s": b, "migration": m} + for d, a, b, m in windows[:show]]}, + note=("Windows are attributed to the migration in flight when there was one; " + f"{len(during)} migration(s) are implicated here." + if during else + "None of these windows fall inside a migration, so the cause is " + "elsewhere — check the fabric and the node's own health."), + ) + if not saw_series: + raise SkipDetector("no fio time series available") + + +@detector +class Throughput(Detector): + """A pod whose average IOPS is far below its peers. + + A weak signal on its own, which is why it is a WARNING: a pod on a shared subsystem that + is being migrated repeatedly will legitimately lag. It earns its place next to the other + detectors — a pod that is both slow and the subject of an ANA finding is a different + story from one that is merely slow. + """ + + name = "fio.throughput-outlier" + summary = "a pod whose average IOPS is far below the run's median" + + def defaults(self) -> dict: + return {"min_fraction_of_median": 0.5, "min_pods": 4} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + jobs = [j for j in ev.fio_jobs() if j.total_iops > 0] + if len(jobs) < int(self.opt("min_pods")): + raise SkipDetector(f"needs at least {self.opt('min_pods')} pods with I/O, " + f"got {len(jobs)}") + vals = sorted(j.total_iops for j in jobs) + median = vals[len(vals) // 2] + frac = float(self.opt("min_fraction_of_median")) + for j in jobs: + if j.total_iops < median * frac: + yield warning( + self.name, + title=f"{j.total_iops:.0f} IOPS against a median of {median:.0f}", + subject=j.pod, + evidence={"total_iops": j.total_iops, "median_iops": median, + "fraction": round(j.total_iops / median, 3)}, + ) diff --git a/test/framework/sbtest/detectors/kernel.py b/test/framework/sbtest/detectors/kernel.py new file mode 100644 index 000000000..4ee85370e --- /dev/null +++ b/test/framework/sbtest/detectors/kernel.py @@ -0,0 +1,637 @@ +"""Kernel-side detectors, from dmesg. + +dmesg is the only source that says what the *host* did about a fabric event, and it turns +out to carry things nothing else does. + +The headline is a severity ladder. When every path to a namespace goes away, the kernel does +not simply fail I/O — it queues, waits, and only fails once a timeout expires: + + all paths inaccessible + -> "block nvmeXnY: no usable path - requeuing I/O" queued; the application waits + -> "nvme nvmeN: failfast expired" fast_io_fail_tmo elapsed + -> "block nvmeXnY: no available path - failing I/O" errors reach the application + -> "XFS (nvmeXnY): log I/O error -5" + -> "XFS (nvmeXnY): Filesystem has been shut down" the volume needs repair + +Every stage was observed across the archived runs, and the ladder is monotone in severity: +one run reached only the first rung (163 requeues, no failures, no filesystem damage) while +a later one reached the last (221 requeues, 30 failures, 20 filesystem shutdowns). That makes +the rung reached a much better statement of blast radius than any count of ANA samples, and +it identifies the knob that decides it: **fast_io_fail_tmo**. A pause shorter than it is +absorbed; a pause longer than it becomes application-visible I/O errors and then filesystem +damage. + +The second thing dmesg alone shows is that leaked controllers **outlive their cluster**. In +one run, 90% of the kernel's NVMe traffic was controllers retrying subsystems belonging to +two clusters that had already been destroyed and reinstalled. An NQN names its cluster, so +that is decidable without any threshold — see ForeignCluster. + +**Everything here is bounded to the run's window**, and that is not a detail. dmesg is a ring +buffer covering hours, so it routinely holds the previous runs' damage and a cluster's worth +of leftover debris. Counting that against this run means a clean run inherits its +predecessor's mess — and, worse, a genuinely broken run hides inside inherited noise. Events +dated before the run are reported separately as Attribution.PRE_EXISTING, which makes the +verdict INCONCLUSIVE rather than FAIL: the action is to clean the environment and run again. + +Timestamps are read from either `--time-format=iso` (which carries an offset, so the +comparison against a UTC run window is sound) or `-T` (local time, no offset — assumed to be +the run's own timezone, and flagged as such). Events with no usable timestamp are +Attribution.UNKNOWN and still count, because "I cannot date this" must not become "not our +problem". Individual migrations are never blamed: the window is the run, not the migration. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Callable, Iterable, Iterator +from datetime import UTC, datetime + +from ..core import ( + Attribution, + Detector, + Evidence, + Finding, + Severity, + SkipDetector, + critical, + detector, + info, + warning, +) + +#: `[Thu Aug 20 05:46:57 2026] ...` — dmesg -T, host-local, no offset. +_TS_CTIME = re.compile(r"^\[(\w{3} \w{3}\s+\d+ \d{2}:\d{2}:\d{2} \d{4})\]\s*(.*)$") +#: `2026-08-20T05:46:57,123456+00:00 ...` — dmesg --time-format=iso, with an offset. +_TS_ISO = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})[,.]\d+([+-]\d{2}:?\d{2})?\s*(.*)$") + +#: Which logs these detectors read. dmesg-.txt by convention. +DEFAULT_LOG_GLOBS = ["dmesg-*"] + + +def _parse_line(raw: str) -> tuple[datetime | None, str]: + """(timestamp, message) from a dmesg line in either format.""" + line = raw.rstrip("\n") + m = _TS_ISO.match(line) + if m: + stamp, offset, msg = m.group(1), m.group(2), m.group(3) + try: + return datetime.fromisoformat(stamp + (offset or "+00:00")), msg + except ValueError: + return None, msg + m = _TS_CTIME.match(line) + if m: + try: + # No offset in this format. Treated as UTC, which is what these hosts run and + # what makes the window comparison meaningful; a host in another zone shifts + # events across the boundary, which is why the ISO form is collected now. + return datetime.strptime(m.group(1), "%a %b %d %H:%M:%S %Y").replace( + tzinfo=UTC), m.group(2) + except ValueError: + return None, m.group(2) + return None, line.strip() + + +def _severity(sev: Severity) -> Callable[..., Finding]: + """The finding constructor for a severity, so a detector can vary it by attribution. + + Which it must: the same observation is a defect when the run caused it and a hygiene note + when the run inherited it, and that difference is severity, not wording. + """ + return {Severity.CRITICAL: critical, Severity.WARNING: warning}.get(sev, info) + + +class _Window: + """Places a dmesg event inside or before the run, and counts each bucket. + + The whole point of this class is that "before the run" is tracked separately rather than + filtered away: inherited damage is worth reporting loudly, just not as this run's fault. + """ + + def __init__(self, ev: Evidence) -> None: + self.start, self.end = ev.run_window() + + def attribute(self, ts: datetime | None) -> Attribution: + if ts is None or self.start is None: + return Attribution.UNKNOWN + return Attribution.PRE_EXISTING if ts < self.start else Attribution.RUN + + @property + def described(self) -> str: + if not self.start: + return "the run window is unknown, so nothing could be dated" + return f"run window {self.start:%b %d %H:%M:%S}..{self.end:%H:%M:%S} UTC" if self.end \ + else f"run started {self.start:%b %d %H:%M:%S} UTC" + + +def _dmesg_lines(ev: Evidence, globs: list[str]) -> Iterator[tuple[str, datetime | None, str]]: + """(log name, timestamp or None, message) for every dmesg line available.""" + names = [n for n in ev.container_logs() if any(fnmatch.fnmatch(n, g) for g in globs)] + for name in names: + for raw in ev.container_log(name): + ts, msg = _parse_line(raw) + yield name, ts, msg + + +def _span(times: list[datetime]) -> str: + if not times: + return "" + lo, hi = min(times), max(times) + return (f"{lo:%b %d %H:%M:%S}" if lo == hi + else f"{lo:%b %d %H:%M:%S}..{hi:%H:%M:%S} host-local") + + +@detector +class PathLoss(Detector): + """How far the kernel got up the path-loss ladder, per namespace device. + + Reports the worst rung reached, because the rungs mean different things: + + * **requeued only** — the pause was absorbed. The application saw latency, not errors. + Worth knowing (it means paths went away at all) but not a failure. + * **failfast expired** — the pause outlasted `fast_io_fail_tmo`. This is the boundary; + past it the kernel stops protecting the application. + * **failing I/O** — errors reached the application. Data loss is now possible and fio + will have noticed too (correlate with fio.job-error / fio.checksum). + + This is stronger evidence than ANA sampling for the same event: it is what the kernel + actually did, not what a sampler happened to catch, so it cannot miss a window shorter + than the sampling interval. + """ + + name = "kernel.path-loss" + summary = "how far the kernel got up the path-loss ladder (requeue -> failfast -> fail I/O)" + + RE_REQUEUE = re.compile(r"block (\S+): no usable path - requeuing I/O") + RE_FAILING = re.compile(r"block (\S+): no available path - failing I/O") + RE_FAILFAST = re.compile(r"(nvme\d+): failfast expired") + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, + # Requeues are expected during a cutover; this bounds how many is normal. + "max_requeues": 0, + # Any failed I/O is a failure — the application saw an error. + "max_failing": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + win = _Window(ev) + # kind -> attribution -> device -> times + seen: dict[str, dict[Attribution, dict[str, list[datetime]]]] = {} + saw_any = False + + for _log, ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + for rx, kind in ((self.RE_FAILING, "failing"), (self.RE_REQUEUE, "requeue"), + (self.RE_FAILFAST, "failfast")): + m = rx.search(msg) + if not m: + continue + bucket = seen.setdefault(kind, {}).setdefault(win.attribute(ts), {}) + stamps = bucket.setdefault(m.group(1), []) + if ts: + stamps.append(ts) + break + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + def count(kind: str, attr: Attribution) -> int: + return sum(len(v) or 1 for v in seen.get(kind, {}).get(attr, {}).values()) + + for attr in (Attribution.RUN, Attribution.UNKNOWN, Attribution.PRE_EXISTING): + devs_fail = seen.get("failing", {}).get(attr, {}) + n_fail = count("failing", attr) + n_req = count("requeue", attr) + n_ff = count("failfast", attr) + if not (devs_fail or n_req or n_ff): + continue + inherited = attr is Attribution.PRE_EXISTING + times = [t for d in seen.values() for v in d.get(attr, {}).values() for t in v] + + if devs_fail and n_fail > int(self.opt("max_failing")): + # Pre-existing damage is reported but never fails the run: I/O the kernel + # gave up on before this run started is the previous run's story. + make = _severity(Severity.WARNING if inherited else Severity.CRITICAL) + yield make( + self.name, + title=(f"the kernel failed I/O on {len(devs_fail)} device(s) for want of " + f"a path" + (" before this run began" if inherited else "")), + subject="fabric", + detail=(f"{n_fail} occurrence(s) on {', '.join(sorted(devs_fail))}" + + (f"; {_span(times)}" if times else "") + + f"; {win.described}"), + evidence={"failing_io": n_fail, "devices": sorted(devs_fail), + "requeues": n_req, "failfast": n_ff, + "attribution": str(attr)}, + artifacts=sorted(set(self.opt("logs"))), + attribution=attr, + note=("Inherited from before the run — worth cleaning up, but it says " + "nothing about this run's code." + if inherited else + "This is the rung past which the kernel stops protecting the " + "application: queued I/O is given up and the error returned. " + "Filesystem damage follows — see kernel.filesystem-shutdown — and " + "fio will have seen it too."), + ) + + if n_ff: + yield _severity(Severity.INFO if inherited else Severity.WARNING)( + self.name, + title=(f"fast_io_fail_tmo expired {n_ff} time(s)" + + (" before this run began" if inherited else "")), + subject="fabric", + detail=f"{len(seen.get('failfast', {}).get(attr, {}))} controller(s)" + + (f"; {_span(times)}" if times else ""), + evidence={"failfast": n_ff, "attribution": str(attr)}, + attribution=attr, + note=("" if inherited else + "The outage outlasted fast_io_fail_tmo, the knob that decides " + "whether an all-paths-inaccessible window is absorbed or becomes " + "application-visible. A cutover pause longer than this value " + "cannot be survived by queueing."), + ) + + if n_req > int(self.opt("max_requeues")): + yield _severity(Severity.INFO if inherited else Severity.WARNING)( + self.name, + title=(f"the kernel had no usable path on " + f"{len(seen.get('requeue', {}).get(attr, {}))} device(s)" + + (" before this run began" if inherited else "")), + subject="fabric", + detail=f"{n_req} requeue(s)" + (f"; {_span(times)}" if times else ""), + evidence={"requeues": n_req, "attribution": str(attr)}, + attribution=attr, + note=("" if inherited else + "I/O was queued rather than failed, so the application survived — " + "but every path to these namespaces was gone. This is the kernel's " + "own record of the window ana.freeze-count samples for, and it " + "cannot miss one shorter than the sampling interval."), + ) + + +@detector +class FilesystemShutdown(Detector): + """A filesystem that took itself offline because its log I/O failed. + + The end of the path-loss ladder, and the most consequential thing a run can leave behind: + the volume is unusable until it is unmounted and repaired, which no amount of retrying + fixes. Separate from kernel.path-loss because it is the *consequence* — someone asking + "did this run damage a filesystem" wants exactly this and nothing else. + """ + + name = "kernel.filesystem-shutdown" + summary = "XFS/ext4 shut down or went read-only after failed log I/O" + + PATTERNS = ( + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*(?:has been shut down|Filesystem has been shut down)"), + "filesystem shut down"), + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*log I/O error"), "log I/O error"), + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*(?:Remounting filesystem read-only|" + r"remounting filesystem read-only)"), "remounted read-only"), + (re.compile(r"(XFS|EXT4-fs) \(([^)]+)\).*metadata I/O error"), "metadata I/O error"), + ) + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + win = _Window(ev) + # attribution -> device -> kinds -> times + hits: dict[Attribution, dict[str, dict[str, list[datetime]]]] = {} + saw_any = False + for _log, ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + for rx, kind in self.PATTERNS: + m = rx.search(msg) + if m: + slot = hits.setdefault(win.attribute(ts), {}).setdefault( + m.group(2), {}).setdefault(kind, []) + if ts: + slot.append(ts) + break + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + for attr, devices in hits.items(): + inherited = attr is Attribution.PRE_EXISTING + shutdown = sorted(d for d, k in devices.items() if "filesystem shut down" in k) + times = [t for k in devices.values() for v in k.values() for t in v] + if shutdown: + # Pre-existing damage is a hygiene warning, not this run's failure: a + # filesystem that died before the run started is the previous run's evidence. + # It only invalidates *this* run if the run went on to use that volume, which + # dmesg alone cannot establish — nvme.dirty-start is the check that can. + yield _severity(Severity.WARNING if inherited else Severity.CRITICAL)( + self.name, + title=(f"{len(shutdown)} filesystem(s) shut down after failed log I/O" + + (" before this run began" if inherited else "")), + subject="fabric", + detail=(", ".join(shutdown) + (f"; {_span(times)}" if times else "") + + f"; {win.described}"), + evidence={"devices": shutdown, "attribution": str(attr), + "kinds": {d: sorted(k) for d, k in sorted(devices.items())}}, + attribution=attr, + note=("Left over from earlier activity on these hosts. Worth cleaning up " + "— the volume needs unmount and repair — but not evidence about " + "this run." + if inherited else + "The volume is unusable until it is unmounted and repaired; " + "retrying does not recover it. This is the end of the path-loss " + "ladder — see kernel.path-loss for how it got here."), + ) + elif devices: + yield _severity(Severity.INFO if inherited else Severity.WARNING)( + self.name, + title=(f"filesystem I/O errors on {len(devices)} device(s), no shutdown" + + (" (before this run)" if inherited else "")), + subject="fabric", + detail=", ".join(f"{d}: {', '.join(sorted(k))}" + for d, k in sorted(devices.items())), + evidence={"kinds": {d: sorted(k) for d, k in sorted(devices.items())}, + "attribution": str(attr)}, + attribution=attr, + ) + + +@detector +class ForeignCluster(Detector): + """A controller retrying a subsystem that belongs to a cluster which no longer exists. + + An NQN names its cluster, so this needs no threshold and no topology knowledge: if the + run is against cluster A and the kernel is retrying NQNs for cluster B, those controllers + are leaked, full stop. + + This is the sharpest form of "controllers not disappearing". Across the archived runs the + leak survived not just the migration that made it but the cluster's *destruction and + reinstallation*: in one run, 90% of the kernel's NVMe log traffic was retries against two + clusters that had already been torn down. Nothing on the host ever removes them, and + because the kernel resets its reconnect counter on a partial reconnect, they are + effectively immortal until the node reboots. + + **Reported as hygiene, never as a run failure.** By construction these belong to a cluster + that no longer exists, so they cannot make a migration of *this* cluster's subsystems fail + — they are noise in the logs and wasted work on the host, which is a real thing to fix and + a bad thing to fail a build over. The leak that does invalidate a run is a leak on the + *live* cluster's subsystems: see nvme.dirty-start. + """ + + name = "nvme.foreign-cluster" + summary = "controllers retrying subsystems of a cluster that is no longer the live one" + + RE_NQN_CLUSTER = re.compile(r"simplyblock:([0-9a-f]{8}-[0-9a-f-]{27}):") + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, "max_lines": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + current = ev.cluster_uuid() + if not current: + raise SkipDetector("the run's cluster uuid is unknown, so a foreign NQN " + "cannot be told from the live one") + per_cluster: dict[str, int] = {} + saw_any = False + for _log, _ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + m = self.RE_NQN_CLUSTER.search(msg) + if m: + per_cluster[m.group(1)] = per_cluster.get(m.group(1), 0) + 1 + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + foreign = {c: n for c, n in per_cluster.items() if c != current} + if not foreign or sum(foreign.values()) <= int(self.opt("max_lines")): + return + total = sum(per_cluster.values()) or 1 + share = sum(foreign.values()) / total + yield warning( + self.name, + title=f"{len(foreign)} dead cluster(s) still being retried by this host", + subject="fabric", + detail=("; ".join(f"{c[:8]}={n} lines" for c, n in + sorted(foreign.items(), key=lambda x: -x[1])) + + f"; live cluster {current[:8]}={per_cluster.get(current, 0)} " + f"({share:.0%} of NVMe log traffic is for dead clusters)"), + evidence={"live_cluster": current, "foreign": foreign, + "foreign_share": round(share, 3)}, + attribution=Attribution.PRE_EXISTING, + note="These controllers outlived the cluster they belong to, so nothing will " + "ever answer them. They cost a reconnect every ctrl_loss_tmo window and " + "bury the live cluster's messages in the log, but they cannot affect a " + "migration of this cluster's subsystems — so this is hygiene, not a " + "verdict. Only a disconnect or a node reboot clears them.", + ) + + +@detector +class ControllerChurn(Detector): + """Controllers created versus removed — "they never disappear", counted. + + A run that connects paths and tears them down again nets out. One that leaks shows a + positive delta, and a reconnect storm shows as a retry count out of all proportion to the + number of controllers involved. + + Both halves are needed. The delta alone misses a leak whose controller was created before + the dmesg ring wrapped; the retry rate alone cannot say whether the retries belong to one + doomed controller or fifty healthy ones. + """ + + name = "nvme.controller-churn" + summary = "controllers created vs removed, and how hard the survivors are retrying" + + RE_NEW = re.compile(r"nvme(\d+): new ctrl|nvme(\d+): creating \d+ I/O queues") + RE_GONE = re.compile(r"nvme(\d+): Removing ctrl") + RE_RETRY = re.compile(r"nvme(\d+): Failed reconnect attempt (\d+)") + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, + # A positive delta this large means controllers are accumulating. + "max_net_created": 4, + # A single controller retrying this many times is not coming back. + "max_retries_per_controller": 100} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + created: set[str] = set() + removed: set[str] = set() + worst_retry: dict[str, int] = {} + saw_any = False + for _log, _ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + m = self.RE_NEW.search(msg) + if m: + created.add(m.group(1) or m.group(2)) + m = self.RE_GONE.search(msg) + if m: + removed.add(m.group(1)) + m = self.RE_RETRY.search(msg) + if m: + ctrl, n = m.group(1), int(m.group(2)) + worst_retry[ctrl] = max(worst_retry.get(ctrl, 0), n) + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + net = len(created - removed) + if net > int(self.opt("max_net_created")): + yield warning( + self.name, + title=f"{net} controller(s) created and never removed", + subject="fabric", + detail=f"created={len(created)} removed={len(removed)}; " + f"never removed: {', '.join(sorted(created - removed)[:16])}", + evidence={"created": len(created), "removed": len(removed), "net": net, + "never_removed": sorted(created - removed)}, + note="dmesg is a ring buffer, so a controller created before it wrapped will " + "look un-created rather than un-removed; the delta is a floor on the " + "leak, not the whole of it.", + ) + + stuck = {c: n for c, n in worst_retry.items() + if n > int(self.opt("max_retries_per_controller"))} + if stuck: + yield warning( + self.name, + title=f"{len(stuck)} controller(s) retrying without ever succeeding", + subject="fabric", + detail="; ".join(f"nvme{c}={n} attempts" for c, n in + sorted(stuck.items(), key=lambda x: -x[1])[:12]), + evidence={"controllers": {f"nvme{c}": n for c, n in sorted(stuck.items())}, + "threshold": int(self.opt("max_retries_per_controller"))}, + note="The reconnect counter resets on a partial reconnect, so a high value " + "means the target is actively refusing this host rather than being " + "briefly unreachable. Check nvme.foreign-cluster: the usual cause is a " + "controller whose subsystem no longer exists.", + ) + if not stuck and net <= int(self.opt("max_net_created")): + yield info(self.name, title=f"{len(created)} created / {len(removed)} removed", + subject="fabric", + evidence={"created": len(created), "removed": len(removed)}) + + +@detector +class FabricErrors(Detector): + """Fabric-level errors that indicate instability short of losing a path. + + None of these is a failure on its own — they are the texture around one, and they are + worth surfacing because they say *where* the fabric hurt: a socket that will not + establish is a different problem from a controller that connects and then cannot be + configured. + """ + + name = "kernel.fabric-errors" + summary = "connect/reset/timeout errors from the NVMe fabric layer" + + PATTERNS = ( + (re.compile(r"starting error recovery"), "error recovery started"), + (re.compile(r"Property Set error"), "property set failed (controller config)"), + (re.compile(r"Identify Descriptors failed"), "identify descriptors failed"), + (re.compile(r"failed to connect socket: -(\d+)"), "socket connect failed"), + (re.compile(r"failed to connect queue: \d+ ret=(\d+)"), "queue connect failed"), + (re.compile(r"queue \d+ socket state (\d+)"), "socket in unexpected state"), + (re.compile(r"I/O \d+ (?:\(\S+\) )?QID \d+ timeout"), "I/O timeout"), + (re.compile(r"Connect Invalid Data Parameter"), "connect refused: no such subsystem"), + (re.compile(r"is not allowed, hostnqn"), "connect refused: host not in allow-list"), + (re.compile(r"rescanning namespaces"), "namespace rescan"), + ) + + def defaults(self) -> dict: + return {"logs": DEFAULT_LOG_GLOBS, + #: kind -> minimum count before it is worth reporting. A rescan or a refused + #: connect happens in ones and twos on any healthy run. + "min_counts": {"namespace rescan": 50, + "connect refused: no such subsystem": 50, + "connect refused: host not in allow-list": 50, + "socket connect failed": 50, + "queue connect failed": 50}, + "default_min_count": 1} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + counts: dict[str, int] = {} + saw_any = False + for _log, _ts, msg in _dmesg_lines(ev, self.opt("logs")): + saw_any = True + for rx, kind in self.PATTERNS: + if rx.search(msg): + counts[kind] = counts.get(kind, 0) + 1 + break + if not saw_any: + raise SkipDetector("no dmesg collected in this run") + + mins = dict(self.opt("min_counts") or {}) + floor = int(self.opt("default_min_count")) + reportable = {k: v for k, v in counts.items() if v >= mins.get(k, floor)} + if not reportable: + return + yield warning( + self.name, + title=f"{sum(reportable.values())} fabric error(s) across {len(reportable)} kind(s)", + subject="fabric", + detail="; ".join(f"{k}={v}" for k, v in + sorted(reportable.items(), key=lambda x: -x[1])), + evidence={"counts": reportable, "all_counts": counts}, + note="Texture rather than a verdict, and spanning whatever the ring buffer held " + "rather than only this run: read it next to kernel.path-loss to see whether " + "the fabric merely wobbled or actually lost a path.", + ) + + +@detector +class DirtyStart(Detector): + """The fabric was already carrying blocking debris for the **live** cluster at setup. + + This is the one pre-existing condition that forfeits a run, and it is worth being precise + about why. A controller that is live and serves no namespace makes VerifyMigrationPaths + reject the migration of its subsystem — permanently, because nothing removes it. If that + is already true when the run starts, then migrations that should have passed will fail, + and the run's completion rate measures the mess it inherited rather than the code under + test. A 13-of-46 result says nothing in that state. + + Everything about the qualification matters: + + * **live cluster only.** Debris for a destroyed cluster cannot block this cluster's + subsystems; that is nvme.foreign-cluster's hygiene warning, not a forfeit. + * **at setup, not at the end.** Debris the run *created* is the run's own finding — see + nvme.stale-controllers — and failing it is correct. + * **blocking shapes only.** A controller stuck in "connecting" is indistinguishable from + a normal HA reconnect in a snapshot, so it does not qualify. + + Requires the nvme.snapshot component's pre-run snapshot; without it there is nothing to + judge and the detector says so. + """ + + name = "nvme.dirty-start" + summary = ("the fabric already held blocking debris for the live cluster when the run " + "started, so the run's results cannot be trusted") + + def defaults(self) -> dict: + return {"max_blocking": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + pre = getattr(ev, "nvme_controllers_pre", None) + ctrls = pre() if callable(pre) else [] + if not ctrls: + raise SkipDetector("no pre-run fabric snapshot (enable the nvme.snapshot " + "component to judge the state the run started from)") + cluster = ev.cluster_uuid() + if not cluster: + raise SkipDetector("the run's cluster uuid is unknown, so debris for it cannot " + "be told from debris for a dead one") + + blocking = [c for c in ctrls + if cluster in c.nqn and c.state == "live" and c.serves_nothing] + if len(blocking) <= int(self.opt("max_blocking")): + return + by_node: dict[str, list[str]] = {} + for c in blocking: + by_node.setdefault(c.node, []).append(f"{c.name}@{c.address}") + yield critical( + self.name, + title=f"{len(blocking)} blocking controller(s) for the live cluster before the " + f"run started", + subject="environment", + detail="; ".join(f"{n}: {', '.join(sorted(v))}" for n, v in sorted(by_node.items())), + evidence={"count": len(blocking), "live_cluster": cluster, + "per_node": {n: sorted(v) for n, v in by_node.items()}}, + attribution=Attribution.PRE_EXISTING, + note="Each of these makes the pre-cutover path check reject its subsystem, so " + "migrations that should pass will fail and the completion rate measures the " + "inherited mess rather than the code. Clear the fabric and run again; do " + "not read this run's migration results.", + ) diff --git a/test/framework/sbtest/detectors/logs.py b/test/framework/sbtest/detectors/logs.py new file mode 100644 index 000000000..8a5b121ae --- /dev/null +++ b/test/framework/sbtest/detectors/logs.py @@ -0,0 +1,206 @@ +"""Log-pattern detectors — user-definable checks over any collected log. + +This is the extension point for "whatever the user wants". A pattern entry is data, so a +new check costs a config block rather than code: + + detectors: + logs.pattern: + patterns: + - id: spdk.undrained-transfer + regex: "still have outstanding io" + logs: ["spdk-*"] + severity: critical + min_count: 1 + note: "the batch transfer started before in-flight I/O was drained" + +The bundled catalogue below is what past runs turned out to care about, and it doubles as +worked examples. Everything in it is overridable: `patterns` replaces the catalogue, +`extra_patterns` appends to it. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any + +from ..core import Detector, Evidence, Finding, Severity, SkipDetector, detector + + +@dataclass +class Pattern: + """One log check. `id` is the finding subject, so keep it stable and dotted.""" + + id: str + regex: str + #: Glob(s) over the available log names. "spdk-*" covers every storage node. + logs: list[str] = field(default_factory=lambda: ["*"]) + severity: str = "warning" + #: Fire only at or above this many matches. The default of 1 suits a line that should + #: never appear; raise it for a line that is normal in small numbers. + min_count: int = 1 + note: str = "" + #: Cap on example lines kept in the finding, to keep reports readable. + examples: int = 3 + + def compiled(self) -> re.Pattern: + return re.compile(self.regex) + + def matches_log(self, name: str) -> bool: + return any(fnmatch.fnmatch(name, g) for g in self.logs) + + +#: Patterns worth having by default. Each one was the visible symptom of a real defect. +CATALOGUE: list[Pattern] = [ + Pattern( + id="spdk.undrained-transfer", + regex=r"still have outstanding io|Task transfer timeout with outstanding", + logs=["spdk-4*", "spdk-*"], + severity="critical", + note="The batch delta copy started while I/O was still in flight, so the transfer " + "failed and the control plane retried it. Each retry replays a non-idempotent " + "step against a source that has been serving writes in between — this is the " + "line that accompanies silent write loss. Correlate with ana.freeze-count.", + ), + Pattern( + id="spdk.migration-subtask-failed", + regex=r"Sub task failed for migration of lvol", + logs=["spdk-*"], + severity="critical", + note="One member of a batch migration failed its transfer; the group is retried or " + "abandoned as a whole.", + ), + Pattern( + id="nvme.host-not-allowed", + regex=r"does not allow host .* to connect at this address", + logs=["spdk-*"], + severity="warning", + min_count=50, + note="A host is retrying an endpoint whose allow-list no longer includes it — the " + "signature of a leaked controller nobody disconnected. Volume, not presence, " + "is the signal: a steady rate means a reconnect storm.", + ), + Pattern( + id="nvme.write-to-readonly", + regex=r"WRITE TO RO RANGE", + logs=["spdk-*"], + severity="warning", + min_count=1, + note="A write reached a range the target considers read-only, which during a " + "migration means it landed on a copy that was already frozen.", + ), + Pattern( + id="operator.path-validation-failed", + regex=r"NVMe path validation failed", + logs=["operator*"], + severity="warning", + note="The pre-cutover check refused a migration. Persistent failures for one " + "subsystem usually mean a leaked controller the check keeps reporting — see " + "nvme.stale-controllers.", + ), + Pattern( + id="operator.migration-group-stuck", + regex=r"is already past pre-create|is not active \(status=cancelled\)", + logs=["operator*"], + severity="warning", + note="The operator and the control plane disagree about a migration group's state; " + "the group needs /continue or an explicit cancel.", + ), + Pattern( + id="kernel.controller-reconnect-loop", + regex=r"Failed reconnect attempt (\d{3,})", + logs=["dmesg-*"], + severity="warning", + note="A controller has retried hundreds of times. The counter resets on a partial " + "reconnect, so a high number means it is being refused rather than merely " + "waiting.", + ), +] + + +@detector +class LogPattern(Detector): + """Count regex hits across collected logs and report the ones over threshold. + + One pass per log, all patterns evaluated per line, because these logs are tens of + megabytes and reading them once per pattern is what makes the difference between a check + that runs and one that gets disabled. + """ + + name = "logs.pattern" + summary = "configurable regex checks over any collected log (ships a catalogue)" + + def defaults(self) -> dict: + return {"patterns": None, "extra_patterns": [], "logs": None} + + def _patterns(self) -> list[Pattern]: + given = self.opt("patterns") + pats = [self._as_pattern(p) for p in given] if given is not None else list(CATALOGUE) + pats += [self._as_pattern(p) for p in (self.opt("extra_patterns") or [])] + return pats + + @staticmethod + def _as_pattern(p: Any) -> Pattern: + if isinstance(p, Pattern): + return p + if not isinstance(p, dict): + raise ValueError(f"pattern must be a mapping, got {type(p).__name__}") + unknown = set(p) - set(Pattern.__dataclass_fields__) + if unknown: + raise ValueError(f"pattern {p.get('id', '?')!r}: unknown key(s) {sorted(unknown)}") + return Pattern(**p) + + def detect(self, ev: Evidence) -> Iterable[Finding]: + available = ev.container_logs() + if not available: + raise SkipDetector("no container logs collected in this run") + pats = self._patterns() + only = self.opt("logs") + if only: + available = [n for n in available if any(fnmatch.fnmatch(n, g) for g in only)] + + # (pattern id, log name) -> [count, examples] + hits: dict[tuple[str, str], list] = {} + for log_name in available: + active = [(p, p.compiled()) for p in pats if p.matches_log(log_name)] + if not active: + continue + for line in ev.container_log(log_name): + for p, rx in active: + if rx.search(line): + slot = hits.setdefault((p.id, log_name), [0, []]) + slot[0] += 1 + if len(slot[1]) < p.examples: + slot[1].append(line.strip()[:220]) + + by_id = {p.id: p for p in pats} + # Aggregate across logs per pattern, but keep the per-log breakdown: "which node" + # is usually the first question, and for the undrained-transfer line it was the + # answer (the node stuck in the retry loop). + per_pattern: dict[str, dict[str, int]] = {} + examples: dict[str, list[str]] = {} + for (pid, log_name), (count, ex) in hits.items(): + per_pattern.setdefault(pid, {})[log_name] = count + examples.setdefault(pid, []).extend(ex) + + for pid, counts in sorted(per_pattern.items()): + p = by_id[pid] + total = sum(counts.values()) + if total < p.min_count: + continue + sev = {"critical": Severity.CRITICAL, "warning": Severity.WARNING}.get( + p.severity.lower(), Severity.INFO) + yield Finding( + detector=self.name, + severity=sev, + title=f"{total} match(es) for {pid}", + subject=pid, + detail="; ".join(f"{k}={v}" for k, v in sorted(counts.items(), key=lambda x: -x[1])) + + ("\n" + "\n".join(examples.get(pid, [])[:p.examples]) if examples.get(pid) else ""), + evidence={"pattern": p.regex, "total": total, "per_log": counts, + "min_count": p.min_count}, + artifacts=[f"{k}.txt" for k in sorted(counts)], + note=p.note, + ) diff --git a/test/framework/sbtest/detectors/meta.py b/test/framework/sbtest/detectors/meta.py new file mode 100644 index 000000000..f74f8dfd8 --- /dev/null +++ b/test/framework/sbtest/detectors/meta.py @@ -0,0 +1,190 @@ +"""Meta-detectors: checks on the evidence itself. + +These answer "can this run be judged at all?", which is a different question from "did it +pass" and one that nothing else asks. They exist because the alternative kept happening: a +log-based verdict was drawn over a log that covered a fraction of the run, and nothing in the +report said so. + +The framework already distinguishes a detector that *skipped* from one that found nothing. +This module extends that to partial evidence, which is the harder and more common case — a log +that exists, opens fine, and is missing the two hours you needed. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, detector, info, warning + + +@detector +class LogCoverage(Detector): + """A collected log that does not span the run. + + Container logs are rotated by the kubelet against a fixed byte budget, so a busy + component's log covers the last N minutes rather than the run. That is survivable if you + know it and misleading if you do not: every `logs.pattern` count, every kernel finding and + every manual grep is silently scoped to whatever survived. + + Reported as a WARNING rather than a failure — incomplete evidence is not a defect in the + system under test — but reported *loudly*, because it bounds what the rest of the report + is allowed to claim. + """ + + name = "evidence.log-coverage" + summary = "a collected log that does not cover the whole run, bounding what can be claimed" + + def defaults(self) -> dict: + return { + # Below this, a gap is startup jitter rather than rotation. + "min_gap_s": 60.0, + # Logs that legitimately begin mid-run: a component created by the run itself has + # nothing to say before it existed. + "ignore": [], + } + + def detect(self, ev: Evidence) -> Iterable[Finding]: + spans = ev.log_spans() + if not spans: + raise SkipDetector("no collected logs to measure") + start, end = ev.run_window() + if not start: + raise SkipDetector("the run window is unknown, so coverage cannot be measured") + + min_gap = float(self.opt("min_gap_s")) + ignore = set(self.opt("ignore") or []) + short: list[tuple[str, float, float]] = [] # name, missing-at-start, covered fraction + empty: list[str] = [] + total = (end - start).total_seconds() if end else 0.0 + + for sp in spans: + if sp.name in ignore: + continue + if not sp.first or not sp.last: + empty.append(sp.name) + continue + missing = (sp.first - start).total_seconds() + covered = max(0.0, (sp.last - max(sp.first, start)).total_seconds()) + if missing > min_gap: + short.append((sp.name, missing, covered / total if total else 0.0)) + + if short: + short.sort(key=lambda x: -x[1]) + worst = short[0] + yield warning( + self.name, + title=f"{len(short)} log(s) do not cover the start of the run", + subject="evidence", + detail="; ".join(f"{n}: missing first {m / 60:.0f} min" + f"{f', covers {c:.0%}' if c else ''}" for n, m, c in short), + evidence={"logs": {n: {"missing_start_s": round(m), + "covered_fraction": round(c, 3)} + for n, m, c in short}, + "worst": worst[0]}, + note="Log-derived findings are scoped to what survived rotation, so an absence " + "in these files is not evidence of absence. Raise the kubelet's " + "containerLogMaxSize/Files, or follow the log live (logs.stream) for the " + "components that outrun it.", + ) + if empty: + yield warning( + self.name, + title=f"{len(empty)} collected log(s) carry no usable timestamp", + subject="evidence", + detail=", ".join(sorted(empty)), + evidence={"logs": sorted(empty)}, + note="Either the collection produced nothing or the format is unrecognised; " + "either way nothing in these can be placed in time.", + ) + + +@detector +class MigrationBlindSpot(Detector): + """A migration that happened while a log was not covering it. + + The sharper form of the coverage problem, and the one that actually costs time: on the run + this was written for, the migration that silently corrupted data ran 09:28:32-09:31:00 and + one node's SPDK log began at 09:31:06 — six seconds after it finished. There is no + post-mortem to do for that node, and the only thing worse than knowing that is not knowing + it. + + Deliberately per (migration, log): "the run is 80% covered" is useless when the missing + 20% is the interesting part. + """ + + name = "evidence.blind-spot" + summary = "a migration that no log covers, so it cannot be post-mortemed" + + def defaults(self) -> dict: + return {"logs": ["spdk-*", "operator*"], "max_blind": 0} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + import fnmatch + migs = ev.migrations() + spans = [s for s in ev.log_spans() + if any(fnmatch.fnmatch(s.name, g) for g in self.opt("logs"))] + if not migs: + raise SkipDetector("no migrations in this run") + if not spans: + raise SkipDetector("none of the requested logs were collected") + + blind: dict[str, list[str]] = {} + for m in migs: + end = m.end or m.start + for sp in spans: + if not sp.first or not sp.last: + continue + # No overlap at all between the migration and what the log holds. + if sp.first > end or sp.last < m.start: + blind.setdefault(m.name, []).append(sp.name) + + if len(blind) <= int(self.opt("max_blind")): + return + worst = sorted(blind.items(), key=lambda kv: -len(kv[1])) + yield warning( + self.name, + title=f"{len(blind)} migration(s) fall outside at least one log's coverage", + subject="evidence", + detail="; ".join(f"{name}: no {', '.join(sorted(logs))}" + for name, logs in worst[:8]) + + (" ..." if len(worst) > 8 else ""), + evidence={"blind": {k: sorted(v) for k, v in blind.items()}}, + note="These migrations cannot be investigated from those logs whatever they turn " + "out to have done. If one of them also carries a CRITICAL finding, collect " + "again with logs.stream before spending time on the analysis.", + ) + + +@detector +class EvidenceInventory(Detector): + """What evidence this run actually produced. Always INFO; never a verdict. + + Worth a finding of its own because "which of the fourteen checks could even run" is the + first thing anyone asks of a report they did not generate, and reconstructing it from the + skip list is guesswork. + """ + + name = "evidence.inventory" + summary = "what evidence the run produced (informational)" + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + have = { + "migrations": len(migs), + "ana_sampled_migrations": sum(1 for m in migs if ev.ana_samples(m.name)), + "fio_pods": len(ev.pods()), + "fio_jobs": len(ev.fio_jobs()), + "container_logs": len(ev.container_logs()), + "control_events": len(ev.control_events()), + "nvme_controllers": len(ev.nvme_controllers()), + } + start, end = ev.run_window() + yield info( + self.name, + title=", ".join(f"{k}={v}" for k, v in have.items() if v), + subject="evidence", + detail=(f"run window {start:%H:%M:%S}..{end:%H:%M:%S} UTC" + if start and end else "run window unknown"), + evidence={**have, "cluster": ev.cluster_uuid(), + "window_known": bool(start)}, + ) diff --git a/test/framework/sbtest/detectors/migration.py b/test/framework/sbtest/detectors/migration.py new file mode 100644 index 000000000..ba951ed4e --- /dev/null +++ b/test/framework/sbtest/detectors/migration.py @@ -0,0 +1,115 @@ +"""Migration-outcome detectors — the run's success rate, and how it failed. + +Deliberately rate-based rather than per-migration: one failed migration in a long run is +noise, and a run where half of them time out is a different defect from a run where one did. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector, info, warning + + +@detector +class Outcomes(Detector): + """Too few migrations completed, or too many ended one particular way. + + The phase breakdown is the finding, not just the count. A run of 46 migrations with 13 + Completed and 25 TIMEOUT says something quite specific — none of the timeouts ever + reached a cutover — that "28% success" alone does not. + """ + + name = "migration.outcomes" + summary = "completion rate and phase breakdown across the run" + + def defaults(self) -> dict: + return {"min_completed_fraction": 0.8, + "max_timeout_fraction": 0.1, + "completed_phases": ["Completed"], + "timeout_phases": ["TIMEOUT", "Timeout"]} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + total = len(migs) + done = set(self.opt("completed_phases")) + tmo = set(self.opt("timeout_phases")) + + phases: dict[str, int] = {} + for m in migs: + phases[m.phase or "?"] = phases.get(m.phase or "?", 0) + 1 + breakdown = ", ".join(f"{k}={v}" for k, v in sorted(phases.items(), key=lambda x: -x[1])) + + completed = sum(v for k, v in phases.items() if k in done) + timeouts = sum(v for k, v in phases.items() if k in tmo) + frac = completed / total + yield info(self.name, title=f"{completed}/{total} migrations completed", + subject="run", detail=breakdown, + evidence={"total": total, "completed": completed, "phases": phases}) + + if frac < float(self.opt("min_completed_fraction")): + yield critical( + self.name, + title=f"only {frac:.0%} of migrations completed", + subject="run", + detail=breakdown, + evidence={"completed_fraction": round(frac, 3), + "min_completed_fraction": float(self.opt("min_completed_fraction")), + "phases": phases}, + ) + if timeouts / total > float(self.opt("max_timeout_fraction")): + yield critical( + self.name, + title=f"{timeouts}/{total} migrations timed out", + subject="run", + detail=breakdown, + evidence={"timeouts": timeouts, "total": total, + "max_timeout_fraction": float(self.opt("max_timeout_fraction"))}, + note="A timeout that never reached a cutover leaves the volume on the " + "source with the target's objects still allocated; check whether the " + "cutover was attempted at all before blaming the copy.", + ) + + +@detector +class Errors(Detector): + """Distinct migration error messages, grouped. + + Grouping matters more than counting here: 16 identical "NVMe path validation failed" + errors are one defect, and the value is in seeing that they are identical. + """ + + name = "migration.errors" + summary = "distinct migration error messages, grouped by shape" + + def defaults(self) -> dict: + return {"max_distinct": 0} + + @staticmethod + def _shape(err: str) -> str: + import re + s = re.sub(r"[0-9a-f]{8}-[0-9a-f-]{27}", "", err) + s = re.sub(r"\d+", "N", s) + return s[:200] + + def detect(self, ev: Evidence) -> Iterable[Finding]: + migs = ev.migrations() + if not migs: + raise SkipDetector("no migrations in this run") + groups: dict[str, list[str]] = {} + for m in migs: + if m.error: + groups.setdefault(self._shape(m.error), []).append(m.name) + if not groups: + return + for shape, names in sorted(groups.items(), key=lambda x: -len(x[1])): + yield warning( + self.name, + title=f"{len(names)} migration(s) failed with the same error", + subject=shape[:60], + detail=f"{shape}\nmigrations: {', '.join(sorted(names)[:12])}" + + (" ..." if len(names) > 12 else ""), + evidence={"count": len(names), "migrations": sorted(names), "shape": shape}, + ) diff --git a/test/framework/sbtest/detectors/nvme.py b/test/framework/sbtest/detectors/nvme.py new file mode 100644 index 000000000..3c57e4261 --- /dev/null +++ b/test/framework/sbtest/detectors/nvme.py @@ -0,0 +1,140 @@ +"""Host NVMe fabric detectors — leaked and non-contributing controllers. + +These judge the state a run *leaves behind*, which turned out to matter as much as what it +does: a leaked controller is invisible to the run that created it and breaks the next one. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector, warning + + +@detector +class StaleControllers(Detector): + """Controllers that cannot carry I/O and will not recover on their own. + + Two shapes, both from the migration path-leak investigation: + + * **live, serving no namespace** — the admin queue is up and enumeration finished, and + the controller was told about nothing. It looks connected from every angle a connect + checks, so `nvme connect` returns "already connected" and a reconciler that counts + paths sees nothing wrong. This is the state that made a single abandoned migration + block every later migration of the same subsystem, because the pre-cutover check + reports it forever. + * **stuck connecting** — retrying an endpoint that has stopped answering for this host. + Bounded only by ctrl_loss_tmo, and the kernel resets its reconnect counter whenever a + reconnect gets far enough, so "bounded" can mean "until the node reboots". + + Run this at the *end* of a run, and on the next run's setup: it is the check that turns + "the last run left a mess" from a guess into a finding. + """ + + name = "nvme.stale-controllers" + summary = "controllers that are live with no namespace, or stuck connecting" + + def defaults(self) -> dict: + return {"max_zero_namespace": 0, "max_connecting": 0, + # Expected paths per subsystem per host: primary plus HA replicas. Above + # this is leak territory, but the healthy number is topology-dependent, so + # it warns rather than fails. + "expected_paths_per_subsystem": 3} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + ctrls = ev.nvme_controllers() + if not ctrls: + raise SkipDetector("no host NVMe controller snapshot in this run") + + zero = [c for c in ctrls if c.state == "live" and c.serves_nothing] + conn = [c for c in ctrls if c.state == "connecting"] + + if len(zero) > int(self.opt("max_zero_namespace")): + by_node: dict[str, list[str]] = {} + for c in zero: + by_node.setdefault(c.node, []).append(f"{c.name}@{c.address}") + yield critical( + self.name, + title=f"{len(zero)} live controller(s) serving no namespace", + subject="fabric", + detail="; ".join(f"{n}: {', '.join(sorted(v))}" for n, v in sorted(by_node.items())), + evidence={"count": len(zero), + "per_node": {n: sorted(v) for n, v in by_node.items()}}, + note="Each of these makes the pre-cutover path check fail for its " + "subsystem, so they block later migrations until something tears them " + "down. Nothing routes I/O over them, so removing them is safe.", + ) + + if len(conn) > int(self.opt("max_connecting")): + by_node = {} + for c in conn: + by_node.setdefault(c.node, []).append(f"{c.name}@{c.address}") + yield warning( + self.name, + title=f"{len(conn)} controller(s) stuck connecting", + subject="fabric", + detail="; ".join(f"{n}: {', '.join(sorted(v))}" for n, v in sorted(by_node.items())), + evidence={"count": len(conn), + "per_node": {n: sorted(v) for n, v in by_node.items()}, + "ctrl_loss_tmo": sorted({c.ctrl_loss_tmo for c in conn + if c.ctrl_loss_tmo is not None})}, + note="A snapshot cannot tell one of these from a normal HA reconnect, hence " + "a warning. What bounds them is ctrl_loss_tmo — a large value here " + "means they can outlive the run.", + ) + + # Path count per (host, subsystem): the leak seen from a different angle, and the + # one that catches an accumulation whose members all still look individually fine. + limit = int(self.opt("expected_paths_per_subsystem")) + per: dict[tuple[str, str], list[str]] = {} + for c in ctrls: + per.setdefault((c.node, c.nqn), []).append(c.address) + for (node, nqn), addrs in sorted(per.items()): + if len(addrs) > limit: + yield warning( + self.name, + title=f"{len(addrs)} paths to one subsystem on {node} (expected <= {limit})", + subject=f"{node}/{nqn.rsplit(':', 1)[-1]}", + detail=", ".join(sorted(addrs)), + evidence={"node": node, "nqn": nqn, "addresses": sorted(addrs), + "limit": limit}, + ) + + +@detector +class LossTimeout(Detector): + """A controller whose ctrl_loss_tmo lets it outlive the run that created it. + + Worth checking because it is the difference between a leak that expires on its own and + one that has to be cleaned up: the control plane answered migration connects with an + hour, while the CSI driver connects every other path with a minute. A probe path that + may be abandoned should expire quickly. + """ + + name = "nvme.loss-timeout" + summary = "a controller connected with a ctrl_loss_tmo long enough to outlive the run" + + def defaults(self) -> dict: + return {"max_ctrl_loss_tmo_s": 60} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + ctrls = [c for c in ev.nvme_controllers() if c.ctrl_loss_tmo is not None] + if not ctrls: + raise SkipDetector("no controller snapshot with ctrl_loss_tmo") + limit = int(self.opt("max_ctrl_loss_tmo_s")) + bad = [c for c in ctrls + if c.ctrl_loss_tmo is not None and c.ctrl_loss_tmo > limit] + if not bad: + return + vals = sorted({c.ctrl_loss_tmo for c in bad if c.ctrl_loss_tmo is not None}) + yield warning( + self.name, + title=f"{len(bad)} controller(s) with ctrl_loss_tmo above {limit}s", + subject="fabric", + detail=f"values seen: {vals}", + evidence={"count": len(bad), "values": vals, "limit": limit, + "controllers": sorted(f"{c.node}/{c.name}@{c.address}" for c in bad)[:32]}, + note="A path that may be abandoned should expire on its own. The kernel also " + "resets its reconnect counter on a partial reconnect, so a large value is " + "a floor on how long a leak survives, not a bound.", + ) diff --git a/test/framework/sbtest/detectors/security.py b/test/framework/sbtest/detectors/security.py new file mode 100644 index 000000000..ec74b0ecf --- /dev/null +++ b/test/framework/sbtest/detectors/security.py @@ -0,0 +1,83 @@ +"""Security detectors over collected evidence. + +One check for now, and it is the one that keeps being true: artifacts get attached to tickets, +copied into chat and committed to repositories, so a credential that reaches a log reaches all +of those. This is cheap to check and expensive to miss, and it is entirely independent of what +the run was testing. +""" + +from __future__ import annotations + +import fnmatch +import re +from collections.abc import Iterable + +from ..core import Detector, Evidence, Finding, SkipDetector, critical, detector + +#: Shapes worth refusing to ship. Each is deliberately narrow: a pattern that fires on ordinary +#: text gets disabled, and a disabled check finds nothing. +SECRET_PATTERNS: tuple[tuple[str, str], ...] = ( + ("private key block", r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), + ("nvme dhchap secret", r"DHHC-1:[0-9]{2}:[A-Za-z0-9+/=]{20,}"), + ("bearer token", r"[Bb]earer\s+[A-Za-z0-9._-]{24,}"), + ("kubernetes sa token", r"eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\."), + ("aws access key", r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), + ("url with credentials", r"[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s:@]+@"), + ("password assignment", r"(?i)\b(?:password|passwd|secret[_-]?key)\s*[=:]\s*['\"]?[^\s'\"]{8,}"), +) + + +@detector +class SecretExposure(Detector): + """A credential-shaped string in a collected log. + + Reports the pattern, the log and the line number — never the match itself. A finding that + quotes the secret has copied it into a second file, and findings.json travels at least as + widely as the log did. + """ + + name = "security.secret-exposure" + summary = "credential-shaped strings in collected logs (reports location, never the value)" + + def defaults(self) -> dict: + return {"logs": ["*"], "extra_patterns": [], "max_reported": 12} + + def detect(self, ev: Evidence) -> Iterable[Finding]: + names = [n for n in ev.container_logs() + if any(fnmatch.fnmatch(n, g) for g in self.opt("logs"))] + if not names: + raise SkipDetector("no collected logs to scan") + + patterns = [(label, re.compile(rx)) for label, rx in SECRET_PATTERNS] + patterns += [(str(p.get("id", "custom")), re.compile(str(p["regex"]))) + for p in (self.opt("extra_patterns") or [])] + + # label -> log -> [line numbers] + hits: dict[str, dict[str, list[int]]] = {} + for name in names: + for lineno, line in enumerate(ev.container_log(name), start=1): + for label, rx in patterns: + if rx.search(line): + hits.setdefault(label, {}).setdefault(name, []).append(lineno) + if not hits: + return + + cap = int(self.opt("max_reported")) + for label, per_log in sorted(hits.items()): + total = sum(len(v) for v in per_log.values()) + where = "; ".join( + f"{log}:{','.join(str(n) for n in lines[:cap])}" + f"{f' (+{len(lines) - cap} more)' if len(lines) > cap else ''}" + for log, lines in sorted(per_log.items())) + yield critical( + self.name, + title=f"{total} line(s) matching {label}", + subject=label, + detail=where, + evidence={"pattern": label, "total": total, + "per_log": {k: len(v) for k, v in per_log.items()}}, + artifacts=sorted(per_log), + note="The value is deliberately not quoted here — findings.json travels at " + "least as widely as the log. Rotate whatever this is, then stop the " + "component logging it; artifacts get attached to tickets and committed.", + ) diff --git a/test/framework/sbtest/suites/analyze-only.yaml b/test/framework/sbtest/suites/analyze-only.yaml new file mode 100644 index 000000000..b4d1fb562 --- /dev/null +++ b/test/framework/sbtest/suites/analyze-only.yaml @@ -0,0 +1,15 @@ +# Judge an existing artifact directory. No components, so nothing is collected and nothing on +# the cluster is touched — this is what `sbtest analyze` does by default, written out as a +# suite for when it needs to be said explicitly. +# +# make analyze RUN=../../operator/fio-mig-1787211447 SUITE=analyze-only +description: >- + Judge an existing artifact directory. Nothing is collected and nothing is touched. + +# Empty rather than absent: an empty mapping says "no components, deliberately", where a +# missing key would rely on the reader knowing that components default to off. +components: {} + +# Empty means every registered detector runs on its defaults — detectors default to *on*, +# because judging is not something you should have to remember to switch on. +detectors: {} diff --git a/test/framework/sbtest/suites/corruption-hunt.yaml b/test/framework/sbtest/suites/corruption-hunt.yaml new file mode 100644 index 000000000..781aa696c --- /dev/null +++ b/test/framework/sbtest/suites/corruption-hunt.yaml @@ -0,0 +1,48 @@ +# Narrowed to the one question: did this run lose or corrupt data? +# +# Everything that measures *quality* is off, so a FAIL from this suite means data — not +# latency, not a slow cutover, not a timed-out migration. That makes it the suite to bisect +# with, where a verdict has to mean exactly one thing. +# +# make analyze RUN= SUITE=corruption-hunt +description: >- + Narrowed to the data-integrity question, for a bisect or a quick verdict. + +components: + ana.sample: + # 1s rather than 2: the freeze count is only as sharp as the interval measuring it, and + # the count is this suite's primary signal. + interval_s: 1.0 + logs.stream: + containers: + - spdk-container + nvme.snapshot: true + +detectors: + ana.freeze-count: true + ana.split-brain: true + fio.checksum: true + fio.job-error: true + + logs.pattern: + patterns: + # SPDK's own admission that the delta copy began before in-flight I/O had drained. + # This is the root cause the corruption RCA landed on, so it is worth matching + # verbatim rather than inferring from the ANA side-effects. + - id: spdk.undrained-transfer + regex: "still have outstanding io|Task transfer timeout with outstanding" + logs: ["spdk-*"] + severity: critical + note: the delta copy started before in-flight I/O was drained + + # Off: these measure how *well* it went, not whether data survived. Keeping them on would + # make a bisect ambiguous — a slow cutover would fail the same run a lost write does. + ana.cutover-pause: false + ana.path-churn: false + ana.unserved-after-cutover: false + fio.outage: false + fio.throughput-outlier: false + migration.outcomes: false + migration.errors: false + nvme.stale-controllers: false + nvme.loss-timeout: false diff --git a/test/framework/sbtest/suites/migration-full.yaml b/test/framework/sbtest/suites/migration-full.yaml new file mode 100644 index 000000000..9233cd954 --- /dev/null +++ b/test/framework/sbtest/suites/migration-full.yaml @@ -0,0 +1,105 @@ +# The full driven migration run: the sbtest equivalent of +# +# fio_migration_test.py --keep --pods 0 --runtime 7200 --ns-pods 25 \ +# --ns-per-subsys 6 --snapshot-chance 0 --target-policy alternate +# +# make run SUITE=migration-full DURATION=7200 KEEP=1 +description: >- + The full driven migration run: 25 namespaced volumes packed 6 to a subsystem, so every + migration is a batch migration — the case that corrupts. + +run: + id_prefix: fiomig + outdir: ./runs + +components: + workload.fio: + # pods=0: the single-namespace path is not what is under investigation. Raise it to + # compare single against batch in one run — they share the geometry, so the comparison + # is meaningful only if both classes format their filesystems identically, which is why + # the workload derives both from the same base parameters. + pods: 0 + ns_pods: 25 + ns_per_subsys: 6 + volume_size_gb: 10 + # Small on purpose: fio always lays out a file-backed target before random I/O and + # cannot be told to skip it. The point is continuous I/O, not capacity. + file_size_gb: 1 + # Longer than the run's DURATION so fio never exits first and leaves the last migrations + # measuring an idle volume. + runtime_s: 7500 + iodepth: 8 + # 1, and it must stay 1 for an integrity run: numjobs > 1 cannot serialize overlapping + # writes without io_submit_mode=offload, so verify would report corruption that never + # happened — and the workload switches verification off rather than lie. + numjobs: 1 + + migration.driver: + # `alternate` puts the harder case (target also hosts a consumer) on the odd migrations, + # so a run cut short still exercised it. + target_policy: alternate + gap_s: 30.0 + timeout_s: 600.0 + + ana.sample: + # 2s. The freeze windows being measured are 2-8s, so a coarser interval cannot count + # them and a finer one costs an exec per node per sample. + interval_s: 2.0 + + # The SPDK containers outrun kubelet rotation within the hour — measured on vm04: rotation + # every ~2 min, so the whole 50 MiB budget bought about 10 minutes — so they are followed + # live. Everything else is still complete from a one-shot grab at the end. + logs.stream: + containers: + - spdk-container + - spdk-proxy-container + ttl_s: 21600 + logs.collect: true + host.dmesg: true + cluster.events: true + nvme.snapshot: true + +detectors: + ana.freeze-count: + # 1. More than one ANA freeze in a migration meant lost writes on 5 of 5 archived runs: + # each extra freeze is a revert-and-retry, and the retry can copy nothing. + max_freezes: 1 + ana.cutover-pause: + # The control plane makes all paths inaccessible for ~2s by design; 5 leaves headroom + # without reaching fast_io_fail_tmo, past which the pause becomes application-visible. + max_pause_s: 5.0 + ana.split-brain: true + ana.unserved-after-cutover: true + + fio.job-error: true + fio.checksum: + # A lost write surfaces when fio next reads the block, which is seconds to tens of + # seconds after the migration that lost it — without the lag, a migration's own + # corruption files itself under "no migration was running". + verify_lag_s: 45 + fio.outage: + min_seconds: 30 + + migration.outcomes: + min_completed_fraction: 0.8 + max_timeout_fraction: 0.1 + migration.errors: true + + nvme.stale-controllers: true + nvme.loss-timeout: true + nvme.controller-churn: true + nvme.foreign-cluster: true + + kernel.path-loss: true + kernel.filesystem-shutdown: true + + control.node-flap: true + control.task-stuck: true + control.volume-health: true + + logs.pattern: true + + # Bounds what the rest of the report is allowed to claim. + evidence.log-coverage: true + evidence.blind-spot: true + evidence.inventory: true diff --git a/test/framework/sbtest/suites/migration-soak.yaml b/test/framework/sbtest/suites/migration-soak.yaml new file mode 100644 index 000000000..93d8e1242 --- /dev/null +++ b/test/framework/sbtest/suites/migration-soak.yaml @@ -0,0 +1,47 @@ +# Observation only: no workload, no driver. For watching a cluster someone else is loading — +# including the existing operator/test/fio_migration_test.py — and for checking that +# collection works before committing a long run to it. +# +# make collect OUT=./runs/soak SUITE=migration-soak DURATION=600 +# +# For a run this framework drives itself, use migration-full. +description: >- + Long migration soak — the collection configuration the corruption work needed. Both log + components are on deliberately. + +run: + id_prefix: fiomig + outdir: ./runs + +components: + # The SPDK containers outrun kubelet rotation within the hour, so they are followed live; + # everything else is still complete from a one-shot grab at the end. That split is the + # whole reason both log components exist. + logs.stream: + containers: + - spdk-container + - spdk-proxy-container + ttl_s: 21600 + logs.collect: true + host.dmesg: true + cluster.events: true + # Before *and* after: "did the last run leave a mess?" is a real question, because a leaked + # controller breaks the next run rather than this one. + nvme.snapshot: true + ana.sample: + interval_s: 2.0 + +detectors: + ana.freeze-count: + # More than one freeze in a migration meant lost writes on 5 of 5 archived runs. + max_freezes: 1 + ana.cutover-pause: + max_pause_s: 5.0 + fio.checksum: + # Corruption surfaces when fio next reads the block, after the migration has ended. + verify_lag_s: 45 + fio.outage: + min_seconds: 30 + migration.outcomes: + min_completed_fraction: 0.8 + max_timeout_fraction: 0.1 diff --git a/test/framework/tests/test_core.py b/test/framework/tests/test_core.py new file mode 100644 index 000000000..bbab934bc --- /dev/null +++ b/test/framework/tests/test_core.py @@ -0,0 +1,551 @@ +"""Core tests: config resolution, lifecycle ordering, findings/report, archive round-trip. + +The lifecycle tests matter more than they look. A component that starts pods must be torn +down even when the run explodes, and a collector that fails must not take the run's +judgement with it — both are properties of the runner, not of any component, so this is the +only place they can be pinned. +""" + +from __future__ import annotations + +import csv +import json +import os +import sys +import tempfile +import unittest +from datetime import UTC, datetime, timedelta +from typing import cast + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import sbtest # noqa: E402,F401 (registers the bundled plugins) +from sbtest.adapters import ArchiveEvidence # noqa: E402 +from sbtest.core import ( # noqa: E402 + Component, + Detector, + Logger, + Report, + RunContext, + Runner, + Severity, + SkipDetector, + apply_cli_toggles, + component, + critical, + detector, + known_components, + known_detectors, + load, + now_utc, +) +from sbtest.core.config import _resolve # noqa: E402 + + +class ConfigResolution(unittest.TestCase): + KNOWN_C = ["logs.stream", "logs.collect"] + KNOWN_D = ["ana.freeze-count", "fio.checksum"] + + def test_absent_component_is_off_and_absent_detector_is_on(self): + """Collecting is a cost you opt into; judging should not need remembering.""" + cfg = load(None, self.KNOWN_C, self.KNOWN_D) + self.assertEqual(cfg.components.enabled, {}) + self.assertEqual(sorted(cfg.components.disabled), sorted(self.KNOWN_C)) + self.assertEqual(sorted(cfg.detectors.enabled), sorted(self.KNOWN_D)) + + def test_bool_and_mapping_forms(self): + sel = _resolve({"logs.stream": True, "logs.collect": {"ttl_s": 99}}, + self.KNOWN_C, False) + self.assertEqual(sel.enabled["logs.stream"], {}) + self.assertEqual(sel.enabled["logs.collect"], {"ttl_s": 99}) + + def test_enabled_false_inside_a_mapping_disables(self): + sel = _resolve({"logs.stream": {"enabled": False, "ttl_s": 5}}, self.KNOWN_C, True) + self.assertIn("logs.stream", sel.disabled) + self.assertNotIn("logs.stream", sel.enabled) + + def test_unknown_name_fails_loudly(self): + with self.assertRaises(ValueError) as cm: + _resolve({"logs.stremm": True}, self.KNOWN_C, False) + self.assertIn("logs.stremm", str(cm.exception)) + + def test_unknown_option_fails_at_build_not_at_use(self): + with self.assertRaises(KeyError): + sbtest.build_detector("nope.not-a-detector") + with self.assertRaises(ValueError): + sbtest.build_detector("ana.freeze-count", wrong_option=1) + + def test_cli_disable_beats_enable(self): + sel = _resolve({}, self.KNOWN_C, True) + apply_cli_toggles(sel, ["logs.stream"], ["logs.stream"]) + self.assertIn("logs.stream", sel.disabled) + self.assertNotIn("logs.stream", sel.enabled) + + def test_yaml_suite_round_trip(self): + """Comments and all — the reason suites are YAML is that a threshold wants a sentence + saying why it is that number, and the parser must not care that one is there.""" + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "suite.yaml") + with open(p, "w") as fh: + fh.write("components:\n" + " logs.stream:\n" + " ttl_s: 7 # six hours was not enough on vm04\n" + "detectors:\n" + " fio.checksum: false\n") + cfg = load(p, self.KNOWN_C, self.KNOWN_D) + self.assertEqual(cfg.components.enabled["logs.stream"], {"ttl_s": 7}) + self.assertIn("fio.checksum", cfg.detectors.disabled) + self.assertIn("ana.freeze-count", cfg.detectors.enabled) + + def test_json_suite_still_loads(self): + """Kept working for anything generating suites programmatically.""" + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "suite.json") + with open(p, "w") as fh: + json.dump({"components": {"logs.stream": {"ttl_s": 7}}, + "detectors": {"fio.checksum": False}}, fh) + cfg = load(p, self.KNOWN_C, self.KNOWN_D) + self.assertEqual(cfg.components.enabled["logs.stream"], {"ttl_s": 7}) + self.assertIn("fio.checksum", cfg.detectors.disabled) + + def test_malformed_yaml_names_the_file(self): + """A parse error must say which suite and where, not surface as a bare YAMLError.""" + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "broken.yaml") + with open(p, "w") as fh: + fh.write("components:\n logs.stream: {ttl_s: 7\n") + with self.assertRaises(ValueError) as cm: + load(p, self.KNOWN_C, self.KNOWN_D) + self.assertIn("broken.yaml", str(cm.exception)) + + def test_a_suite_that_is_not_a_mapping_is_refused(self): + with tempfile.TemporaryDirectory() as d: + p = os.path.join(d, "list.yaml") + with open(p, "w") as fh: + fh.write("- logs.stream\n- logs.collect\n") + with self.assertRaises(ValueError) as cm: + load(p, self.KNOWN_C, self.KNOWN_D) + self.assertIn("mapping", str(cm.exception)) + + def test_every_bundled_suite_loads_against_the_real_registry(self): + """The suites ship with the package, so a typo in one is a broken release. Loading + resolves names against the registry, so this also pins that no suite references a + detector that has since been renamed.""" + names = ["analyze-only", "corruption-hunt", "migration-soak", "migration-full"] + for name in names: + p = sbtest.suite_path(name) + self.assertIsNotNone(p, f"bundled suite {name} not found") + assert p is not None + self.assertTrue(p.endswith(".yaml"), f"{name} should be YAML, got {p}") + cfg = load(p, list(known_components()), list(known_detectors())) + self.assertTrue(cfg.raw.get("description"), f"{name} has no description") + # migration-full is the one that must actually drive a run. + full = load(sbtest.suite_path("migration-full"), + list(known_components()), list(known_detectors())) + self.assertIn("workload.fio", full.components.enabled) + self.assertIn("migration.driver", full.components.enabled) + + +class Lifecycle(unittest.TestCase): + def ctx(self, d): + return RunContext(run_id="t", outdir=d, log=Logger(None)) + + def test_hooks_run_in_order(self): + calls = [] + + @component + class Ordered(Component): + name = "test.ordered" + def setup(self, ctx): calls.append("setup") + def start(self, ctx): calls.append("start") + def tick(self, ctx): calls.append("tick") + def stop(self, ctx): calls.append("stop") + def collect(self, ctx): calls.append("collect") + def teardown(self, ctx): calls.append("teardown") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.ordered"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, self.ctx(d)).build() + for phase in ("setup", "start", "tick", "stop", "collect", "teardown"): + getattr(r, phase)() + self.assertEqual(calls, ["setup", "start", "tick", "stop", "collect", "teardown"]) + + def test_teardown_runs_even_when_setup_failed(self): + """A component that allocates in setup must still get its teardown.""" + torn = [] + + @component + class Exploding(Component): + name = "test.exploding" + required = True # this component *is* the run; failing it must abort + def setup(self, ctx): raise RuntimeError("boom") + def teardown(self, ctx): torn.append(self.name) + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.exploding"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, self.ctx(d)).build() + with self.assertRaises(RuntimeError): + r.setup() + r.teardown() + self.assertEqual(torn, ["test.exploding"]) + + def test_a_failing_collector_does_not_stop_the_others(self): + done = [] + + @component + class BadCollect(Component): + name = "test.badcollect" + def collect(self, ctx): raise RuntimeError("nope") + + @component + class GoodCollect(Component): + name = "test.goodcollect" + def collect(self, ctx): done.append("good") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.badcollect", "test.goodcollect"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, self.ctx(d)).build() + r.collect() + self.assertEqual(done, ["good"]) + # and the failure is recorded rather than swallowed + self.assertTrue(any(f.detector == "component/test.badcollect" for f in r.report.findings)) + + +class Judging(unittest.TestCase): + def test_a_raising_detector_is_reported_not_treated_as_clean(self): + @detector + class Boom(Detector): + name = "test.boom" + def detect(self, ev): raise ValueError("bug in me") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + cfg.components.enabled = {} + cfg.detectors.enabled = {"test.boom": {}} + r = Runner(cfg, RunContext(run_id="t", outdir=d, log=Logger(None))).build() + # Evidence is never touched: the detector raises first. That is the point. + rep = r.judge(cast("sbtest.core.Evidence", object())) + self.assertIn("test.boom", rep.skipped) + self.assertTrue(any("raised" in f.title for f in rep.findings)) + + def test_skip_is_distinguishable_from_clean(self): + @detector + class Skipper(Detector): + name = "test.skipper" + def detect(self, ev): raise SkipDetector("no evidence here") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + cfg.components.enabled = {} + cfg.detectors.enabled = {"test.skipper": {}} + r = Runner(cfg, RunContext(run_id="t", outdir=d, log=Logger(None))).build() + rep = r.judge(cast("sbtest.core.Evidence", object())) + self.assertEqual(rep.findings, []) + self.assertEqual(rep.skipped["test.skipper"], "no evidence here") + self.assertEqual(rep.verdict, "PASS") # nothing found, but the gap is on record + + +class Findings(unittest.TestCase): + def test_critical_fails_the_verdict(self): + r = Report(run_id="x") + self.assertEqual(r.verdict, "PASS") + r.add(critical("d", "bad thing", subject="s")) + self.assertEqual(r.verdict, "FAIL") + + def test_by_subject_groups_and_orders_worst_first(self): + r = Report() + r.add(sbtest.info("a", "note", subject="mig-1"), + critical("b", "boom", subject="mig-1")) + self.assertEqual([f.severity for f in r.by_subject()["mig-1"]], + [Severity.CRITICAL, Severity.INFO]) + + def test_json_is_serialisable(self): + r = Report(run_id="x") + r.add(critical("d", "t", subject="s", evidence={"n": 1})) + parsed = json.loads(r.to_json()) + self.assertEqual(parsed["verdict"], "FAIL") + self.assertEqual(parsed["findings"][0]["severity"], "CRITICAL") + + +class Archive(unittest.TestCase): + """The archive reader is the seam that makes a check testable offline, so its + tolerance for missing files is a property worth pinning.""" + + def _write_run(self, d: str) -> None: + t0 = datetime(2026, 8, 19, 22, 0, 0, tzinfo=UTC) + state = {"run_id": "fiomig-test", "pods": ["fiomig-test-fio-0"], "migrations": [ + {"name": "mig-1", "start": t0.isoformat().replace("+00:00", "Z"), + "end": (t0 + timedelta(seconds=30)).isoformat().replace("+00:00", "Z"), + "phase": "Completed", "source": "srcuuid", "target": "tgtuuid", + "pv": "pvc-a", "group_pvs": ["pvc-a", "pvc-b"]}]} + with open(os.path.join(d, "state.json"), "w") as fh: + json.dump(state, fh) + os.makedirs(os.path.join(d, "ana"), exist_ok=True) + with open(os.path.join(d, "ana", "mig-1.csv"), "w", newline="") as fh: + w = csv.writer(fh) + w.writerow(["ts", "node", "phase", "address", "role", "ctrl_state", "nsid", "ana_state"]) + for off, st in ((0, "optimized"), (2, "inaccessible"), (4, "optimized")): + stamp = (t0 + timedelta(seconds=off)).strftime("%Y-%m-%dT%H:%M:%SZ") + for nsid in (1, 2): + w.writerow([stamp, "vm03", "Running", "10.0.0.1:4420", "source", + "live", nsid, st]) + pod = os.path.join(d, "fiomig-test-fio-0") + os.makedirs(pod, exist_ok=True) + with open(os.path.join(pod, "result.json"), "w") as fh: + json.dump({"jobs": [{"error": 121, "read": {"iops": 100.0}, + "write": {"iops": 50.0}}]}, fh) + with open(os.path.join(pod, "fio.log"), "w") as fh: + fh.write("2026-08-19T22:00:05.000000000Z stderr F all fine\n") + with open(os.path.join(pod, "timeseries.csv"), "w") as fh: + fh.write("t,wall,total_iops\n0,,100\n1,,0\n") + with open(os.path.join(d, "spdk-4420.txt"), "w") as fh: + fh.write("boring line\n") + + def test_reads_a_run_directory(self): + with tempfile.TemporaryDirectory() as d: + self._write_run(d) + ev = ArchiveEvidence(d) + self.assertEqual(ev.run_id, "fiomig-test") + migs = ev.migrations() + self.assertEqual(len(migs), 1) + self.assertTrue(migs[0].batch) # two group_pvs + self.assertEqual(len(ev.ana_samples("mig-1")), 3) # one per (ts, node, address) + self.assertEqual(ev.fio_jobs()[0].error, 121) + self.assertAlmostEqual(ev.fio_jobs()[0].total_iops, 150.0) + self.assertEqual(len(ev.fio_timeseries("fiomig-test-fio-0")), 2) + self.assertIn("spdk-4420", ev.container_logs()) + + def test_missing_files_yield_empty_not_an_exception(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "state.json"), "w") as fh: + json.dump({"run_id": "r", "migrations": []}, fh) + ev = ArchiveEvidence(d) + self.assertEqual(ev.migrations(), []) + self.assertEqual(ev.ana_samples("nope"), []) + self.assertEqual(ev.fio_jobs(), []) + self.assertEqual(list(ev.fio_log("nope")), []) + self.assertEqual(ev.nvme_controllers(), []) + + def test_falls_back_to_test_log_when_state_is_absent(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "test.log"), "w") as fh: + fh.write( + "2026-08-19T22:00:00Z [EVENT ] MIGRATION START mig-9 kind=namespaced " + "pod=p pv=pvc-x source=s target=t\n" + "2026-08-19T22:01:00Z [EVENT ] MIGRATION STOP mig-9 phase=TIMEOUT " + "error='timed out'\n") + ev = ArchiveEvidence(d) + migs = ev.migrations() + self.assertEqual(len(migs), 1) + self.assertEqual(migs[0].phase, "TIMEOUT") + self.assertEqual(migs[0].error, "timed out") + + def test_absent_directory_is_an_error(self): + with self.assertRaises(FileNotFoundError): + ArchiveEvidence("/definitely/not/here") + + +class Registry(unittest.TestCase): + @staticmethod + def _bundled(reg): + # Other tests register throwaway plugins into the same global registry. + return {k: v for k, v in reg.items() if not k.startswith("test.")} + + def test_every_bundled_detector_has_a_name_and_a_summary(self): + for name, cls in self._bundled(known_detectors()).items(): + self.assertTrue(name, cls) + self.assertTrue(cls.summary, f"{name} needs a summary") + + def test_every_bundled_component_has_a_name_and_a_summary(self): + for name, cls in self._bundled(known_components()).items(): + self.assertTrue(name, cls) + self.assertTrue(cls.summary, f"{name} needs a summary") + + def test_detector_defaults_are_the_option_allow_list(self): + for name, cls in self._bundled(known_detectors()).items(): + d = cls() + self.assertEqual(set(d.options), set(d.defaults()), + f"{name}: options must start from defaults") + + def test_duplicate_registration_is_rejected(self): + with self.assertRaises(ValueError): + @detector + class Dup(Detector): + name = "ana.freeze-count" + + +if __name__ == "__main__": + unittest.main(verbosity=2) + + +class RunWindowRecording(unittest.TestCase): + """A live run must record its own window. + + Found by running the collector against a real cluster: with no window, nothing in the + artifact directory says when the run was, so every detector reading a ring buffer marks + what it finds UNKNOWN — and UNKNOWN counts. The collector failed on twenty-nine filesystem + shutdowns that had happened hours before it started. + """ + + def test_run_json_is_written_and_read_back(self): + with tempfile.TemporaryDirectory() as d: + ctx = RunContext(run_id="t", outdir=d, log=Logger(None)) + ctx.mark_window() + ctx.mark_window(end=now_utc()) + with open(os.path.join(d, "run.json")) as fh: + rec = json.load(fh) + self.assertEqual(rec["run_id"], "t") + self.assertTrue(rec["start"] and rec["end"]) + start, end = ArchiveEvidence(d).run_window() + self.assertIsNotNone(start) + self.assertIsNotNone(end) + + def test_run_json_wins_over_inference_from_test_log(self): + """The run's own record beats whatever happened to get logged.""" + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "test.log"), "w") as fh: + fh.write("1999-01-01T00:00:00Z [INFO] ancient\n") + with open(os.path.join(d, "run.json"), "w") as fh: + json.dump({"run_id": "t", "start": "2026-08-20T11:40:09Z", + "end": "2026-08-20T11:42:20Z"}, fh) + start, _end = ArchiveEvidence(d).run_window() + assert start is not None + self.assertEqual(start.year, 2026) + + def test_a_recorded_window_demotes_older_damage(self): + """The end-to-end point: with a window, inherited damage stops failing the run.""" + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, "run.json"), "w") as fh: + json.dump({"run_id": "t", "start": "2026-08-20T11:40:09Z", + "end": "2026-08-20T11:42:20Z"}, fh) + # A filesystem shutdown from hours before the run began. + with open(os.path.join(d, "dmesg-vm03.txt"), "w") as fh: + fh.write("[Thu Aug 20 05:46:57 2026] XFS (nvme0n1): " + "Filesystem has been shut down due to log error (0x2).\n") + ev = ArchiveEvidence(d) + found = list(sbtest.build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, sbtest.Attribution.PRE_EXISTING) + rep = Report() + rep.add(*found) + self.assertFalse(rep.failed) # must not fail the run + self.assertEqual(rep.verdict, "PASS") # a WARNING, so not even inconclusive + + +class GrabberNaming(unittest.TestCase): + """Two components wanting a grabber on one node must not collide. + + Found by running it: a Pod is immutable, so the second component to apply the same name + failed on a field it may not change — and logs.collect silently produced empty files for + every node logs.stream had already claimed. + """ + + def test_stream_and_collect_pick_different_pod_names(self): + from sbtest.components.logs import LogCollect, LogStream + ctx = RunContext(run_id="run1", outdir="/tmp", log=Logger(None)) + node = "vm02.example.com" + stream = LogStream() + collect = LogCollect() + n1 = stream._grabber_manifest(ctx, node, f"sbtest-{stream.name.replace('.', '-')}" + f"-vm02-run1", 100) + n2 = collect._grabber_manifest(ctx, node, f"sbtest-{collect.name.replace('.', '-')}" + f"-vm02-run1", 100) + name1 = json.loads(n1)["metadata"]["name"] + name2 = json.loads(n2)["metadata"]["name"] + self.assertNotEqual(name1, name2) + self.assertIn("logs-stream", name1) + self.assertIn("logs-collect", name2) + + +class GrabberReuse(unittest.TestCase): + """logs.collect must reuse logs.stream's grabbers, and must not delete them. + + Two components wanting a privileged pod on the same node is the normal case, and the + first attempt at fixing their collision only gave them different names — which works but + leaves two pods per node doing the same job. Reuse is the intended behaviour; the distinct + names are the backstop for when reuse is not possible. + """ + + def test_collect_reuses_published_grabbers_and_starts_only_the_rest(self): + from sbtest.components.logs import LogCollect + started: list[list[str]] = [] + + class Probe(LogCollect): + def _start_grabbers(self, ctx, nodes, ttl_s): + started.append(list(nodes)) + return {n: f"own-{n}" for n in nodes} + + c = Probe() + with tempfile.TemporaryDirectory() as d: + ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) + ctx.shared["logs.grabbers"] = {"vm02": "stream-vm02", "vm03": "stream-vm03"} + # Stand in for the pod census: two nodes already covered, one not. + plan_nodes = ["vm02", "vm03", "vm04"] + inherited = dict(ctx.shared["logs.grabbers"]) + missing = sorted(n for n in plan_nodes if n not in inherited) + own = c._start_grabbers(ctx, missing, 60) + c._own = own + c._grabbers = {**{n: g for n, g in inherited.items() if n in plan_nodes}, **own} + + self.assertEqual(started, [["vm04"]]) # only the uncovered node + self.assertEqual(c._grabbers["vm02"], "stream-vm02") # reused, not replaced + self.assertEqual(c._grabbers["vm04"], "own-vm04") + + deleted: list[list[str]] = [] + c._delete_grabbers = lambda _ctx, names: deleted.append(list(names)) # type: ignore[assignment] + c.teardown(ctx) + # Only its own; deleting a borrowed pod would pull it from under its owner. + self.assertEqual(deleted, [["own-vm04"]]) + + def test_collect_works_standalone_when_nothing_published(self): + """logs.stream may be disabled — collect must still start what it needs.""" + from sbtest.components.logs import LogCollect + started: list[list[str]] = [] + + class Probe(LogCollect): + def _start_grabbers(self, ctx, nodes, ttl_s): + started.append(list(nodes)) + return {n: f"own-{n}" for n in nodes} + + c = Probe() + with tempfile.TemporaryDirectory() as d: + ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) + inherited = dict(ctx.shared.get("logs.grabbers", {})) + self.assertEqual(inherited, {}) + own = c._start_grabbers(ctx, ["vm02", "vm03"], 60) + self.assertEqual(started, [["vm02", "vm03"]]) + self.assertEqual(sorted(own), ["vm02", "vm03"]) + + def test_lifecycle_order_makes_reuse_safe(self): + """stop precedes collect, and teardown follows it — so the borrowed pod is idle and + still alive exactly when collection needs it.""" + order: list[str] = [] + + @component + class Streamish(Component): + name = "test.streamish" + def start(self, ctx): order.append("stream.start") + def stop(self, ctx): order.append("stream.stop") + def teardown(self, ctx): order.append("stream.teardown") + + @component + class Collectish(Component): + name = "test.collectish" + def collect(self, ctx): order.append("collect.collect") + + with tempfile.TemporaryDirectory() as d: + cfg = load(None, list(known_components()), list(known_detectors())) + apply_cli_toggles(cfg.components, ["test.streamish", "test.collectish"], []) + cfg.detectors.enabled = {} + r = Runner(cfg, RunContext(run_id="r", outdir=d, log=Logger(None))).build() + for phase in ("setup", "start", "stop", "collect", "teardown"): + getattr(r, phase)() + + self.assertLess(order.index("stream.stop"), order.index("collect.collect")) + self.assertLess(order.index("collect.collect"), order.index("stream.teardown")) diff --git a/test/framework/tests/test_detectors.py b/test/framework/tests/test_detectors.py new file mode 100644 index 000000000..eb4621cf4 --- /dev/null +++ b/test/framework/tests/test_detectors.py @@ -0,0 +1,852 @@ +"""Detector unit tests. + +Detectors are pure functions from Evidence to findings, which is what makes this file +possible: every check below is a handful of synthetic samples rather than a cluster. The +cases are the ones the real runs taught — a healthy single freeze, a retried cutover, a +completed-but-corrupting migration, a verify failure that surfaces after its migration +ended. +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sbtest.core import ( # noqa: E402 + AnaSample, + Attribution, + ControlEvent, + FioJob, + IopsSample, + LogSpan, + Migration, + NvmeController, + Report, + Severity, + SkipDetector, + build_detector, + freeze_windows, +) + +T0 = datetime(2026, 8, 19, 22, 0, 0, tzinfo=UTC) + + +def ts(sec: int) -> datetime: + return T0 + timedelta(seconds=sec) + + +class FakeEvidence: + """Evidence from literals. Only what a test sets is present; the rest is empty. + + Typed explicitly rather than **kwargs so that mypy checks it against the Evidence + protocol — a test double that has drifted from the real contract is worse than no double, + because every detector test keeps passing while the detector itself has stopped matching. + """ + + def __init__( + self, + run_id: str = "test-run", + outdir: str = "/nonexistent", + migrations: list[Migration] | None = None, + ana: dict[str, list[AnaSample]] | None = None, + jobs: list[FioJob] | None = None, + series: dict[str, list[IopsSample]] | None = None, + fio_logs: dict[str, list[str]] | None = None, + logs: dict[str, list[str]] | None = None, + controllers: list[NvmeController] | None = None, + cluster: str = "", + window: tuple[datetime | None, datetime | None] = (None, None), + events: list[ControlEvent] | None = None, + spans: list[LogSpan] | None = None, + ) -> None: + self.run_id = run_id + self.outdir = outdir + self._migrations = migrations or [] + self._ana = ana or {} + self._jobs = jobs or [] + self._series = series or {} + self._fio_logs = fio_logs or {} + self._logs = logs or {} + self._ctrls = controllers or [] + self._cluster = cluster + self._window = window + self._events = events or [] + self._spans = spans + + def migrations(self) -> list[Migration]: + return list(self._migrations) + + def ana_samples(self, migration: str) -> list[AnaSample]: + return list(self._ana.get(migration, [])) + + def fio_jobs(self) -> list[FioJob]: + return list(self._jobs) + + def fio_timeseries(self, pod: str) -> list[IopsSample]: + return list(self._series.get(pod, [])) + + def fio_log(self, pod: str) -> Iterator[str]: + return iter(self._fio_logs.get(pod, [])) + + def container_logs(self) -> list[str]: + return sorted(self._logs) + + def container_log(self, name: str) -> Iterator[str]: + return iter(self._logs.get(name, [])) + + def nvme_controllers(self) -> list[NvmeController]: + return list(self._ctrls) + + def pods(self) -> list[str]: + return sorted(set(self._fio_logs) | {j.pod for j in self._jobs} | set(self._series)) + + def cluster_uuid(self) -> str: + return self._cluster + + def run_window(self) -> tuple[datetime | None, datetime | None]: + return self._window + + def control_events(self) -> list[ControlEvent]: + return list(self._events) + + def log_spans(self) -> list[LogSpan]: + if self._spans is not None: + return list(self._spans) + return [LogSpan(name=n, first=None, last=None, lines=len(self._logs[n])) + for n in sorted(self._logs)] + + +def ana_series(node: str, address: str, spec: list[tuple[int, str]], nsids=(1, 2), + state="live", role="") -> list[AnaSample]: + """Samples for one path: spec is [(offset_seconds, ana_state), ...].""" + return [AnaSample(ts=ts(off), node=node, address=address, state=state, + ana=dict.fromkeys(nsids, st), role=role) + for off, st in spec] + + +class FreezeWindows(unittest.TestCase): + def test_single_healthy_freeze(self): + s = ana_series("vm03", "10.0.0.113:4430", + [(0, "optimized"), (2, "inaccessible"), (4, "optimized"), + (6, "optimized")]) + w = freeze_windows(s) + self.assertEqual(len(w), 1) + self.assertAlmostEqual(w[0][1], 2.0) + + def test_zero_length_window_is_not_a_freeze(self): + # One sample inaccessible then immediately accessible again at the same instant is + # a sampling artefact, not a window. + s = [AnaSample(ts=ts(0), node="v", address="a", state="live", ana={1: "optimized"}), + AnaSample(ts=ts(0), node="v", address="b", state="live", ana={1: "inaccessible"})] + self.assertEqual(freeze_windows(s), []) + + def test_counts_per_node_not_summed(self): + # Two nodes see the same freeze; the count is one, not two. + s = ana_series("vm02", "a", [(0, "optimized"), (2, "inaccessible"), (4, "optimized")]) + s += ana_series("vm03", "b", [(0, "optimized"), (2, "inaccessible"), (4, "optimized")]) + self.assertEqual(len(freeze_windows(s)), 1) + + def test_trailing_freeze_is_counted(self): + s = ana_series("vm03", "a", [(0, "optimized"), (2, "inaccessible"), (6, "inaccessible")]) + w = freeze_windows(s) + self.assertEqual(len(w), 1) + self.assertAlmostEqual(w[0][1], 4.0) + + +class AnaFreezeCount(unittest.TestCase): + def detector(self, **opts): + return build_detector("ana.freeze-count", **opts) + + def test_one_freeze_is_clean(self): + m = Migration(name="mig-1", start=ts(0), end=ts(10), phase="Completed") + ev = FakeEvidence(migrations=[m], ana={"mig-1": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (4, "optimized")])}) + self.assertEqual(list(self.detector().detect(ev)), []) + + def test_two_freezes_is_critical(self): + """The mig-20 shape: froze, reverted, froze again — and it Completed.""" + m = Migration(name="mig-20", start=ts(0), end=ts(30), phase="Completed") + ev = FakeEvidence(migrations=[m], ana={"mig-20": ana_series( + "vm03", "a", + [(0, "optimized"), (2, "inaccessible"), (8, "optimized"), + (12, "inaccessible"), (18, "optimized")])}) + found = list(self.detector().detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].subject, "mig-20") + self.assertEqual(found[0].evidence["freezes"], 2) + # Phase must not exempt it: a Completed migration can still have lost writes. + self.assertEqual(found[0].evidence["phase"], "Completed") + + def test_threshold_is_configurable(self): + m = Migration(name="mig-20", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"mig-20": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (8, "optimized"), + (12, "inaccessible"), (18, "optimized")])}) + self.assertEqual(list(self.detector(max_freezes=2).detect(ev)), []) + + def test_skips_when_no_samples(self): + ev = FakeEvidence(migrations=[Migration(name="m", start=ts(0))]) + with self.assertRaises(SkipDetector): + list(self.detector().detect(ev)) + + def test_skips_when_no_migrations(self): + with self.assertRaises(SkipDetector): + list(self.detector().detect(FakeEvidence())) + + def test_rejects_unknown_option(self): + with self.assertRaises(ValueError): + build_detector("ana.freeze-count", maxfreezes=2) + + +class AnaCutoverPause(unittest.TestCase): + def test_pause_within_budget_is_clean(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (5, "optimized")])}) + self.assertEqual(list(build_detector("ana.cutover-pause").detect(ev)), []) + + def test_overlong_pause_is_critical(self): + m = Migration(name="m", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (14, "optimized")])}) + found = list(build_detector("ana.cutover-pause").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["worst_pause_s"], 12.0) + + def test_single_long_pause_is_caught_where_freeze_count_is_not(self): + """Why both detectors exist: one window, too long. Count says fine, pause does not.""" + m = Migration(name="m", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (20, "optimized")])}) + self.assertEqual(list(build_detector("ana.freeze-count").detect(ev)), []) + self.assertEqual(len(list(build_detector("ana.cutover-pause").detect(ev))), 1) + + def test_two_short_freezes_are_caught_where_pause_is_not(self): + """And the converse: two design-length windows. Pause says fine, count does not.""" + m = Migration(name="m", start=ts(0), end=ts(30)) + ev = FakeEvidence(migrations=[m], ana={"m": ana_series( + "vm03", "a", [(0, "optimized"), (2, "inaccessible"), (5, "optimized"), + (10, "inaccessible"), (13, "optimized")])}) + self.assertEqual(list(build_detector("ana.cutover-pause").detect(ev)), []) + self.assertEqual(len(list(build_detector("ana.freeze-count").detect(ev))), 1) + + +class AnaSplitBrain(unittest.TestCase): + def test_both_sides_optimized_is_critical(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + s = ana_series("vm03", "10.0.0.112:4426", [(0, "optimized")], role="source") + s += ana_series("vm03", "10.0.0.114:4428", [(0, "optimized")], role="target") + found = list(build_detector("ana.split-brain").detect( + FakeEvidence(migrations=[m], ana={"m": s}))) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + + def test_target_non_optimized_is_normal_ha_standby(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + s = ana_series("vm03", "10.0.0.112:4426", [(0, "optimized")], role="source") + s += ana_series("vm03", "10.0.0.114:4428", [(0, "non-optimized")], role="target") + self.assertEqual(list(build_detector("ana.split-brain").detect( + FakeEvidence(migrations=[m], ana={"m": s}))), []) + + def test_skips_without_roles(self): + m = Migration(name="m", start=ts(0), end=ts(20)) + s = ana_series("vm03", "a", [(0, "optimized")]) + with self.assertRaises(SkipDetector): + list(build_detector("ana.split-brain").detect( + FakeEvidence(migrations=[m], ana={"m": s}))) + + +class FioChecksum(unittest.TestCase): + LINE = ("2026-08-19T22:00:{sec:02d}.123456789Z stderr F verify: bad magic header 0, " + "wanted acca at file /data/fiotest offset 43999232, length 4096") + + def test_attributes_a_lagged_detection_to_its_migration(self): + """The mig-20 case: the loss surfaces after the migration ended, inside the backlog. + + Without the lag this lands in "outside-any-migration", which is how a + Completed-but-corrupting migration hid. + """ + m = Migration(name="mig-20", start=ts(0), end=ts(10), phase="Completed") + ev = FakeEvidence(migrations=[m], + fio_logs={"fio-16": [self.LINE.format(sec=28)]}) + found = list(build_detector("fio.checksum").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].subject, "mig-20") + self.assertIn("verify backlog", found[0].detail) + + def test_beyond_the_lag_is_not_attributed(self): + m = Migration(name="mig-20", start=ts(0), end=ts(10), phase="Completed") + ev = FakeEvidence(migrations=[m], + fio_logs={"fio-16": [self.LINE.format(sec=59)]}) + found = list(build_detector("fio.checksum", verify_lag_s=5).detect(ev)) + self.assertEqual(found[0].subject, "outside-any-migration") + + def test_groups_blocks_per_migration_and_counts_pods(self): + m = Migration(name="mig-29", start=ts(0), end=ts(10), phase="TIMEOUT") + ev = FakeEvidence(migrations=[m], fio_logs={ + "fio-6": [self.LINE.format(sec=5), self.LINE.format(sec=6)], + "fio-7": [self.LINE.format(sec=7)]}) + found = list(build_detector("fio.checksum").detect(ev)) + self.assertEqual(found[0].evidence["blocks"], 3) + self.assertEqual(found[0].evidence["pods"], {"fio-6": 2, "fio-7": 1}) + + def test_clean_log_yields_nothing(self): + ev = FakeEvidence(fio_logs={"fio-1": ["all good\n"]}) + self.assertEqual(list(build_detector("fio.checksum").detect(ev)), []) + + def test_skips_without_logs(self): + with self.assertRaises(SkipDetector): + list(build_detector("fio.checksum").detect(FakeEvidence(jobs=[FioJob(pod="p")]))) + + +class FioJobError(unittest.TestCase): + def test_eremoteio_carries_the_hint_that_points_at_ana(self): + ev = FakeEvidence(jobs=[FioJob(pod="fio-0", error=121)]) + found = list(build_detector("fio.job-error").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIn("EREMOTEIO", found[0].detail) + + def test_eilseq_is_named_as_corruption_not_an_io_failure(self): + """Linux errno 84. Worth pinning: it is EOVERFLOW on macOS, so a local lookup would + mislabel the one code that means the data was wrong.""" + ev = FakeEvidence(jobs=[FioJob(pod="fio-13", error=84)]) + found = list(build_detector("fio.job-error").detect(ev)) + self.assertIn("EILSEQ", found[0].detail) + self.assertIn("corruption", found[0].detail) + + def test_clean_jobs_yield_nothing(self): + ev = FakeEvidence(jobs=[FioJob(pod="fio-0", error=0)]) + self.assertEqual(list(build_detector("fio.job-error").detect(ev)), []) + + def test_ignore_list(self): + ev = FakeEvidence(jobs=[FioJob(pod="fio-0", error=121)]) + self.assertEqual(list(build_detector("fio.job-error", ignore_errnos=[121]).detect(ev)), []) + + +class FioOutage(unittest.TestCase): + def series(self, pattern: str) -> list[IopsSample]: + # "1" = doing I/O, "0" = stopped; one character per second. + return [IopsSample(offset_s=i, wall=ts(i), total_iops=100.0 if c == "1" else 0.0) + for i, c in enumerate(pattern)] + + def test_short_dip_is_not_an_outage(self): + ev = FakeEvidence(series={"p": self.series("1" * 10 + "0" * 5 + "1" * 10)}) + self.assertEqual(list(build_detector("fio.outage", min_seconds=30).detect(ev)), []) + + def test_sustained_stop_is_critical(self): + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40 + "1" * 5)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["worst_s"], 40) + self.assertEqual(found[0].evidence["downtime_s"], 40) + + def test_trailing_outage_is_reported(self): + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40)}) + self.assertEqual(len(list(build_detector("fio.outage", min_seconds=30).detect(ev))), 1) + + def test_many_windows_collapse_into_one_finding_per_pod(self): + """A repeatedly stalling pod stalls hundreds of times in a soak — one archived run + produced 781 qualifying windows. Per-window findings make the report unreadable, so + the count and the total are the finding and the worst windows are named.""" + # Trailing "1" so all 20 windows are closed by a resumption: a window still open at + # the end of the series is measured to the last sample, which is one second short. + ev = FakeEvidence(series={"p": self.series(("1" * 5 + "0" * 31) * 20 + "1")}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["windows"], 20) + self.assertEqual(found[0].evidence["downtime_s"], 20 * 31) + self.assertIn("20x", found[0].title) + + def test_windows_outside_any_migration_say_so(self): + """Attribution is the difference between "migration cost this" and "the cluster is + unwell", and the note must not imply the former when it was neither.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["migrations"], []) + self.assertIn("elsewhere", found[0].note) + + +class NvmeStaleControllers(unittest.TestCase): + def ctrl(self, name, state, ns, node="vm03", nqn="nqn:lvol:x", addr="10.0.0.1:4420", clt=60): + return NvmeController(node=node, name=name, nqn=nqn, address=addr, state=state, + namespaces=ns, ctrl_loss_tmo=clt) + + def test_live_with_no_namespace_is_critical(self): + """The state that blocked every later migration of a subsystem.""" + ev = FakeEvidence(controllers=[ + self.ctrl("nvme12", "live", {}), + self.ctrl("nvme13", "live", {1: "optimized"}, addr="10.0.0.2:4420")]) + found = [f for f in build_detector("nvme.stale-controllers").detect(ev) + if f.severity == Severity.CRITICAL] + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["count"], 1) + + def test_connecting_is_only_a_warning(self): + # A snapshot cannot tell a leak from a normal HA reconnect. + ev = FakeEvidence(controllers=[self.ctrl("nvme6", "connecting", {})]) + sevs = {f.severity for f in build_detector("nvme.stale-controllers").detect(ev)} + self.assertNotIn(Severity.CRITICAL, sevs) + self.assertIn(Severity.WARNING, sevs) + + def test_healthy_fabric_is_clean(self): + ev = FakeEvidence(controllers=[ + self.ctrl("nvme1", "live", {1: "optimized"}, addr="10.0.0.1:4420"), + self.ctrl("nvme2", "live", {1: "non-optimized"}, addr="10.0.0.2:4420")]) + self.assertEqual(list(build_detector("nvme.stale-controllers").detect(ev)), []) + + def test_skips_without_a_snapshot(self): + with self.assertRaises(SkipDetector): + list(build_detector("nvme.stale-controllers").detect(FakeEvidence())) + + def test_loss_timeout_flags_a_value_that_outlives_the_run(self): + ev = FakeEvidence(controllers=[self.ctrl("nvme1", "live", {1: "optimized"}, clt=3600)]) + found = list(build_detector("nvme.loss-timeout").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["values"], [3600]) + + +class LogPattern(unittest.TestCase): + def test_catalogue_catches_the_undrained_transfer(self): + ev = FakeEvidence(logs={"spdk-4422": [ + "transfer task failed: ----- but still have outstanding io 1\n"] * 3}) + found = [f for f in build_detector("logs.pattern").detect(ev) + if f.subject == "spdk.undrained-transfer"] + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].evidence["total"], 3) + + def test_min_count_suppresses_a_line_that_is_normal_in_small_numbers(self): + ev = FakeEvidence(logs={"spdk-4422": ["does not allow host X to connect at this address\n"]}) + found = [f for f in build_detector("logs.pattern").detect(ev) + if f.subject == "nvme.host-not-allowed"] + self.assertEqual(found, []) # catalogue min_count is 50 + + def test_user_defined_pattern_replaces_the_catalogue(self): + ev = FakeEvidence(logs={"mylog": ["something odd happened\n"]}) + d = build_detector("logs.pattern", patterns=[ + {"id": "my.check", "regex": "something odd", "logs": ["mylog"], + "severity": "critical"}]) + found = list(d.detect(ev)) + self.assertEqual([f.subject for f in found], ["my.check"]) + + def test_log_glob_scopes_the_pattern(self): + ev = FakeEvidence(logs={"spdk-4420": ["boom\n"], "operator": ["boom\n"]}) + d = build_detector("logs.pattern", patterns=[ + {"id": "only.spdk", "regex": "boom", "logs": ["spdk-*"], "severity": "warning"}]) + found = list(d.detect(ev)) + self.assertEqual(found[0].evidence["per_log"], {"spdk-4420": 1}) + + def test_rejects_a_malformed_pattern(self): + ev = FakeEvidence(logs={"x": ["y"]}) + d = build_detector("logs.pattern", patterns=[{"id": "a", "rgex": "y"}]) + with self.assertRaises(ValueError): + list(d.detect(ev)) + + def test_skips_without_logs(self): + with self.assertRaises(SkipDetector): + list(build_detector("logs.pattern").detect(FakeEvidence())) + + +class MigrationOutcomes(unittest.TestCase): + def test_low_completion_is_critical(self): + migs = ([Migration(name=f"m{i}", start=ts(i), phase="TIMEOUT") for i in range(25)] + + [Migration(name=f"c{i}", start=ts(100 + i), phase="Completed") for i in range(13)]) + found = list(build_detector("migration.outcomes").detect(FakeEvidence(migrations=migs))) + crit = [f for f in found if f.severity == Severity.CRITICAL] + self.assertEqual(len(crit), 2) # completion rate and timeout rate + self.assertTrue(any("timed out" in f.title for f in crit)) + + def test_healthy_run_reports_info_only(self): + migs = [Migration(name=f"c{i}", start=ts(i), phase="Completed") for i in range(10)] + found = list(build_detector("migration.outcomes").detect(FakeEvidence(migrations=migs))) + self.assertTrue(all(f.severity == Severity.INFO for f in found)) + + def test_errors_group_by_shape(self): + migs = [Migration(name=f"m{i}", start=ts(i), phase="Failed", + error=f"NVMe path validation failed on node vm0{i}; cancelled") + for i in range(3)] + found = list(build_detector("migration.errors").detect(FakeEvidence(migrations=migs))) + self.assertEqual(len(found), 1) # three messages, one shape + self.assertEqual(found[0].evidence["count"], 3) + + +if __name__ == "__main__": + unittest.main(verbosity=2) + + +def dmesg(*msgs: str, day: int = 20, start: int = 0) -> list[str]: + """dmesg -T lines. Local time, which is why these detectors do not attribute to migrations.""" + return [f"[Thu Aug {day} 05:{46 + (start + i) // 60:02d}:{(start + i) % 60:02d} 2026] {m}\n" + for i, m in enumerate(msgs)] + + +class KernelPathLoss(unittest.TestCase): + """The ladder: requeue (absorbed) -> failfast -> failing I/O (application-visible).""" + + def test_requeue_only_is_a_warning_not_a_failure(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "block nvme0n1: no usable path - requeuing I/O", + "block nvme0n1: no usable path - requeuing I/O")}) + found = list(build_detector("kernel.path-loss").detect(ev)) + self.assertEqual([f.severity for f in found], [Severity.WARNING]) + self.assertEqual(found[0].evidence["requeues"], 2) + + def test_failing_io_is_critical(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "block nvme0n1: no usable path - requeuing I/O", + "nvme nvme10: failfast expired", + "block nvme0n1: no available path - failing I/O")}) + found = list(build_detector("kernel.path-loss").detect(ev)) + crit = [f for f in found if f.severity == Severity.CRITICAL] + self.assertEqual(len(crit), 1) + self.assertEqual(crit[0].evidence["failing_io"], 1) + + def test_failfast_is_reported_separately_as_the_boundary(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg("nvme nvme10: failfast expired")}) + found = list(build_detector("kernel.path-loss").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIn("fast_io_fail_tmo", found[0].title) + + def test_clean_dmesg_yields_nothing(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg("nvme nvme1: creating 3 I/O queues.")}) + self.assertEqual(list(build_detector("kernel.path-loss").detect(ev)), []) + + def test_skips_without_dmesg(self): + with self.assertRaises(SkipDetector): + list(build_detector("kernel.path-loss").detect(FakeEvidence())) + + +class KernelFilesystemShutdown(unittest.TestCase): + def test_shutdown_is_critical_and_names_the_devices(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "XFS (nvme0n1): log I/O error -5", + "XFS (nvme0n1): Filesystem has been shut down due to log error (0x2).", + "XFS (nvme1n6): Filesystem has been shut down due to log error (0x2).")}) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].evidence["devices"], ["nvme0n1", "nvme1n6"]) + + def test_io_errors_without_a_shutdown_are_only_a_warning(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg("XFS (nvme0n1): metadata I/O error")}) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual([f.severity for f in found], [Severity.WARNING]) + + def test_healthy_mount_messages_are_not_findings(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "XFS (nvme0n1): Ending clean mount", + "XFS (nvme0n1): Unmounting Filesystem abc")}) + self.assertEqual(list(build_detector("kernel.filesystem-shutdown").detect(ev)), []) + + +class NvmeForeignCluster(unittest.TestCase): + LIVE = "d26b8f37-2b45-47c0-9d20-983e6c5ee3fe" + DEAD = "5fd9ad70-3cd1-4fc8-b8e6-e085081601f6" + + def test_a_dead_cluster_being_retried_is_reported_as_hygiene(self): + """The sharpest form of 'controllers never disappear': they outlive the cluster. + + Reported, but never as this run's failure — a controller for a destroyed cluster + cannot affect a migration of the live cluster's subsystems. See NvmeDirtyStart for + the leak that does invalidate a run. + """ + ev = FakeEvidence(cluster=self.LIVE, logs={"dmesg-vm03": dmesg( + f'nvme nvme6: Connect Invalid Data Parameter, subsysnqn "nqn.2023-02.io.simplyblock:{self.DEAD}:lvol:x"', + f'nvme nvme6: Connect Invalid Data Parameter, subsysnqn "nqn.2023-02.io.simplyblock:{self.DEAD}:lvol:x"', + f'nvme nvme7: connected to nqn.2023-02.io.simplyblock:{self.LIVE}:lvol:y')}) + found = list(build_detector("nvme.foreign-cluster").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.WARNING) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + self.assertFalse(found[0].counts_against_the_run) + self.assertEqual(found[0].evidence["foreign"], {self.DEAD: 2}) + self.assertEqual(found[0].evidence["live_cluster"], self.LIVE) + + def test_only_the_live_cluster_is_clean(self): + ev = FakeEvidence(cluster=self.LIVE, logs={"dmesg-vm03": dmesg( + f'nvme nvme7: connected to nqn.2023-02.io.simplyblock:{self.LIVE}:lvol:y')}) + self.assertEqual(list(build_detector("nvme.foreign-cluster").detect(ev)), []) + + def test_skips_when_the_live_cluster_is_unknown(self): + """Without it a foreign NQN cannot be told from the live one, so do not guess.""" + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + f'nvme nvme6: nqn.2023-02.io.simplyblock:{self.DEAD}:lvol:x')}) + with self.assertRaises(SkipDetector): + list(build_detector("nvme.foreign-cluster").detect(ev)) + + +class NvmeControllerChurn(unittest.TestCase): + def test_created_but_never_removed_is_reported(self): + msgs = [f"nvme nvme{i}: new ctrl: NQN \"nqn.x\"" for i in range(8)] + msgs.append("nvme nvme0: Removing ctrl: NQN \"nqn.x\"") + found = list(build_detector("nvme.controller-churn").detect( + FakeEvidence(logs={"dmesg-vm03": dmesg(*msgs)}))) + churn = [f for f in found if "never removed" in f.title] + self.assertEqual(len(churn), 1) + self.assertEqual(churn[0].evidence["net"], 7) + + def test_balanced_churn_is_only_info(self): + msgs = ["nvme nvme0: new ctrl: NQN \"nqn.x\"", "nvme nvme0: Removing ctrl: NQN \"nqn.x\""] + found = list(build_detector("nvme.controller-churn").detect( + FakeEvidence(logs={"dmesg-vm03": dmesg(*msgs)}))) + self.assertTrue(all(f.severity == Severity.INFO for f in found)) + + def test_a_controller_retrying_forever_is_reported(self): + found = list(build_detector("nvme.controller-churn").detect( + FakeEvidence(logs={"dmesg-vm03": dmesg( + "nvme nvme9: Failed reconnect attempt 5", + "nvme nvme9: Failed reconnect attempt 834")}))) + stuck = [f for f in found if "without ever succeeding" in f.title] + self.assertEqual(len(stuck), 1) + self.assertEqual(stuck[0].evidence["controllers"], {"nvme9": 834}) + + +class KernelFabricErrors(unittest.TestCase): + def test_groups_by_kind_and_respects_per_kind_floors(self): + ev = FakeEvidence(logs={"dmesg-vm03": dmesg( + "nvme nvme1: starting error recovery", + "nvme nvme1: Property Set error: 880, offset 0x14", + "nvme nvme2: rescanning namespaces.")}) + found = list(build_detector("kernel.fabric-errors").detect(ev)) + self.assertEqual(len(found), 1) + counts = found[0].evidence["counts"] + # a single rescan is normal and must not be reported; the two errors must be + self.assertNotIn("namespace rescan", counts) + self.assertIn("error recovery started", counts) + self.assertIn("property set failed (controller config)", counts) + + +class Attribution_(unittest.TestCase): + """Old, unrelated damage must not be counted against a run. + + dmesg spans hours and a cluster outlives its runs, so evidence routinely contains the + previous runs' mess. These pin the rule: only what happened inside the window counts. + """ + + RUN_START = datetime(2026, 8, 20, 6, 0, 0, tzinfo=UTC) + RUN_END = datetime(2026, 8, 20, 7, 0, 0, tzinfo=UTC) + + def ev(self, *msgs: str, hour: int = 6) -> FakeEvidence: + lines = [f"2026-08-20T{hour:02d}:30:0{i},000000+00:00 {m}\n" for i, m in enumerate(msgs)] + return FakeEvidence(logs={"dmesg-vm03": lines}, + window=(self.RUN_START, self.RUN_END)) + + def test_damage_inside_the_window_is_critical_and_fails(self): + ev = self.ev("block nvme0n1: no available path - failing I/O", hour=6) + found = [f for f in build_detector("kernel.path-loss").detect(ev) + if f.severity == Severity.CRITICAL] + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, Attribution.RUN) + self.assertTrue(found[0].counts_against_the_run) + + def test_the_same_damage_before_the_window_does_not_fail(self): + ev = self.ev("block nvme0n1: no available path - failing I/O", hour=5) + found = list(build_detector("kernel.path-loss").detect(ev)) + self.assertTrue(found) + self.assertTrue(all(f.attribution is Attribution.PRE_EXISTING for f in found)) + self.assertFalse(any(f.severity == Severity.CRITICAL for f in found)) + self.assertFalse(any(f.counts_against_the_run for f in found)) + + def test_a_filesystem_killed_before_the_run_is_hygiene_not_failure(self): + ev = self.ev("XFS (nvme0n1): Filesystem has been shut down due to log error (0x2).", + hour=5) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + self.assertEqual(found[0].severity, Severity.WARNING) + + def test_a_filesystem_killed_during_the_run_is_critical(self): + ev = self.ev("XFS (nvme0n1): Filesystem has been shut down due to log error (0x2).", + hour=6) + found = list(build_detector("kernel.filesystem-shutdown").detect(ev)) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertIs(found[0].attribution, Attribution.RUN) + + def test_undated_events_still_count(self): + """'I cannot date this' must not become 'not our problem'.""" + ev = FakeEvidence(logs={"dmesg-vm03": ["block nvme0n1: no available path - failing I/O\n"]}, + window=(self.RUN_START, self.RUN_END)) + found = [f for f in build_detector("kernel.path-loss").detect(ev) + if f.severity == Severity.CRITICAL] + self.assertEqual(len(found), 1) + self.assertIs(found[0].attribution, Attribution.UNKNOWN) + self.assertTrue(found[0].counts_against_the_run) + + def test_dead_cluster_debris_is_hygiene_never_a_verdict(self): + """It cannot make a different cluster's migration fail, so it must not fail the run.""" + live, dead = "d26b8f37-2b45-47c0-9d20-983e6c5ee3fe", "5fd9ad70-3cd1-4fc8-b8e6-e085081601f6" + ev = FakeEvidence(cluster=live, window=(self.RUN_START, self.RUN_END), logs={ + "dmesg-vm03": [f'nvme nvme6: subsysnqn "nqn.2023-02.io.simplyblock:{dead}:lvol:x"\n'] * 100}) + found = list(build_detector("nvme.foreign-cluster").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.WARNING) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + + +class NvmeDirtyStart(unittest.TestCase): + """The one pre-existing condition that does forfeit a run.""" + + LIVE = "d26b8f37-2b45-47c0-9d20-983e6c5ee3fe" + DEAD = "5fd9ad70-3cd1-4fc8-b8e6-e085081601f6" + + class WithPre(FakeEvidence): + """FakeEvidence plus the optional pre-run snapshot nvme.dirty-start asks for.""" + + def __init__(self, pre: list[NvmeController], cluster: str = "") -> None: + super().__init__(cluster=cluster) + self._pre = pre + + def nvme_controllers_pre(self) -> list[NvmeController]: + return self._pre + + def ctrl(self, nqn: str, state: str, ns: dict) -> NvmeController: + return NvmeController(node="vm03", name="nvme12", nqn=nqn, address="10.0.0.1:4420", + state=state, namespaces=ns, ctrl_loss_tmo=60) + + def test_live_cluster_debris_at_setup_forfeits_the_run(self): + ev = self.WithPre([self.ctrl(f"nqn:{self.LIVE}:lvol:x", "live", {})], + cluster=self.LIVE) + found = list(build_detector("nvme.dirty-start").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertIs(found[0].attribution, Attribution.PRE_EXISTING) + # critical + pre-existing => the run is inconclusive, not failed + rep = Report() + rep.add(*found) + self.assertEqual(rep.verdict, "INCONCLUSIVE") + self.assertFalse(rep.failed) + + def test_dead_cluster_debris_at_setup_does_not_forfeit(self): + ev = self.WithPre([self.ctrl(f"nqn:{self.DEAD}:lvol:x", "live", {})], + cluster=self.LIVE) + self.assertEqual(list(build_detector("nvme.dirty-start").detect(ev)), []) + + def test_a_connecting_controller_does_not_forfeit(self): + ev = self.WithPre([self.ctrl(f"nqn:{self.LIVE}:lvol:x", "connecting", {})], + cluster=self.LIVE) + self.assertEqual(list(build_detector("nvme.dirty-start").detect(ev)), []) + + def test_skips_without_a_pre_snapshot(self): + with self.assertRaises(SkipDetector): + list(build_detector("nvme.dirty-start").detect(FakeEvidence(cluster=self.LIVE))) + + +# ── control-plane and evidence families ───────────────────────────────────────────── + +def cevent(sec: int, msg: str, subject: str = "node-a", level: str = "Info") -> ControlEvent: + return ControlEvent(ts=ts(sec), level=level, kind="STATUS_CHANGE", message=msg, + subject=subject) + + +class ControlNodeFlap(unittest.TestCase): + """The vela shape: a node marked down and back in seconds because the liveness check + depended on something other than the node.""" + + def test_a_short_flap_is_critical(self): + ev = FakeEvidence(events=[ + cevent(0, "Storage node status changed from: online to: down"), + cevent(13, "Storage node status changed from: down to: online")]) + found = list(build_detector("control.node-flap").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + self.assertEqual(found[0].evidence["flaps"][0]["seconds"], 13) + + def test_a_node_that_stays_down_is_not_a_flap(self): + ev = FakeEvidence(events=[ + cevent(0, "Storage node status changed from: online to: down")]) + self.assertEqual(list(build_detector("control.node-flap").detect(ev)), []) + + def test_a_slow_recovery_is_not_a_flap(self): + ev = FakeEvidence(events=[ + cevent(0, "Storage node status changed from: online to: down"), + cevent(9000, "Storage node status changed from: down to: online")]) + self.assertEqual(list(build_detector("control.node-flap").detect(ev)), []) + + def test_skips_without_the_event_log(self): + with self.assertRaises(SkipDetector): + list(build_detector("control.node-flap").detect(FakeEvidence())) + + +class ControlVolumeHealth(unittest.TestCase): + def test_health_that_never_returns_is_critical(self): + ev = FakeEvidence(events=[ + cevent(0, "LVol health check changed from: True to: False", subject="vol-1"), + cevent(5, "LVol health check changed from: True to: False", subject="vol-2"), + cevent(60, "LVol health check changed from: False to: True", subject="vol-2")]) + found = list(build_detector("control.volume-health").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["unhealthy"], ["vol-1"]) + + def test_all_recovered_is_clean(self): + ev = FakeEvidence(events=[ + cevent(0, "LVol health check changed from: True to: False", subject="vol-1"), + cevent(60, "LVol health check changed from: False to: True", subject="vol-1")]) + self.assertEqual(list(build_detector("control.volume-health").detect(ev)), []) + + +class EvidenceCoverage(unittest.TestCase): + """Partial evidence is not the same as clean evidence.""" + + START = datetime(2026, 8, 20, 7, 0, 0, tzinfo=UTC) + END = datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC) + + def test_a_log_that_misses_the_start_is_reported(self): + ev = FakeEvidence(window=(self.START, self.END), spans=[ + LogSpan("spdk-4424", self.START + timedelta(minutes=114), self.END, 100), + LogSpan("operator", self.START, self.END, 100)]) + found = list(build_detector("evidence.log-coverage").detect(ev)) + self.assertEqual(len(found), 1) + self.assertIn("spdk-4424", found[0].evidence["logs"]) + self.assertNotIn("operator", found[0].evidence["logs"]) + + def test_full_coverage_is_clean(self): + ev = FakeEvidence(window=(self.START, self.END), spans=[ + LogSpan("operator", self.START, self.END, 100)]) + self.assertEqual(list(build_detector("evidence.log-coverage").detect(ev)), []) + + def test_skips_without_a_run_window(self): + ev = FakeEvidence(spans=[LogSpan("x", self.START, self.END, 1)]) + with self.assertRaises(SkipDetector): + list(build_detector("evidence.log-coverage").detect(ev)) + + def test_a_migration_no_log_covers_is_a_blind_spot(self): + """The real case: the corrupting migration ended six seconds before a log began.""" + mig_start = self.START + timedelta(minutes=110) + ev = FakeEvidence( + window=(self.START, self.END), + migrations=[Migration(name="mig-19", start=mig_start, + end=mig_start + timedelta(minutes=2))], + spans=[LogSpan("spdk-4424", mig_start + timedelta(minutes=2, seconds=6), + self.END, 100)]) + found = list(build_detector("evidence.blind-spot").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["blind"], {"mig-19": ["spdk-4424"]}) + + +class SecuritySecretExposure(unittest.TestCase): + def test_finds_a_private_key_without_quoting_it(self): + secret = "-----BEGIN RSA PRIVATE KEY-----" + ev = FakeEvidence(logs={"operator": [f"oops {secret} MIIEow\n"]}) + found = list(build_detector("security.secret-exposure").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].severity, Severity.CRITICAL) + blob = json.dumps(found[0].to_dict()) + self.assertNotIn("BEGIN RSA", blob) # the value must not travel with the finding + self.assertIn("operator:1", found[0].detail) + + def test_finds_a_dhchap_secret(self): + ev = FakeEvidence(logs={"operator": [ + "connect --dhchap-secret DHHC-1:00:abcdefghijklmnopqrstuvwxyz012345+/=\n"]}) + found = list(build_detector("security.secret-exposure").detect(ev)) + self.assertEqual([f.subject for f in found], ["nvme dhchap secret"]) + + def test_ordinary_logs_are_clean(self): + ev = FakeEvidence(logs={"operator": ["migration started for volume abc\n"]}) + self.assertEqual(list(build_detector("security.secret-exposure").detect(ev)), []) diff --git a/test/framework/tests/test_driver.py b/test/framework/tests/test_driver.py new file mode 100644 index 000000000..18773b260 --- /dev/null +++ b/test/framework/tests/test_driver.py @@ -0,0 +1,384 @@ +"""Driver and workload tests. + +These cover the decisions the driver makes on its own — which node to migrate to, which +volumes move together, what a non-terminal poll means — because those are the parts that +silently produce a *valid-looking* run when they are wrong. A driver that always picks the +same target still completes migrations and still reports PASS; the run just never exercised +the case it claimed to. + +Everything here fakes kubectl at the module boundary. That is the whole point of routing +cluster access through one `kube.run`: the logic above it stays testable without a cluster. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from datetime import UTC, datetime, timedelta + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import sbtest # noqa: E402,F401 +from sbtest.components import kube, migration, workload # noqa: E402 +from sbtest.core import Logger, Migration, RunContext # noqa: E402 + + +def _cp(stdout: str = "", rc: int = 0) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(args=["kubectl"], returncode=rc, stdout=stdout, + stderr="") + + +def _nodes_json(*specs: tuple[str, str, str]) -> str: + """specs: (uuid, k8s host, status).""" + return json.dumps({"items": [ + {"spec": {"workerNode": host}, "status": {"uuid": uuid, "status": status}} + for uuid, host, status in specs]}) + + +class _FakeKube: + """Records every kubectl invocation and answers from a scripted table.""" + + def __init__(self, answers: dict[str, str] | None = None) -> None: + self.calls: list[list[str]] = [] + self.stdins: list[str | None] = [] + self.answers = answers or {} + + def run(self, args: list[str], timeout: int = 60, check: bool = True, + stdin: str | None = None) -> subprocess.CompletedProcess[str]: + self.calls.append(list(args)) + self.stdins.append(stdin) + for key, out in self.answers.items(): + if key in " ".join(args): + return _cp(out) + return _cp("") + + +class _Ctx: + """A RunContext with a real temp dir, for components that write artifacts.""" + + def __enter__(self) -> RunContext: + self._d = tempfile.TemporaryDirectory() + self.ctx = RunContext(run_id="run1", outdir=self._d.__enter__(), log=Logger(None)) + return self.ctx + + def __exit__(self, *exc: object) -> None: + self._d.__exit__(*exc) # type: ignore[arg-type] + + +class TargetPolicy(unittest.TestCase): + """The policy decides whether the target also hosts a consumer — the harder case.""" + + def _driver(self, ctx: RunContext, policy: str, + consumers: dict[str, list[str]]) -> migration.MigrationDriver: + d = migration.MigrationDriver(target_policy=policy) + d._nodes = ["uuid-a", "uuid-b", "uuid-c"] + d._node_host = {"uuid-a": "vm02", "uuid-b": "vm03", "uuid-c": "vm04"} + # pv -> pod -> node, which is how the driver learns where consumers run. + ctx.shared["workload.pod_of"] = {"pv1": "fio-0"} + ctx.shared["workload.node_of"] = {"fio-0": next(iter(consumers), "")} + return d + + def test_consumer_policy_picks_a_node_running_a_consumer(self): + with _Ctx() as ctx: + d = self._driver(ctx, "consumer", {"vm03": ["fio-0"]}) + target, policy, named = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertEqual(target, "uuid-b") # vm03, where fio-0 runs + self.assertEqual(policy, "consumer") + self.assertEqual(named, ["fio-0"]) + + def test_no_consumer_policy_avoids_it(self): + with _Ctx() as ctx: + d = self._driver(ctx, "no-consumer", {"vm03": ["fio-0"]}) + target, policy, named = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertEqual(target, "uuid-c") # vm04: the only other non-consumer + self.assertEqual(policy, "no-consumer") + self.assertEqual(named, []) + + def test_alternate_starts_with_the_harder_case(self): + """Odd migrations get `consumer`, so a run cut short still exercised it.""" + with _Ctx() as ctx: + d = self._driver(ctx, "alternate", {"vm03": ["fio-0"]}) + _, first, _ = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + _, second, _ = d._pick_target(ctx, ["pv1"], idx=2, source="uuid-a") + self.assertEqual(first, "consumer") + self.assertEqual(second, "no-consumer") + + def test_unmet_policy_falls_back_and_says_so(self): + """A migration under the other condition is still evidence; skipping it is not. + + The recorded policy has to show it was unmet, or the run's own record claims a case + it never exercised. + """ + with _Ctx() as ctx: + d = self._driver(ctx, "consumer", {}) # no consumer anywhere + target, policy, _ = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertIn(target, ("uuid-b", "uuid-c")) + self.assertEqual(policy, "consumer(unmet)") + + def test_the_source_is_never_the_target(self): + with _Ctx() as ctx: + d = self._driver(ctx, "random", {}) + for _ in range(20): + target, _, _ = d._pick_target(ctx, ["pv1"], idx=1, source="uuid-a") + self.assertNotEqual(target, "uuid-a") + + def test_consumers_are_counted_across_the_whole_subsystem(self): + """Every pod holding any namespace of the subsystem has its paths moved.""" + with _Ctx() as ctx: + d = migration.MigrationDriver(target_policy="consumer") + d._nodes = ["uuid-a", "uuid-b"] + d._node_host = {"uuid-a": "vm02", "uuid-b": "vm03"} + ctx.shared["workload.pod_of"] = {"pv1": "fio-0", "pv2": "fio-1"} + ctx.shared["workload.node_of"] = {"fio-0": "vm02", "fio-1": "vm03"} + _, _, named = d._pick_target(ctx, ["pv1", "pv2"], idx=1, source="uuid-a") + self.assertEqual(named, ["fio-1"]) # the sibling on the target counts + + +class Grouping(unittest.TestCase): + """A migration moves the whole subsystem, so the group is what must be tracked.""" + + def test_group_is_the_shared_subsystem(self): + d = migration.MigrationDriver() + d._nqn_of = {"pv1": "nqn.a", "pv2": "nqn.a", "pv3": "nqn.b"} + d._groups = d._regroup() + self.assertEqual(d._group_of("pv1"), ["pv1", "pv2"]) + self.assertEqual(d._group_of("pv3"), ["pv3"]) + + def test_an_unknown_subsystem_migrates_alone(self): + d = migration.MigrationDriver() + self.assertEqual(d._group_of("pv9"), ["pv9"]) + + def test_a_changed_subsystem_regroups(self): + """A previous migration can repack the subsystems; a stale group samples the wrong + nodes and verifies the wrong volumes.""" + class FakeSb: + def subsystem_of(self, lvol: str) -> tuple[str, int]: + return "nqn.new", 1 + + with _Ctx() as ctx: + d = migration.MigrationDriver() + d._sb = FakeSb() # type: ignore[assignment] + d._volume_of = {"pv1": "lvol-1"} + d._nqn_of = {"pv1": "nqn.old", "pv2": "nqn.old"} + d._groups = d._regroup() + self.assertEqual(d._group_of("pv1"), ["pv1", "pv2"]) + d._reread_subsystem(ctx, "pv1") + self.assertEqual(d._nqn_of["pv1"], "nqn.new") + self.assertEqual(d._group_of("pv1"), ["pv1"]) # no longer with pv2 + + +class PollLoop(unittest.TestCase): + def test_terminal_phase_ends_the_wait(self): + fake = _FakeKube({"get volumemigration": json.dumps( + {"status": {"phase": "Completed", "sourceNodeUUID": "uuid-real"}})}) + with _Ctx() as ctx, _patch(kube, "run", fake.run): + d = migration.MigrationDriver(poll_s=0.01) + rec = Migration(name="m1", start=datetime.now(UTC), pv="pv1", source="uuid-guess") + d._await_terminal(ctx, rec, "m1", sampler=None) + self.assertEqual(rec.phase, "Completed") + self.assertIsNotNone(rec.end) + # The operator's resolved source overrides the driver's guess. + self.assertEqual(rec.source, "uuid-real") + + def test_never_reaching_a_terminal_phase_is_a_timeout_not_a_failure(self): + """A rejected migration and one that never finished are different defects, and only + one of them has an error to read.""" + fake = _FakeKube({"get volumemigration": json.dumps( + {"status": {"phase": "Migrating"}})}) + with _Ctx() as ctx, _patch(kube, "run", fake.run): + d = migration.MigrationDriver(poll_s=0.01, timeout_s=0.05) + rec = Migration(name="m1", start=datetime.now(UTC), pv="pv1") + d._await_terminal(ctx, rec, "m1", sampler=None) + self.assertEqual(rec.phase, "TIMEOUT") + self.assertEqual(rec.error, "") + + def test_phase_changes_reach_the_sampler_and_the_timeline(self): + class Sampler: + def __init__(self) -> None: + self.phases: list[str] = [] + + def set_phase(self, p: str) -> None: + self.phases.append(p) + + seq = [json.dumps({"status": {"phase": p}}) + for p in ("Preparing", "Preparing", "Cutover", "Completed")] + + def run(args: list[str], timeout: int = 60, check: bool = True, + stdin: str | None = None) -> subprocess.CompletedProcess[str]: + return _cp(seq.pop(0) if seq else "") + + s = Sampler() + with _Ctx() as ctx, _patch(kube, "run", run): + d = migration.MigrationDriver(poll_s=0.01) + rec = Migration(name="m1", start=datetime.now(UTC), pv="pv1") + d._await_terminal(ctx, rec, "m1", sampler=s) + # Deduplicated: only transitions, not every poll. + self.assertEqual(s.phases, ["Preparing", "Cutover", "Completed"]) + self.assertEqual([e.data["phase"] for e in ctx.timeline.of_kind("migration.phase")], + ["Preparing", "Cutover", "Completed"]) + + +class Manifest(unittest.TestCase): + def test_the_cr_carries_the_run_label_so_teardown_can_find_it(self): + d = migration.MigrationDriver(namespace="default") + m = json.loads(d._manifest("run1-mig-3", "pvc-abc", "uuid-b")) + self.assertEqual(m["kind"], "VolumeMigration") + self.assertEqual(m["spec"], {"pvName": "pvc-abc", "targetNodeUUID": "uuid-b"}) + self.assertEqual(m["metadata"]["labels"]["sbtest-run"], "true") + + def test_online_nodes_only(self): + """Migrating to an offline node is a rejected request, not a test.""" + fake = _FakeKube({"get storagenodes": _nodes_json( + ("uuid-a", "vm02", "online"), ("uuid-b", "vm03", "offline"), + ("uuid-c", "vm04", "online"))}) + with _patch(kube, "run", fake.run): + uuids, hosts = migration.MigrationDriver()._storage_nodes() + self.assertEqual(uuids, ["uuid-a", "uuid-c"]) + self.assertEqual(hosts["uuid-c"], "vm04") + + +class SetupGuards(unittest.TestCase): + def test_one_node_cannot_be_migrated_between(self): + fake = _FakeKube({"get storagenodes": _nodes_json(("uuid-a", "vm02", "online"))}) + with _Ctx() as ctx, _patch(kube, "run", fake.run), \ + self.assertRaises(RuntimeError) as e: + migration.MigrationDriver().setup(ctx) + self.assertIn("at least two", str(e.exception)) + + def test_no_volumes_is_an_explicit_error_not_an_empty_run(self): + """Silently migrating nothing would report PASS for a test that never ran.""" + fake = _FakeKube({"get storagenodes": _nodes_json( + ("uuid-a", "vm02", "online"), ("uuid-b", "vm03", "online"))}) + with _Ctx() as ctx, _patch(kube, "run", fake.run), \ + self.assertRaises(RuntimeError) as e: + migration.MigrationDriver().setup(ctx) + self.assertIn("no volumes to migrate", str(e.exception)) + + +class Persistence(unittest.TestCase): + def test_migrations_round_trip_through_the_file(self): + """What the driver writes is what the analyser reads — the seam that lets a run be + re-judged later against a detector that did not exist when it ran.""" + with _Ctx() as ctx: + d = migration.MigrationDriver() + start = datetime.now(UTC).replace(microsecond=0) + d._records = [Migration(name="run1-mig-1", start=start, + end=start + timedelta(seconds=42), phase="Completed", + source="uuid-a", target="uuid-b", pv="pv1", pod="fio-0", + members=["pv1", "pv2"]), + Migration(name="run1-mig-2", start=start + timedelta(minutes=1), + phase="TIMEOUT", pv="pv3")] + d._nqn_of = {"pv1": "nqn.a"} + d.collect(ctx) + back = migration.migrations_from_file(os.path.join(ctx.outdir, + "migrations.json")) + self.assertEqual([m.name for m in back], ["run1-mig-1", "run1-mig-2"]) + self.assertEqual(back[0].members, ["pv1", "pv2"]) + self.assertTrue(back[0].batch) + self.assertEqual(back[0].phase, "Completed") + self.assertIsNotNone(back[0].end) + assert back[0].end is not None + self.assertEqual((back[0].end - back[0].start).total_seconds(), 42) + self.assertIsNone(back[1].end) + + def test_a_record_without_a_start_is_dropped_rather_than_guessed(self): + with _Ctx() as ctx: + p = os.path.join(ctx.outdir, "migrations.json") + with open(p, "w") as fh: + json.dump([{"name": "broken", "phase": "Completed"}], fh) + self.assertEqual(migration.migrations_from_file(p), []) + + +class WorkloadFio(unittest.TestCase): + def test_verification_is_off_when_it_cannot_be_trusted(self): + """numjobs>1 cannot serialize overlapping writes, so verify would report corruption + that never happened. A throughput run must not look like an integrity run.""" + with _Ctx() as ctx: + single = workload.FioWorkload(numjobs=1, iodepth=8)._fio_script(ctx) + multi = workload.FioWorkload(numjobs=4, iodepth=8)._fio_script(ctx) + self.assertIn("--verify=md5", single) + self.assertIn("--serialize_overlap=1", single) + self.assertNotIn("--verify=md5", multi) + + def test_verify_is_not_fatal_by_default_so_every_bad_block_is_counted(self): + with _Ctx() as ctx: + s = workload.FioWorkload(numjobs=1)._fio_script(ctx) + self.assertIn("--verify_fatal=0", s) + + def test_the_file_stays_inside_the_volume(self): + with _Ctx() as ctx: + s = workload.FioWorkload(volume_size_gb=10, file_size_gb=50)._fio_script(ctx) + self.assertIn("--size=8G", s) # 10 - 2 of filesystem headroom + + def test_logs_live_off_the_volume_under_test(self): + """Collecting the evidence must not depend on the health of what it is about.""" + with _Ctx() as ctx: + s = workload.FioWorkload()._fio_script(ctx) + self.assertIn("--output=/logs/result.json", s) + self.assertIn("--filename=/data/fiotest", s) + + def test_timeseries_is_written_where_the_analyser_reads_it(self): + raw = "\n".join([ + "1000, 500, 0, 4096", # 1s: 500 read + "1000, 100, 1, 4096", # 1s: 100 write + "2000, 400, 0, 4096", + ]) + start = datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC) + migs = [Migration(name="mig-1", start=start + timedelta(seconds=1), + end=start + timedelta(seconds=1), pv="pv1")] + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=start) + w = workload.FioWorkload() + d = ctx.dir("fio-0") + w._write_timeseries(ctx, "default", "fio-0", d, migs) + with open(os.path.join(d, "timeseries.csv")) as fh: + rows = list(__import__("csv").DictReader(fh)) + self.assertEqual(rows[0]["second"], "1") + self.assertEqual(float(rows[0]["total_iops"]), 600.0) + self.assertEqual(rows[0]["wall_clock"], "2026-08-20T09:00:01Z") + # The migration column is the point: it makes a dip attributable. + self.assertEqual(rows[0]["active_migration"], "mig-1") + self.assertEqual(rows[1]["active_migration"], "") + + def test_the_analyser_reads_back_what_the_workload_wrote(self): + """Guards the column-name seam. Reading only fio's own names silently placed every + sample at offset 0 — the series stayed the right length with the whole run collapsed + onto one instant, so nothing looked like a parse failure.""" + from sbtest.adapters import ArchiveEvidence + raw = "1000, 500, 0, 4096\n2000, 400, 0, 4096" + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC)) + workload.FioWorkload()._write_timeseries( + ctx, "default", "run1-fio-0", ctx.dir("run1-fio-0"), []) + series = ArchiveEvidence(ctx.outdir).fio_timeseries("run1-fio-0") + self.assertEqual([s.offset_s for s in series], [1, 2]) + self.assertEqual([s.total_iops for s in series], [500.0, 400.0]) + self.assertIsNotNone(series[0].wall) + + def test_a_workload_with_no_pods_is_refused(self): + with _Ctx() as ctx, self.assertRaises(RuntimeError) as e: + workload.FioWorkload(pods=0, ns_pods=0)._create(ctx) + self.assertIn("no I/O", str(e.exception)) + + +class _patch: + """Minimal attribute patcher — the stdlib one needs a dotted target string.""" + + def __init__(self, obj: object, attr: str, value: object) -> None: + self.obj, self.attr, self.value = obj, attr, value + + def __enter__(self) -> object: + self.old = getattr(self.obj, self.attr) + setattr(self.obj, self.attr, self.value) + return self.value + + def __exit__(self, *exc: object) -> None: + setattr(self.obj, self.attr, self.old) + + +if __name__ == "__main__": + unittest.main() From af8e47fcc402e3690def3978ea35e36a773230ca Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Wed, 26 Aug 2026 17:27:55 +0200 Subject: [PATCH 2/3] Give an outage the migration it belongs to, and say whether the volume came back MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ported from the fio harness's own fix (058bb005), which found three things wrong with how a stopped volume was read. All three were here too, and the last one silently. fio's log offsets are milliseconds since *that job* started, so the only base that is right by construction is the job's own start. The workload component used the run's start instead — stamped before the PVCs, the pods and the fio processes existed, so ahead of every pod's fio by a different amount: 207s and 210s in the two archived runs that still carry a pre-fix timeseries.csv. That is not merely a shifted chart. The same base decides which migration an outage overlaps, so a wrong base names a wrong migration. ArchiveEvidence re-derives the base rather than trusting the wall_clock column, because the column is only as right as whatever wrote it, and a detector that cannot be re-run against the run that motivated it is a detector nobody trusts. Replaying fio-mig-1787557273 with the base corrected moves 90 of its 160 outage windows onto a different migration, every one of them from the migration after the one that caused it. A window was then attributed by its first second, which fails whenever the host goes dry before the operator records the migration as started — those gaps came out belonging to no migration at all, which reads as "the cluster is unwell" rather than "the cutover cost this". attribute_window measures the overlap instead and names the migration holding most of the window, so a long gap touching two of them lands on the one it mostly sits in. Last, fio.outage called both kinds of stopped volume an outage. A gap that recovered is a freeze: every write the application issued was eventually taken, and what the length measures is how long an application had to survive with the volume gone — a cutover is one by design. A gap still open when fio stopped is a loss: I/O the volume was supposed to accept and never did. Both are CRITICAL, both fail the run, but they need different next steps, and a report that gives them one name hands the reader that work. The tests were written against the unchanged code first: the clock tests failed on the three-minute shift, the attribution tests on ['mig-a'] != ['mig-b'] and [] != ['mig-1'], and the freeze/loss tests on the missing `kind`. Co-Authored-By: Claude Opus 5 (1M context) --- test/framework/README.md | 10 +- test/framework/sbtest/adapters/archive.py | 53 +++++-- test/framework/sbtest/components/workload.py | 32 ++++- test/framework/sbtest/core/__init__.py | 4 +- test/framework/sbtest/core/evidence.py | 28 ++++ test/framework/sbtest/detectors/fio.py | 143 ++++++++++++++----- test/framework/tests/test_detectors.py | 78 ++++++++++ test/framework/tests/test_driver.py | 58 ++++++++ 8 files changed, 352 insertions(+), 54 deletions(-) diff --git a/test/framework/README.md b/test/framework/README.md index edd135306..f0a7cad97 100644 --- a/test/framework/README.md +++ b/test/framework/README.md @@ -279,7 +279,7 @@ Every one of these came from a real defect. Defaults encode what the runs measur | `ana.path-churn` | More distinct path addresses per host than the topology should produce. Informational: healthy counts are topology-dependent. | | `fio.checksum` | **fio read back data it never wrote.** Reads succeeded, so nothing else notices. Attributes to a migration through a verify lag (see below). | | `fio.job-error` | An fio job ended with a non-zero errno, with the errno's meaning — 121/EREMOTEIO points straight at the ANA detectors. | -| `fio.outage` | A pod's I/O stopped for longer than a cutover should cost. | +| `fio.outage` | A pod's I/O stopped for longer than a cutover should cost — reported as a **freeze** when it came back and a **loss** when it never did. Both fail; only one means writes went missing. | | `fio.throughput-outlier` | A pod far below the run's median IOPS. Weak alone; strong next to an ANA finding on the same subject. | | `logs.pattern` | **User-definable regex checks over any collected log.** Ships a catalogue: undrained transfer, migration sub-task failure, host-not-allowed reconnect storm, write-to-RO-range, path-validation failure, stuck migration group, kernel reconnect loop. | | `migration.outcomes` | Completion rate and phase breakdown. | @@ -302,7 +302,7 @@ Every one of these came from a real defect. Defaults encode what the runs measur | `evidence.inventory` | What evidence the run produced (INFO). | | `security.secret-exposure` | Credential-shaped strings in collected logs. Reports the location, never the value. | -Two defaults are load-bearing and worth knowing: +Three things are load-bearing and worth knowing: * **`fio.checksum` verify lag (45s).** fio detects a lost write when it next *reads* that block, 3–34s later in practice. Without the lag a migration's own losses are filed under @@ -311,6 +311,12 @@ Two defaults are load-bearing and worth knowing: freezes look like one healthy pause on any longest-window measure), more specific (one migration's single 5–6s pause lost nothing), and robust to sampling granularity. Keep both; they catch different shapes, and `tests/test_detectors.py` pins exactly that. +* **The fio clock is fio's own.** Every offset in `timeseries.csv` is milliseconds since + *that pod's* fio started, so the wall clock hangs off `job_start` from its `result.json`, + never off the run's start — the run begins minutes before any fio does, by a different + amount per pod. The base decides which migration an outage overlaps, so getting it wrong + does not merely shift a chart: it names the wrong migration. `ArchiveEvidence` re-derives + it on replay, which corrects archives written before this was fixed. ## Component catalogue diff --git a/test/framework/sbtest/adapters/archive.py b/test/framework/sbtest/adapters/archive.py index b221b7a22..af49cdcbb 100644 --- a/test/framework/sbtest/adapters/archive.py +++ b/test/framework/sbtest/adapters/archive.py @@ -29,7 +29,7 @@ import os import re from collections.abc import Iterator -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from ..core import ( AnaSample, @@ -86,6 +86,7 @@ def __init__(self, outdir: str) -> None: self.run_id = self._state.get("run_id") or os.path.basename(self.outdir).replace( "fio-mig-", "fiomig-") self._ana_cache: dict[str, list[AnaSample]] = {} + self._fio_start_cache: dict[str, datetime | None] = {} # ── bookkeeping ──────────────────────────────────────────────────────────────── @@ -213,16 +214,45 @@ def pods(self) -> list[str]: d for d in os.listdir(self.outdir) if os.path.isdir(os.path.join(self.outdir, d)) and "-fio-" in d) + def _result_json(self, pod: str) -> dict: + p = os.path.join(self.outdir, pod, "result.json") + if not os.path.exists(p): + return {} + try: + with open(p) as fh: + loaded = json.load(fh) + except (OSError, json.JSONDecodeError): + return {} + return loaded if isinstance(loaded, dict) else {} + + def _fio_start(self, pod: str) -> datetime | None: + """The wall clock of second 0 of this pod's fio time series. + + Taken from fio's own `job_start` (epoch milliseconds) rather than from the + wall_clock column, because the column is only as right as whatever wrote it: runs + archived before the base was fixed derived it from a shared run-start stamp taken + minutes before each pod's fio actually began. fio's offsets are milliseconds since + *that job* started, so the job's own start is the only base that is right by + construction, and it is right for old archives too — which is the point, since a + detector that cannot be re-run against the run that motivated it is a detector + nobody trusts. + + Falls back to the CSV's own column when result.json is missing or carries no + job_start. + """ + if pod not in self._fio_start_cache: + jobs = self._result_json(pod).get("jobs") or [] + start_ms = jobs[0].get("job_start") if jobs else None + self._fio_start_cache[pod] = ( + datetime.fromtimestamp(start_ms / 1000.0, tz=UTC) + if isinstance(start_ms, int | float) and start_ms > 0 else None) + return self._fio_start_cache[pod] + def fio_jobs(self) -> list[FioJob]: out = [] for pod in self.pods(): - p = os.path.join(self.outdir, pod, "result.json") - if not os.path.exists(p): - continue - try: - with open(p) as fh: - res = json.load(fh) - except (OSError, json.JSONDecodeError): + res = self._result_json(pod) + if not res: continue for job in res.get("jobs", []): rd, wr = job.get("read", {}), job.get("write", {}) @@ -250,6 +280,7 @@ def _first(r: dict[str, str], *keys: str) -> str: return v return "" + base = self._fio_start(pod) try: with open(p, newline="") as fh: for r in csv.DictReader(fh): @@ -258,9 +289,9 @@ def _first(r: dict[str, str], *keys: str) -> str: tot = float(_first(r, "total_iops") or 0) except (TypeError, ValueError): continue - out.append(IopsSample(offset_s=off, - wall=_dt(_first(r, "wall", "wall_clock")), - total_iops=tot)) + wall = (base + timedelta(seconds=off) if base + else _dt(_first(r, "wall", "wall_clock"))) + out.append(IopsSample(offset_s=off, wall=wall, total_iops=tot)) except OSError: return [] out.sort(key=lambda s: s.offset_s) diff --git a/test/framework/sbtest/components/workload.py b/test/framework/sbtest/components/workload.py index 8131fc764..81928fd0c 100644 --- a/test/framework/sbtest/components/workload.py +++ b/test/framework/sbtest/components/workload.py @@ -25,6 +25,7 @@ import json import time +from datetime import UTC, datetime, timedelta from typing import Any from ..core import Component, RunContext, component @@ -503,7 +504,7 @@ def _write_timeseries(self, ctx: RunContext, ns: str, pod: str, d: str, if not per_sec: return - start, _ = ctx.window() + start = self._fio_time_base(ctx, pod, d) with open(f"{d}/timeseries.csv", "w") as fh: fh.write("second,wall_clock,total_iops,read_iops,write_iops,active_migration\n") for sec in sorted(per_sec): @@ -512,12 +513,39 @@ def _write_timeseries(self, ctx: RunContext, ns: str, pod: str, d: str, wall = "" active = "" if start: - from datetime import timedelta ts = start + timedelta(seconds=sec) wall = ts.strftime("%Y-%m-%dT%H:%M:%SZ") active = next((m.name for m in migs if m.covers(ts)), "") fh.write(f"{sec},{wall},{total},{row['read']},{row['write']},{active}\n") + def _fio_time_base(self, ctx: RunContext, pod: str, d: str) -> datetime | None: + """The wall clock of second 0 of one pod's fio time series. + + fio's log timestamps are milliseconds since *that job* started, so the only correct + base is the job's own start — `job_start` in its result.json, which fio records in + epoch milliseconds. The run's own window start is not that base: it is stamped when + the run began, before the PVCs, the pods and their fio processes existed, so it sits + well ahead of every pod's fio start (162-211s in fiomig-1787685649, and every pod by a + different amount). Using it shifts every wall clock in timeseries.csv by that much + and, because the same base decides which migration an outage overlaps, names the + wrong migration for gaps that fall within a few minutes of a real one. + + Falls back to the run's start when a pod's result.json is unreadable — a wrong base + is still better than an empty wall_clock column, and the fallback is reported. + """ + try: + with open(f"{d}/result.json") as fh: + jobs = json.load(fh).get("jobs") or [] + start_ms = jobs[0].get("job_start") if jobs else None + if isinstance(start_ms, int | float) and start_ms > 0: + return datetime.fromtimestamp(start_ms / 1000.0, tz=UTC) + except (OSError, json.JSONDecodeError, AttributeError, IndexError): + pass + run_start, _ = ctx.window() + ctx.log.warn(f"{self.name}: {pod}: fio result.json carries no job_start; falling back " + "to the run's start, which leads each pod's real fio start") + return run_start + def teardown(self, ctx: RunContext) -> None: if ctx.shared.get("keep"): ctx.log.info(f"{self.name}: keep set; leaving {len(self._pods)} pod(s), " diff --git a/test/framework/sbtest/core/__init__.py b/test/framework/sbtest/core/__init__.py index 0ddf689d2..eb4caf883 100644 --- a/test/framework/sbtest/core/__init__.py +++ b/test/framework/sbtest/core/__init__.py @@ -12,6 +12,7 @@ Migration, NvmeController, attribute, + attribute_window, freeze_windows, ) from .findings import Attribution, Finding, Report, Severity, critical, info, warning @@ -33,7 +34,8 @@ "ControlEvent", "Finding", "LogSpan", "FioJob", "IopsSample", "Logger", "Migration", "NvmeController", "Report", "RunContext", "Runner", "Selection", "Severity", "SkipDetector", "Timeline", - "apply_cli_toggles", "attribute", "build_component", "build_detector", "component", + "apply_cli_toggles", "attribute", "attribute_window", "build_component", + "build_detector", "component", "critical", "detector", "findings_by_subject_table", "freeze_windows", "info", "iso", "known_components", "known_detectors", "load", "now_utc", "suite_path", "warning", ] diff --git a/test/framework/sbtest/core/evidence.py b/test/framework/sbtest/core/evidence.py index 1185be9d9..5b2b5745e 100644 --- a/test/framework/sbtest/core/evidence.py +++ b/test/framework/sbtest/core/evidence.py @@ -275,3 +275,31 @@ def attribute(migrations: list[Migration], ts: datetime, if m.covers(ts, lag): return m return None + + +def attribute_window(migrations: list[Migration], start: datetime, end: datetime, + lag: timedelta = timedelta(0)) -> Migration | None: + """The migration a symptom that *lasted* belongs to: the one it shares most seconds with. + + Not the same question as `attribute`, and answering it with `attribute(start)` is what + made outages disappear. A gap does not have to begin inside a migration's window to + belong to it — the host goes dry a few seconds before the operator records the migration + as started — so testing only the first second files those gaps under "no migration was + running", which reads as "the cluster is unwell" rather than "the cutover cost this". + + Overlap is measured, not merely tested, because a long window can touch two migrations; + the one holding most of it is the one worth naming. Zero counts as an overlap: a + zero-length window inside a migration, and a window that only touches one, are both + inside rather than outside. + """ + t0, t1 = start.timestamp(), end.timestamp() + best: Migration | None = None + best_overlap: float | None = None + for m in migrations: + m_end = (m.end or m.start) + lag + overlap = min(t1, m_end.timestamp()) - max(t0, m.start.timestamp()) + if overlap < 0: + continue + if best_overlap is None or overlap > best_overlap: + best, best_overlap = m, overlap + return best diff --git a/test/framework/sbtest/detectors/fio.py b/test/framework/sbtest/detectors/fio.py index 8f528e805..650cefd8d 100644 --- a/test/framework/sbtest/detectors/fio.py +++ b/test/framework/sbtest/detectors/fio.py @@ -8,9 +8,20 @@ import re from collections.abc import Iterable +from dataclasses import dataclass from datetime import UTC, datetime, timedelta -from ..core import Detector, Evidence, Finding, SkipDetector, attribute, critical, detector, warning +from ..core import ( + Detector, + Evidence, + Finding, + SkipDetector, + attribute, + attribute_window, + critical, + detector, + warning, +) #: How long after a migration ends one of its lost writes may still surface. #: @@ -163,6 +174,21 @@ def detect(self, ev: Evidence) -> Iterable[Finding]: ) +@dataclass(frozen=True) +class _Window: + """One stretch of a pod doing no I/O. + + `recovered` is the whole distinction between a freeze and a loss, so it travels with the + window rather than being re-derived from where it sits in the series. + """ + + seconds: int + from_offset_s: int + to_offset_s: int + migration: str + recovered: bool + + @detector class Outage(Detector): """A sustained window where a pod did no I/O at all. @@ -171,15 +197,28 @@ class Outage(Detector): returned wrong data, it stopped. Short dips are normal during a cutover, so only runs at or above `min_seconds` count. - Reported per pod rather than per window. A pod that stalls repeatedly stalls *hundreds* of - times in a soak — one run produced 781 qualifying windows at a 10s threshold — and a - report with 781 entries for one symptom is a report nobody reads to the end. The worst - windows are named, the rest are counted, and the total downtime is what the finding leads - with, because "18% of the run did no I/O" is the fact that decides anything. + A stopped volume is reported as one of two things, because they are not the same defect: + + * an I/O **loss** is a gap that was still open when fio stopped. I/O the volume was + supposed to accept and never did. + * an I/O **freeze** is a gap that recovered. Every write the application issued was + eventually taken, so nothing was lost; what the freeze measures is how long an + application had to survive with the volume gone. A cutover is a freeze by design, + which is why its *length* is the thing under test and its existence is not. + + Both are CRITICAL — a volume that stops for longer than a cutover should cost is a + defect either way — but a run that lost I/O and a run that stalled and recovered need + different next steps, and a report that calls both "outage" hands the reader that work. + + Reported per pod and per kind rather than per window. A pod that stalls repeatedly stalls + *hundreds* of times in a soak — one run produced 781 qualifying windows at a 10s + threshold — and a report with 781 entries for one symptom is a report nobody reads to the + end. The worst windows are named, the rest are counted, and the total downtime is what the + finding leads with, because "18% of the run did no I/O" is the fact that decides anything. """ name = "fio.outage" - summary = "a pod's I/O stopped for longer than a cutover should cost" + summary = "a pod's I/O stopped for longer than a cutover should cost — a freeze if it came back, a loss if it did not" # noqa: E501 def defaults(self) -> dict: return {"min_seconds": 30, "iops_floor": 0.0, "name_worst": 6} @@ -190,7 +229,6 @@ def detect(self, ev: Evidence) -> Iterable[Finding]: raise SkipDetector("no fio pods") floor = float(self.opt("iops_floor")) need = int(self.opt("min_seconds")) - show = int(self.opt("name_worst")) migs = ev.migrations() saw_series = False @@ -199,7 +237,8 @@ def detect(self, ev: Evidence) -> Iterable[Finding]: if not series: continue saw_series = True - windows: list[tuple[int, int, int, str]] = [] # dur, from, to, migration + wall_at = {x.offset_s: x.wall for x in series} + windows: list[_Window] = [] run_start: int | None = None prev_off: int | None = None # None is a sentinel that closes a run still open at the end of the series. @@ -210,48 +249,76 @@ def detect(self, ev: Evidence) -> Iterable[Finding]: if run_start is None: run_start = s.offset_s elif run_start is not None: + # A run the sentinel closes was still down when fio stopped: nothing + # observed it come back, so it is a loss rather than a freeze. + recovered = s is not None end = s.offset_s if s is not None else (prev_off or run_start) dur = end - run_start if dur >= need: - wall = next((x.wall for x in series if x.offset_s == run_start), None) - mig = attribute(migs, wall) if wall else None - windows.append((dur, run_start, end, mig.name if mig else "")) + w0, w1 = wall_at.get(run_start), wall_at.get(end) + mig = attribute_window(migs, w0, w1 or w0) if w0 else None + windows.append(_Window(dur, run_start, end, + mig.name if mig else "", recovered)) run_start = None if s is not None: prev_off = s.offset_s if not windows: continue - windows.sort(key=lambda w: -w[0]) - downtime = sum(w[0] for w in windows) span = series[-1].offset_s - series[0].offset_s or 1 - named = ", ".join( - f"{d}s at +{a}s" + (f" ({m})" if m else " (no migration)") - for d, a, _, m in windows[:show]) - during = sorted({w[3] for w in windows if w[3]}) - yield critical( - self.name, - title=(f"I/O stopped {len(windows)}x for {downtime}s total " - f"({downtime / span:.0%} of the run), worst {windows[0][0]}s"), - subject=pod, - detail=named + (f", and {len(windows) - show} more" if len(windows) > show - else ""), - evidence={"windows": len(windows), "downtime_s": downtime, - "worst_s": windows[0][0], - "fraction_of_run": round(downtime / span, 4), - "migrations": during, - "worst_windows": [{"seconds": d, "from_offset_s": a, - "to_offset_s": b, "migration": m} - for d, a, b, m in windows[:show]]}, - note=("Windows are attributed to the migration in flight when there was one; " - f"{len(during)} migration(s) are implicated here." - if during else - "None of these windows fall inside a migration, so the cause is " - "elsewhere — check the fabric and the node's own health."), - ) + # Losses first: both fail the run, but a gap that never closed outranks one that + # did, and the order findings are emitted in is the order they are read in. + for kind in ("loss", "freeze"): + of_kind = [w for w in windows if (w.recovered != (kind == "loss"))] + if of_kind: + yield self._finding(pod, kind, of_kind, span) if not saw_series: raise SkipDetector("no fio time series available") + def _finding(self, pod: str, kind: str, windows: list[_Window], span: int) -> Finding: + show = int(self.opt("name_worst")) + windows = sorted(windows, key=lambda w: -w.seconds) + downtime = sum(w.seconds for w in windows) + named = ", ".join( + f"{w.seconds}s at +{w.from_offset_s}s" + + (f" ({w.migration})" if w.migration else " (no migration)") + for w in windows[:show]) + during = sorted({w.migration for w in windows if w.migration}) + if kind == "loss": + title = (f"I/O LOSS: stopped {len(windows)}x for {downtime}s total " + f"({downtime / span:.0%} of the run) and never resumed, " + f"worst {windows[0].seconds}s") + meaning = ("I/O was still dead when fio stopped, so these are writes the volume " + "was supposed to accept and never did. ") + else: + title = (f"I/O froze {len(windows)}x for {downtime}s total " + f"({downtime / span:.0%} of the run), worst {windows[0].seconds}s, " + "all recovered") + meaning = ("Every freeze recovered, so no I/O was lost — what this measures is " + "how long an application had to survive with the volume gone. ") + return critical( + self.name, + title=title, + subject=pod, + detail=named + (f", and {len(windows) - show} more" if len(windows) > show else ""), + evidence={"kind": kind, "windows": len(windows), "downtime_s": downtime, + "worst_s": windows[0].seconds, + "fraction_of_run": round(downtime / span, 4), + "migrations": during, + "worst_windows": [{"seconds": w.seconds, + "from_offset_s": w.from_offset_s, + "to_offset_s": w.to_offset_s, + "migration": w.migration, + "recovered": w.recovered} + for w in windows[:show]]}, + note=meaning + ( + "Windows are attributed to the migration they share the most seconds with; " + f"{len(during)} migration(s) are implicated here." + if during else + "None of these windows overlap a migration, so the cause is " + "elsewhere — check the fabric and the node's own health."), + ) + @detector class Throughput(Detector): diff --git a/test/framework/tests/test_detectors.py b/test/framework/tests/test_detectors.py index eb4621cf4..a9cb879ce 100644 --- a/test/framework/tests/test_detectors.py +++ b/test/framework/tests/test_detectors.py @@ -30,6 +30,7 @@ Report, Severity, SkipDetector, + attribute_window, build_detector, freeze_windows, ) @@ -372,6 +373,83 @@ def test_windows_outside_any_migration_say_so(self): self.assertEqual(found[0].evidence["migrations"], []) self.assertIn("elsewhere", found[0].note) + def test_a_gap_that_recovered_is_a_freeze_not_a_loss(self): + """A cutover is a freeze by design: every write was eventually taken. It still fails + the run at this length, but calling it loss says data went missing when none did.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40 + "1" * 5)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].evidence["kind"], "freeze") + self.assertIn("froze", found[0].title) + self.assertIn("no I/O was lost", found[0].note) + self.assertTrue(found[0].evidence["worst_windows"][0]["recovered"]) + + def test_a_gap_still_open_when_fio_stopped_is_a_loss(self): + """Nothing observed the volume come back, so this is I/O it was supposed to accept + and never did — the finding a reader must not have to dig for.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["kind"], "loss") + self.assertIn("LOSS", found[0].title) + self.assertFalse(found[0].evidence["worst_windows"][0]["recovered"]) + + def test_a_pod_with_both_reports_the_loss_first(self): + """Both fail the run; the order they are emitted in is the order they are read in, + and a gap that never closed outranks one that did.""" + ev = FakeEvidence(series={"p": self.series("1" * 5 + "0" * 40 + "1" * 5 + "0" * 40)}) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual([f.evidence["kind"] for f in found], ["loss", "freeze"]) + self.assertEqual([f.evidence["windows"] for f in found], [1, 1]) + + def test_a_gap_beginning_before_the_migration_still_belongs_to_it(self): + """The host goes dry before the operator records the migration as started. Testing + only the window's first second files those gaps under no migration at all.""" + mig = Migration(name="mig-1", start=ts(20), end=ts(60)) + ev = FakeEvidence(series={"p": self.series("1" * 10 + "0" * 40 + "1")}, + migrations=[mig]) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["migrations"], ["mig-1"]) + + def test_a_window_spanning_two_migrations_names_the_one_holding_most_of_it(self): + ev = FakeEvidence( + series={"p": self.series("1" * 10 + "0" * 40 + "1")}, + migrations=[Migration(name="mig-a", start=ts(5), end=ts(20)), + Migration(name="mig-b", start=ts(30), end=ts(70))]) + found = list(build_detector("fio.outage", min_seconds=30).detect(ev)) + self.assertEqual(found[0].evidence["migrations"], ["mig-b"]) + + +class AttributeWindow(unittest.TestCase): + """The primitive behind outage attribution: a symptom that lasted, not one that fired.""" + + def named(self, migs: list[Migration], start: datetime, end: datetime) -> str: + m = attribute_window(migs, start, end) + return m.name if m else "" + + def test_a_window_wholly_inside_a_migration(self): + migs = [Migration(name="m", start=ts(0), end=ts(100))] + self.assertEqual(self.named(migs, ts(10), ts(20)), "m") + + def test_a_window_that_misses_every_migration(self): + migs = [Migration(name="m", start=ts(0), end=ts(10))] + self.assertIsNone(attribute_window(migs, ts(20), ts(30))) + + def test_touching_counts_as_overlapping(self): + """Zero is an overlap: a gap that begins the second a migration ends is the + migration's, and the sampling interval must not decide that.""" + migs = [Migration(name="m", start=ts(0), end=ts(10))] + self.assertEqual(self.named(migs, ts(10), ts(50)), "m") + + def test_the_largest_overlap_wins_not_the_first(self): + migs = [Migration(name="early", start=ts(0), end=ts(15)), + Migration(name="late", start=ts(20), end=ts(60))] + self.assertEqual(self.named(migs, ts(10), ts(50)), "late") + + def test_a_running_migration_is_a_point_in_time(self): + """`end` is None while it is in flight — the same convention `covers` uses.""" + migs = [Migration(name="m", start=ts(30), end=None)] + self.assertEqual(self.named(migs, ts(10), ts(50)), "m") + class NvmeStaleControllers(unittest.TestCase): def ctrl(self, name, state, ns, node="vm03", nqn="nqn:lvol:x", addr="10.0.0.1:4420", clt=60): diff --git a/test/framework/tests/test_driver.py b/test/framework/tests/test_driver.py index 18773b260..1e60296bd 100644 --- a/test/framework/tests/test_driver.py +++ b/test/framework/tests/test_driver.py @@ -359,6 +359,64 @@ def test_the_analyser_reads_back_what_the_workload_wrote(self): self.assertEqual([s.total_iops for s in series], [500.0, 400.0]) self.assertIsNotNone(series[0].wall) + def test_the_clock_comes_from_fio_not_from_the_run(self): + """fio counts from its own launch, which is minutes after the run's start: the PVCs, + the pods and the fio processes all have to exist first. Basing the wall clock on the + run shifts every sample by that gap and hands an outage to the wrong migration.""" + raw = "1000, 500, 0, 4096" + run_start = datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC) + fio_start = run_start + timedelta(seconds=180) + # The migration ran while fio was running, i.e. nowhere near the run's own start. + migs = [Migration(name="mig-1", start=fio_start, end=fio_start + timedelta(seconds=30))] + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=run_start) + d = ctx.dir("fio-0") + with open(os.path.join(d, "result.json"), "w") as fh: + json.dump({"jobs": [{"job_start": int(fio_start.timestamp() * 1000)}]}, fh) + workload.FioWorkload()._write_timeseries(ctx, "default", "fio-0", d, migs) + with open(os.path.join(d, "timeseries.csv")) as fh: + rows = list(__import__("csv").DictReader(fh)) + self.assertEqual(rows[0]["wall_clock"], "2026-08-20T09:03:01Z") + self.assertEqual(rows[0]["active_migration"], "mig-1") + + def test_a_result_without_job_start_falls_back_to_the_run(self): + """A wrong base still beats an empty wall_clock column — but it is reported.""" + raw = "1000, 500, 0, 4096" + with _Ctx() as ctx, _patch(kube, "exec_sh", lambda *a, **k: raw): + ctx.mark_window(start=datetime(2026, 8, 20, 9, 0, 0, tzinfo=UTC)) + d = ctx.dir("fio-0") + with open(os.path.join(d, "result.json"), "w") as fh: + json.dump({"jobs": [{}]}, fh) + workload.FioWorkload()._write_timeseries(ctx, "default", "fio-0", d, []) + with open(os.path.join(d, "timeseries.csv")) as fh: + rows = list(__import__("csv").DictReader(fh)) + self.assertEqual(rows[0]["wall_clock"], "2026-08-20T09:00:01Z") + + def test_the_analyser_re_derives_the_base_from_fio(self): + """Replay has to correct archives written before the base was fixed: the wall_clock + column is only as right as whatever wrote it, and job_start is right by + construction.""" + from sbtest.adapters import ArchiveEvidence + fio_start = datetime(2026, 8, 20, 9, 3, 0, tzinfo=UTC) + with _Ctx() as ctx: + d = ctx.dir("run1-fio-0") + with open(os.path.join(d, "result.json"), "w") as fh: + json.dump({"jobs": [{"job_start": int(fio_start.timestamp() * 1000)}]}, fh) + with open(os.path.join(d, "timeseries.csv"), "w") as fh: + fh.write("second,wall_clock,total_iops\n" + "1,2026-08-20T09:00:01Z,500.0\n") # the pre-fix, run-based clock + series = ArchiveEvidence(ctx.outdir).fio_timeseries("run1-fio-0") + self.assertEqual(series[0].wall, fio_start + timedelta(seconds=1)) + + def test_the_wall_clock_column_is_used_when_fio_says_nothing(self): + from sbtest.adapters import ArchiveEvidence + with _Ctx() as ctx: + d = ctx.dir("run1-fio-0") + with open(os.path.join(d, "timeseries.csv"), "w") as fh: + fh.write("second,wall_clock,total_iops\n1,2026-08-20T09:00:01Z,500.0\n") + series = ArchiveEvidence(ctx.outdir).fio_timeseries("run1-fio-0") + self.assertEqual(series[0].wall, datetime(2026, 8, 20, 9, 0, 1, tzinfo=UTC)) + def test_a_workload_with_no_pods_is_refused(self): with _Ctx() as ctx, self.assertRaises(RuntimeError) as e: workload.FioWorkload(pods=0, ns_pods=0)._create(ctx) From 95931660a19f373ff42e976a26350178783a62c4 Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Wed, 26 Aug 2026 17:35:11 +0200 Subject: [PATCH 3/3] Stop leaking grabber pods, keep the recorded window, and name the file a finding is in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the review on #445, each with the test that would have caught it written and run red first. logs.collect started a privileged pod on every node it wanted a log from and never looked at ctx.shared["logs.grabbers"], which logs.stream publishes for exactly this purpose — so a run carried two privileged pods per node doing the same job. Worse, it recorded nothing in `_own`, and `_own` is all teardown is allowed to delete, so every pod it started outlived the run until its TTL expired. It now borrows what has been published and owns only what it had to start. The test that was supposed to cover this passed throughout, because it re-implemented the reuse arithmetic in its own body and never called collect(). It now drives the real collect() with the cluster faked at the component's boundary, which is what made both defects visible: the first case fails on ['vm02','vm03','vm04'] != ['vm04'], the second on [[]] != [['own-vm03','own-vm04']]. mark_window persisted its `end` argument rather than the end it had stored, so any later call without one rewrote run.json with end: null. Nothing in the current sequence calls it that way, which is precisely why it would have gone unnoticed: the window is what bounds a ring-buffer detector to this run, and an open window silently lets evidence from after the run count against it. security.secret-exposure listed log names where the run directory holds .txt, so the one field that says where to look did not name a file. Every other detector reporting a container log names it with its extension. Co-Authored-By: Claude Opus 5 (1M context) --- test/framework/sbtest/components/logs.py | 11 +- test/framework/sbtest/core/context.py | 9 +- test/framework/sbtest/detectors/security.py | 4 +- test/framework/tests/test_core.py | 116 ++++++++++++++------ test/framework/tests/test_detectors.py | 16 +++ 5 files changed, 115 insertions(+), 41 deletions(-) diff --git a/test/framework/sbtest/components/logs.py b/test/framework/sbtest/components/logs.py index 4650cb310..b29a26acf 100644 --- a/test/framework/sbtest/components/logs.py +++ b/test/framework/sbtest/components/logs.py @@ -383,7 +383,16 @@ def collect(self, ctx: RunContext) -> None: return nodes = {p.node for p, _c, _n in plan if p.node} - self._grabbers = self._start_grabbers(ctx, sorted(nodes), int(self.opt("ttl_s"))) + # logs.stream publishes its grabbers for exactly this. Its TTL already covers the + # whole run, and by collection time its followers have stopped, so the pod is idle and + # free to be exec'd into — a second privileged pod per node doing the same job is what + # the distinct names were only ever a backstop against. + borrowed = {n: g for n, g in ctx.shared.get("logs.grabbers", {}).items() if n in nodes} + missing = sorted(nodes - set(borrowed)) + # Recorded separately from `_grabbers`, because teardown may only delete these: a + # borrowed pod belongs to the component that published it. + self._own = self._start_grabbers(ctx, missing, int(self.opt("ttl_s"))) if missing else {} + self._grabbers = {**borrowed, **self._own} grouped: dict[str, list[tuple[kube.Pod, str]]] = {} for p, c, artifact in plan: diff --git a/test/framework/sbtest/core/context.py b/test/framework/sbtest/core/context.py index 11a36c349..a8dd0bd8d 100644 --- a/test/framework/sbtest/core/context.py +++ b/test/framework/sbtest/core/context.py @@ -151,12 +151,15 @@ def mark_window(self, start: datetime | None = None, end: datetime | None = None filesystem shutdowns that had happened hours before it started. """ self._window_start = getattr(self, "_window_start", None) or start or now_utc() - if end: - self._window_end = end + # The *stored* end, not the argument. A later mark without one — a second collect + # phase, a component closing its own books — must not rewrite the record with no end + # at all: the window is what bounds every ring-buffer detector to this run, and an + # open window makes evidence from after it count as the run's. + self._window_end = end or getattr(self, "_window_end", None) self.save_json("run.json", { "run_id": self.run_id, "start": iso(self._window_start), - "end": iso(end) if end else None, + "end": iso(self._window_end) if self._window_end else None, }) def save_json(self, name: str, obj: Any) -> str: diff --git a/test/framework/sbtest/detectors/security.py b/test/framework/sbtest/detectors/security.py index ec74b0ecf..99c4bf3cf 100644 --- a/test/framework/sbtest/detectors/security.py +++ b/test/framework/sbtest/detectors/security.py @@ -76,7 +76,9 @@ def detect(self, ev: Evidence) -> Iterable[Finding]: detail=where, evidence={"pattern": label, "total": total, "per_log": {k: len(v) for k, v in per_log.items()}}, - artifacts=sorted(per_log), + # The file, not the log's name: the run directory holds .txt, and + # a pointer a reader has to complete is not a pointer. + artifacts=[f"{log}.txt" for log in sorted(per_log)], note="The value is deliberately not quoted here — findings.json travels at " "least as widely as the log. Rotate whatever this is, then stop the " "component logging it; artifacts get attached to tickets and committed.", diff --git a/test/framework/tests/test_core.py b/test/framework/tests/test_core.py index bbab934bc..e9c57a8d2 100644 --- a/test/framework/tests/test_core.py +++ b/test/framework/tests/test_core.py @@ -16,11 +16,13 @@ import unittest from datetime import UTC, datetime, timedelta from typing import cast +from unittest import mock sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import sbtest # noqa: E402,F401 (registers the bundled plugins) from sbtest.adapters import ArchiveEvidence # noqa: E402 +from sbtest.components import kube # noqa: E402 from sbtest.core import ( # noqa: E402 Component, Detector, @@ -406,6 +408,24 @@ def test_run_json_is_written_and_read_back(self): self.assertIsNotNone(start) self.assertIsNotNone(end) + def test_a_later_mark_does_not_erase_the_recorded_end(self): + """Regression: 2026-08-26-mark-window-erases-end (PR #445 review). + + mark_window persisted its `end` *argument* rather than the end it had stored, so any + later call without one rewrote run.json with end: null. Every detector that bounds + what it may blame on the run reads that window; with no end, evidence from after the + run counts as the run's. + """ + with tempfile.TemporaryDirectory() as d: + ctx = RunContext(run_id="t", outdir=d, log=Logger(None)) + ctx.mark_window() + ctx.mark_window(end=now_utc()) + ctx.mark_window() # a second collect phase, a component, a retry + with open(os.path.join(d, "run.json")) as fh: + rec = json.load(fh) + self.assertIsNotNone(rec["end"]) + self.assertIsNotNone(ArchiveEvidence(d).run_window()[1]) + def test_run_json_wins_over_inference_from_test_log(self): """The run's own record beats whatever happened to get logged.""" with tempfile.TemporaryDirectory() as d: @@ -472,55 +492,79 @@ class GrabberReuse(unittest.TestCase): names are the backstop for when reuse is not possible. """ - def test_collect_reuses_published_grabbers_and_starts_only_the_rest(self): - from sbtest.components.logs import LogCollect + def _probe(self): + """A LogCollect with the cluster faked at the component's own boundaries. + + Everything below `collect()` is stubbed and everything inside it runs, because the + previous version of these cases re-implemented the reuse arithmetic in the test body + and never called `collect()` at all — so it went on passing while the component + borrowed nothing and recorded nothing to tear down. + """ + from sbtest.components import logs as logs_mod started: list[list[str]] = [] + deleted: list[list[str]] = [] - class Probe(LogCollect): + class Probe(logs_mod.LogCollect): def _start_grabbers(self, ctx, nodes, ttl_s): started.append(list(nodes)) return {n: f"own-{n}" for n in nodes} - c = Probe() - with tempfile.TemporaryDirectory() as d: + def _delete_grabbers(self, ctx, names): + deleted.append(list(names)) + + pods = [kube.Pod(name=f"snode-spdk-{i}", namespace="default", node=node, + containers=("spdk-container",)) + for i, node in enumerate(("vm02", "vm03", "vm04"))] + c = Probe(targets=[{"pods": ["snode-spdk"], "containers": ["spdk-container"], + "name_from": "pod-container"}]) + return c, started, deleted, pods + + def test_collect_reuses_published_grabbers_and_starts_only_the_rest(self): + """Regression: 2026-08-26-logcollect-ignores-published-grabbers (PR #445 review). + + logs.collect started its own privileged pod on every node regardless of what + logs.stream had already published in ctx.shared["logs.grabbers"], so a run carried two + privileged pods per node doing the same job. + """ + c, started, _deleted, pods = self._probe() + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(kube, "list_pods", lambda *a, **k: pods), \ + mock.patch.object(kube, "run_bytes", lambda *a, **k: b"log line\n"): ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) ctx.shared["logs.grabbers"] = {"vm02": "stream-vm02", "vm03": "stream-vm03"} - # Stand in for the pod census: two nodes already covered, one not. - plan_nodes = ["vm02", "vm03", "vm04"] - inherited = dict(ctx.shared["logs.grabbers"]) - missing = sorted(n for n in plan_nodes if n not in inherited) - own = c._start_grabbers(ctx, missing, 60) - c._own = own - c._grabbers = {**{n: g for n, g in inherited.items() if n in plan_nodes}, **own} - - self.assertEqual(started, [["vm04"]]) # only the uncovered node - self.assertEqual(c._grabbers["vm02"], "stream-vm02") # reused, not replaced - self.assertEqual(c._grabbers["vm04"], "own-vm04") - - deleted: list[list[str]] = [] - c._delete_grabbers = lambda _ctx, names: deleted.append(list(names)) # type: ignore[assignment] + c.collect(ctx) + self.assertEqual(started, [["vm04"]]) # only the uncovered node + self.assertEqual(c._grabbers["vm02"], "stream-vm02") # borrowed, not replaced + self.assertEqual(c._grabbers["vm04"], "own-vm04") + + def test_teardown_removes_the_grabbers_collect_started(self): + """Regression: 2026-08-26-logcollect-grabber-leak (PR #445 review). + + collect() never recorded what it started in `_own`, so teardown deleted nothing and + every privileged pod it created outlived the run, up to its TTL. + """ + c, _started, deleted, pods = self._probe() + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(kube, "list_pods", lambda *a, **k: pods), \ + mock.patch.object(kube, "run_bytes", lambda *a, **k: b"log line\n"): + ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) + ctx.shared["logs.grabbers"] = {"vm02": "stream-vm02"} + c.collect(ctx) c.teardown(ctx) - # Only its own; deleting a borrowed pod would pull it from under its owner. - self.assertEqual(deleted, [["own-vm04"]]) + # Its own three, and none of logs.stream's: deleting a borrowed pod would pull it + # out from under the component that owns it. + self.assertEqual(deleted, [["own-vm03", "own-vm04"]]) def test_collect_works_standalone_when_nothing_published(self): """logs.stream may be disabled — collect must still start what it needs.""" - from sbtest.components.logs import LogCollect - started: list[list[str]] = [] - - class Probe(LogCollect): - def _start_grabbers(self, ctx, nodes, ttl_s): - started.append(list(nodes)) - return {n: f"own-{n}" for n in nodes} - - c = Probe() - with tempfile.TemporaryDirectory() as d: + c, started, _deleted, pods = self._probe() + with tempfile.TemporaryDirectory() as d, \ + mock.patch.object(kube, "list_pods", lambda *a, **k: pods), \ + mock.patch.object(kube, "run_bytes", lambda *a, **k: b"log line\n"): ctx = RunContext(run_id="r", outdir=d, log=Logger(None)) - inherited = dict(ctx.shared.get("logs.grabbers", {})) - self.assertEqual(inherited, {}) - own = c._start_grabbers(ctx, ["vm02", "vm03"], 60) - self.assertEqual(started, [["vm02", "vm03"]]) - self.assertEqual(sorted(own), ["vm02", "vm03"]) + c.collect(ctx) + self.assertEqual(started, [["vm02", "vm03", "vm04"]]) + self.assertEqual(sorted(c._grabbers), ["vm02", "vm03", "vm04"]) def test_lifecycle_order_makes_reuse_safe(self): """stop precedes collect, and teardown follows it — so the borrowed pod is idle and diff --git a/test/framework/tests/test_detectors.py b/test/framework/tests/test_detectors.py index a9cb879ce..1345d5aec 100644 --- a/test/framework/tests/test_detectors.py +++ b/test/framework/tests/test_detectors.py @@ -451,6 +451,22 @@ def test_a_running_migration_is_a_point_in_time(self): self.assertEqual(self.named(migs, ts(10), ts(50)), "m") +class SecretExposureArtifacts(unittest.TestCase): + def test_the_artifacts_are_the_files_a_reader_can_open(self): + """Regression: 2026-08-26-secret-artifacts-miss-the-extension (PR #445 review). + + The finding listed log *names* ("operator") where the run directory holds + "operator.txt", so the one field that says where to look did not name a file. Every + other detector reporting a container log names it with its extension. + """ + ev = FakeEvidence(logs={ + "operator": ["password=hunter2hunter2"], + "webappapi": ["nothing to see"]}) + found = list(build_detector("security.secret-exposure").detect(ev)) + self.assertEqual(len(found), 1) + self.assertEqual(found[0].artifacts, ["operator.txt"]) + + class NvmeStaleControllers(unittest.TestCase): def ctrl(self, name, state, ns, node="vm03", nqn="nqn:lvol:x", addr="10.0.0.1:4420", clt=60): return NvmeController(node=node, name=name, nqn=nqn, address=addr, state=state,