diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 24535ed..1486b6b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "name": "adversarial-mutation-test", "source": "./", "description": "Find BUGS and harden the test suite for a whole repository — adversarial (spec as oracle, code as suspect; surface candidates for triage) + mutation (break each line, prove a test catches it). Whole-repo, resumable, language-agnostic.", - "version": "0.34.0", + "version": "0.35.0", "author": { "name": "Rain Open Source Software Ltd" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 6778263..0df819a 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "adversarial-mutation-test", "displayName": "Adversarial Mutation Testing", - "version": "0.34.0", + "version": "0.35.0", "description": "Find BUGS and harden the test suite for a whole repository. Two co-equal halves: ADVERSARIAL — treat the spec as the oracle and the code as suspect, hunt for inputs where the code is wrong, and surface candidates for triage (never self-adjudicate); and MUTATION — break each line and prove a test catches it. Whole-repo, resumable, language-agnostic.", "author": { "name": "Rain Open Source Software Ltd", diff --git a/.github/scripts/check-scan-record-schema.sh b/.github/scripts/check-scan-record-schema.sh index 7b3b1bf..620d053 100644 --- a/.github/scripts/check-scan-record-schema.sh +++ b/.github/scripts/check-scan-record-schema.sh @@ -1,15 +1,18 @@ #!/bin/sh -# The scan-record schema lives in two places — the README's "Scan record -# template" (canonical) and SKILL.md's "Committed scan record" (what a closing -# run has in front of it) — and it is documentation, so nothing else in this -# repo executes it. That is how it shipped with no field naming the tree its -# after-campaign counts hold at: rain.sol.codegen committed "testsAfter": 102, a -# count occurring at no commit in the range its record covered, and no reader or -# tool had anything to check it against. +# The scan-ledger schema lives in two places — the README's "Scan ledger" +# (canonical) and SKILL.md's "Committed scan record" (what a closing run has in +# front of it) — and it is documentation, so nothing else in this repo executes +# it. That is how it shipped with no field naming the tree its after-campaign +# counts hold at: rain.sol.codegen committed "testsAfter": 102, a count +# occurring at no commit in the range its record covered, and no reader or tool +# had anything to check it against. It is also how the ledger shipped as a bare +# array with no stated ordering rule, so that same repo's record still reads +# "commitsAheadOfTag": 0 from 35 commits and 5 tags behind its default branch. # # This pins the parts of the schema a reader needs in order to falsify a record: -# the template is real JSON, both trees are named, both are full SHAs, and both -# documents state it. Run it from anywhere: sh .github/scripts/check-scan-record-schema.sh +# the ledger is a wrapper object that can carry its own read rule, the template +# is real JSON, both trees are named, both are full SHAs, and both documents +# state it. Run from anywhere: sh .github/scripts/check-scan-record-schema.sh set -eu root=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) @@ -21,37 +24,60 @@ fail() { exit 1 } -# The fenced JSON block under the README's "## Scan record template" heading. -template=$(awk ' - /^## Scan record template$/ { in_section = 1; next } +# The fenced JSON block under the README's "## Scan ledger" heading. +ledger=$(awk ' + /^## Scan ledger$/ { in_section = 1; next } in_section && /^## / { exit } in_section && /^```json$/ { in_block = 1; next } in_block && /^```$/ { exit } in_block { print } ' "$readme") -[ -n "$template" ] || fail "README.md has no fenced json block under '## Scan record template'" - -echo "$template" | jq -e . >/dev/null 2>&1 || - fail "the README scan record template is not valid JSON — it is the schema consumers copy" - -keys=$(echo "$template" | jq -r 'keys_unsorted | join(" ")') -commit=$(echo "$template" | jq -r '.commit // ""') -tests_after_commit=$(echo "$template" | jq -r '.testsAfterCommit // ""') -echo "template keys: $keys" -echo "template commit=$commit testsAfterCommit=$tests_after_commit" +[ -n "$ledger" ] || fail "README.md has no fenced json block under '## Scan ledger'" + +echo "$ledger" | jq -e . >/dev/null 2>&1 || + fail "the README scan ledger template is not valid JSON — it is the schema consumers copy" + +# The envelope. A bare array structurally cannot carry a schema version or an +# ordering invariant, so the rule for reading it could only live in prose that +# does not ship with the file — which is exactly how a record 35 commits stale +# stayed indistinguishable from a current one. +echo "$ledger" | jq -e 'type == "object"' >/dev/null || + fail "the ledger template must be a wrapper object, not a bare array — a bare array cannot state its own read rule" +echo "$ledger" | jq -e '.schemaVersion | type == "number"' >/dev/null || + fail "the ledger template has no top-level numeric 'schemaVersion'" +echo "$ledger" | jq -e '.records | type == "array" and length > 0' >/dev/null || + fail "the ledger template has no non-empty 'records' array — the array a campaign appends to must be shown, not just one element of it" +envelope_keys=$(echo "$ledger" | jq -r 'keys_unsorted | join(" ")') +echo "ledger envelope keys: $envelope_keys" +echo "OK envelope is an object carrying schemaVersion and a non-empty records array" + +record=$(echo "$ledger" | jq -c '.records[0]') +keys=$(echo "$record" | jq -r 'keys_unsorted | join(" ")') +commit=$(echo "$record" | jq -r '.commit // ""') +tests_after_commit=$(echo "$record" | jq -r '.testsAfterCommit // ""') +echo "record keys: $keys" +echo "record commit=$commit testsAfterCommit=$tests_after_commit" # The must-haves. A record missing any of these cannot be checked against the -# repo it describes. -for key in timestamp commit testsAfterCommit publishedTag commitsAheadOfTag; do - echo "$template" | jq -e --arg key "$key" 'has($key)' >/dev/null || - fail "the template is missing must-have '$key'" +# repo it describes. schemaVersion joins them because migration preserves older +# records untouched: its absence is what tells a reader "this one predates the +# wrapper, do not assume the current shape". +for key in schemaVersion timestamp commit testsAfterCommit publishedTag commitsAheadOfTag; do + echo "$record" | jq -e --arg key "$key" 'has($key)' >/dev/null || + fail "the template record is missing must-have '$key'" done -echo "OK must-haves present: timestamp commit testsAfterCommit publishedTag commitsAheadOfTag" +echo "OK must-haves present: schemaVersion timestamp commit testsAfterCommit publishedTag commitsAheadOfTag" + +# Distinct from skillVersion, which names the skill that wrote the record and +# has already failed to describe its shape. +echo "$record" | jq -e '.schemaVersion != .skillVersion' >/dev/null || + fail "the record's schemaVersion must be distinct from skillVersion — skillVersion is not a statement about shape" +echo "OK record schemaVersion is distinct from skillVersion" # Adjacency is what makes the pair readable as a pair: the before tree and the # after tree sit together, ahead of everything measured at either. -echo "$template" | jq -e 'keys_unsorted | index("testsAfterCommit") == (index("commit") + 1)' >/dev/null || +echo "$record" | jq -e 'keys_unsorted | index("testsAfterCommit") == (index("commit") + 1)' >/dev/null || fail "testsAfterCommit must come immediately after commit; key order is: $keys" echo "OK testsAfterCommit is immediately after commit" @@ -59,7 +85,7 @@ echo "OK testsAfterCommit is immediately after commit" # ambiguous as history grows — which is the same class of defect the after-tree # field exists to close, and rain.solmem's record already carries one. for key in commit testsAfterCommit; do - echo "$template" | jq -e --arg key "$key" '.[$key] | type == "string" and test("^[0-9a-f]{40}$")' >/dev/null || + echo "$record" | jq -e --arg key "$key" '.[$key] | type == "string" and test("^[0-9a-f]{40}$")' >/dev/null || fail "template '$key' must be a full 40-character lowercase hex SHA" done echo "OK commit and testsAfterCommit are full 40-character SHAs" @@ -71,13 +97,13 @@ echo "OK commit and testsAfterCommit are full 40-character SHAs" # each rule is pinned by the shortest phrase that carries it. Reword freely — the # failure names which rule went missing, and re-pinning it is a one-line edit. readme_prose=$(awk ' - /^## Scan record template$/ { in_section = 1; next } + /^## Scan ledger$/ { in_section = 1; next } in_section && /^## / { exit } in_section && /^```/ { fenced = !fenced; next } in_section && !fenced { print } ' "$readme") -[ -n "$readme_prose" ] || fail "README.md has no prose under '## Scan record template' — only the template" +[ -n "$readme_prose" ] || fail "README.md has no prose under '## Scan ledger' — only the template" # Newlines are collapsed first: these documents are reflowed by `deno fmt`, so a # phrase may be split across lines at any time and that is not a rule going @@ -98,6 +124,20 @@ readme_states 'never null' "that testsAfterCommit is never null" readme_states 'never omitted' "that testsAfterCommit is never omitted" echo "OK README prose states every testsAfterCommit rule" +# The ledger's own read rule. The file is the artifact a reader holds, so every +# rule for reading it has to be written where the file's schema is defined; a +# rule kept anywhere else is the defect this section exists to close. +readme_states 'append-only' "that records is append-only" +readme_states 'never rewritten, reordered, or removed' "that a landed record is a historical fact" +readme_states 'whose timestamp is greatest' "which field is authoritative for newest" +readme_states 'not the last array element' "that append order is NOT the authority — a PR queue scrambles it" +readme_states 'later wins' "how a timestamp tie is broken" +readme_states 'Newest is not current' "that a frozen record does not decay into looking stale" +readme_states 'Neither of these is skillVersion' "that schemaVersion is distinct from skillVersion" +readme_states 'wraps it in place' "what a campaign does when it opens an existing bare array" +readme_states 'no field back-filled' "that migration preserves existing records untouched" +echo "OK README prose states every ledger ordering, authority and migration rule" + # SKILL.md is what a run closing its record actually has in front of it. A field # documented only in the README is a field campaigns will not write, and a rule # only the README states is a rule the closing run does not apply. @@ -116,4 +156,14 @@ skill_states 'equal to' "what a run that landed nothing writes" skill_states 'never null' "that testsAfterCommit is never null and never omitted" echo "OK SKILL.md's committed scan record states the field and its rules" -echo "scan record schema OK" +# The closing run is the only writer, so a rule the README states and SKILL.md +# does not is a rule nothing applies. +skill_states 'not a bare array' "that the ledger is a wrapper object" +skill_states 'append-only' "that records is append-only" +skill_states 'whose timestamp is greatest' "which field is authoritative for newest" +skill_states 'later wins' "how a timestamp tie is broken" +skill_states 'Newest is not current' "that a frozen record does not decay into looking stale" +skill_states 'wrap it on this append' "what a run opening an existing bare array does" +echo "OK SKILL.md's committed scan record states the ledger read and migration rules" + +echo "scan ledger schema OK" diff --git a/README.md b/README.md index 7c74539..bc221dd 100644 --- a/README.md +++ b/README.md @@ -119,30 +119,37 @@ probed mutant is killed; 1 on any non-kill; 2 when the pass cannot be trusted. `--only ` re-runs a subset while strengthening a killer; `--json ` writes the machine-readable report. -## Scan record template +## Scan ledger -Campaigns close by appending one entry per run to a committed -`audit/mutation-test-scans.json` on the default branch (see SKILL.md). Valid -JSON, no comments: +Campaigns close by appending one record per run to a committed +`audit/mutation-test-scans.json` on the default branch (see SKILL.md). The file +is a wrapper object, not a bare array: `schemaVersion` says how to read it, +`records` holds the runs. Valid JSON, no comments: ```json { - "timestamp": "2026-08-12T19:40:00Z", - "commit": "08d547fdeadbeefc0ffee1122334455667788990", - "testsAfterCommit": "1f9be22cafebabe0ddf00d998877665544332211", - "publishedTag": "v1.2.3", - "commitsAheadOfTag": 0, - "scope": "whole repo", - "tool": "adversarial-mutation-test", - "skillVersion": "0.34.0", - "summary": { - "behaviours": 600, - "candidates": 89, - "confirmed": 30, - "testsBefore": 41, - "testsAfter": 84, - "filed": ["#2651", "#2660"] - } + "schemaVersion": 1, + "records": [ + { + "schemaVersion": 1, + "timestamp": "2026-08-12T19:40:00Z", + "commit": "08d547fdeadbeefc0ffee1122334455667788990", + "testsAfterCommit": "1f9be22cafebabe0ddf00d998877665544332211", + "publishedTag": "v1.2.3", + "commitsAheadOfTag": 0, + "scope": "whole repo", + "tool": "adversarial-mutation-test", + "skillVersion": "0.35.0", + "summary": { + "behaviours": 600, + "candidates": 89, + "confirmed": 30, + "testsBefore": 41, + "testsAfter": 84, + "filed": ["#2651", "#2660"] + } + } + ] } ``` @@ -151,6 +158,8 @@ JSON, no comments: the release at `commit` (null if unreleased) with `commitsAheadOfTag` its distance. All five are must-haves; `summary` is nice-to-have. +### Two trees, and which numbers hold at each + A record spans two trees, and every number in it is measured at one of them: `commit` is the tree the scan ran against, which every _before_ number (`testsBefore`, baseline counts) holds at; `testsAfterCommit` is the tree with @@ -167,6 +176,52 @@ after-state count checkable at all — `rain.sol.codegen` committed covers, and nothing could catch it because the record named no tree to check it against. +### Two versions, and neither is `skillVersion` + +The top-level `schemaVersion` versions the envelope — the wrapper shape and the +read rules below. The per-record `schemaVersion` versions that one record's +field set. They are separate fields because the file provably holds records of +more than one shape (see Migration), so a single number could not honestly +describe both the file and everything in it. Neither of these is skillVersion, +which names the skill that wrote a record and is not a statement about its shape +— it has already failed to be one, the `0.30.0` summary and the current template +sharing only `filed`. + +### Ordering, and what "newest" means + +`records` is append-only. A record is a historical fact about a tree, so it is +never rewritten, reordered, or removed. + +The newest record is the one **whose timestamp is greatest** — not the last +array element. Append order is the weakest of the candidates precisely because +it is what a PR queue scrambles: two campaigns can land in the opposite order to +the order they ran. Ties in `timestamp` break by array position, later wins. +Nothing else is the rule — not `commit`'s position in history, not file order. + +**Newest is not current.** Every value in a record is frozen at run end and +describes the tree it names, so `commitsAheadOfTag` keeps reading `0` however +far the repo moves on afterwards — a stale record does not decay into looking +stale. The ledger answers "what was audited, and when", never "is that still +true": a reader asking whether a repo is audited at its current release must +compare the newest record's `commit` against the repo today. `rain.sol.codegen` +is the live case — its one record says `commitsAheadOfTag: 0` while sitting 35 +commits and 5 published tags behind the default branch. + +### Migration from a bare array + +The ledger was a bare array before `schemaVersion` 1, and committed ones still +are. A campaign that opens a bare array **wraps it in place on that run's +append**: the array becomes `records`, the top-level `schemaVersion` is added, +and every record already in it is preserved exactly — no field back-filled, no +value corrected, nothing reordered. Back-filling `schemaVersion` or +`testsAfterCommit` into an existing record would assert a measurement no run +made. + +So a record carrying no `schemaVersion` predates the wrapper: its must-haves may +be absent and its `summary` is not the current shape. Read it as a record of its +own `skillVersion`, and never assume the newest record's shape holds across the +file. + ## License [DecentraLicense 1.0](LICENSE) (`LicenseRef-DCL-1.0`). diff --git a/skills/adversarial-mutation-test/SKILL.md b/skills/adversarial-mutation-test/SKILL.md index e550b59..d509c1b 100644 --- a/skills/adversarial-mutation-test/SKILL.md +++ b/skills/adversarial-mutation-test/SKILL.md @@ -1,6 +1,6 @@ --- name: adversarial-mutation-test -version: 0.34.0 +version: 0.35.0 description: Use to systematically find BUGS in and harden the test suite for a WHOLE repository (or a whole module of it). Two co-equal goals the name carries: ADVERSARIAL (treat spec/intent as the oracle and the code as suspect — derive expected behavior independently and hunt for inputs where the code is wrong) and MUTATION (prove tests cover the code). Mutation-drives a behavior-centric coverage ledger — for each behavior, break the line and check the whole suite: existing tests that kill the mutant are validated and logged (so existing coverage is audited and in scope), and only surviving mutants (real gaps) get a new discriminating test. An existing test that already kills mutants is left as-is; one meant to cover a behavior but that a mutant survives is strengthened in place (not duplicated); one broken on the unmutated baseline is fixed or its underlying code bug surfaced; a test is never edited to swallow a mutation. Designed for long campaigns that outlive the context window: progress lives in a durable gitignored scratch file so it survives compaction. A single change/PR/function is just a narrowed scope. Triggers on "test the whole repo", "harden the test suite", "mutation test the codebase", "audit the tests", "adversarial tests", "prove these tests cover the code", "exhaust the eventualities". --- @@ -194,12 +194,22 @@ One mutation, one behavior — the failing-test set stays diagnostic. ## Committed scan record -Close every run — including a clean one — by appending an entry to a committed +Close every run — including a clean one — by appending a record to a committed `audit/mutation-test-scans.json` and landing it on the default branch: -timestamp, scanned commit, `testsAfterCommit`, published tag (+ commits ahead), -scope, tool + skill version, summary with filed issue numbers. The org health -check reads the newest entry for "which release was last audited"; the JSON -template lives in this repo's README. +`schemaVersion`, timestamp, scanned commit, `testsAfterCommit`, published tag (+ +commits ahead), scope, tool + skill version, summary with filed issue numbers. +The org health check reads the newest record — as defined below — for "which +release was last audited"; the JSON template lives in this repo's README. + +The file is `{"schemaVersion": 1, "records": [...]}`, not a bare array, and +`records` is append-only: never rewrite, reorder, or drop a record. Newest is +the record **whose timestamp is greatest**, not the last element — a PR queue +lands runs out of the order they ran — and ties break by array position, later +wins. Newest is not current: every value is frozen at run end, so +`commitsAheadOfTag` stays `0` however far the repo moves, and "is this still +true" is answered against the repo, never from the file. Opening a bare array, +wrap it on this append — the array becomes `records`, existing records +untouched, nothing back-filled. Two trees, both full 40-char SHAs: the scanned `commit` is what every _before_ number holds at; `testsAfterCommit` — the tree with this run's coverage PRs