diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 3f71414..6434425 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -1,5 +1,14 @@ name: Mutation Testing +# All mutation-testing CI lives here. Three concerns, one file: +# * cargo-mutants — generic structural operators (diff-scoped on PRs, sharded +# full-crate run nightly) +# * spec-gate — universalmutator + comby custom operators for SALT's trie +# arithmetic / canonicalization / endianness invariants +# (diff-scoped blocking gate on PRs, per-pack sweep nightly) +# * suppression hygiene — flag suppressions that no longer match a live mutant +# All three share the scorer scripts/mutation_gate.py. See mutants/operators/. + on: pull_request: workflow_dispatch: @@ -126,6 +135,176 @@ jobs: mutation-report.md if-no-files-found: ignore + # ── spec-gate: universalmutator + comby custom operators, diff-scoped on PRs ─ + # The domain-specific complement to cargo-mutants: comby structural operators + # for trie arithmetic, canonicalization, and endianness (mutants/operators/). + # Only files+lines the PR changed are mutated, so most PRs generate zero + # spec-gate mutants and this is near-instant. Shares the scorer with + # cargo-mutants (scripts/mutation_gate.py) via the umutate.py results contract. + spec-gate: + name: spec-gate + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # need base history for the diff + persist-credentials: false + + - name: Install Rust + # Lint-tolerant mutant analysis: a valid mutant can trip a warning (e.g. + # deleting a sort leaves its `let mut` binding unused), and under the + # action's default RUSTFLAGS="-D warnings" that testable mutant would + # fail `cargo check` and be miscounted unviable instead of exercised. + # Empty rustflags keeps warnings non-fatal; genuine type/scope errors + # still fail, so real unviable mutants are still caught. (flyq, PR #143) + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "" + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Install universalmutator + # Pin the version: a new release can change generated mutants and shift + # the gate baseline. + run: pipx install universalmutator==1.14.1 + + - name: Install comby + # Pin the version (reproducible mutant generation). The prebuilt binary + # links against libev/libpcre, which are not on the runner by default. + run: | + sudo apt-get update + sudo apt-get install -y libpcre3 libev4 + sudo curl -fsSL https://github.com/comby-tools/comby/releases/download/1.8.1/comby-1.8.1-x86_64-linux -o /usr/local/bin/comby + sudo chmod +x /usr/local/bin/comby + comby -version + + - name: Run diff-scoped spec-gate mutation + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + python3 scripts/umutate.py run \ + --diff "origin/${BASE_REF}" \ + --output target/umutate + + - name: Score and gate + run: | + python3 scripts/mutation_gate.py report \ + --results target/umutate \ + --suppressions mutants/suppressions.toml \ + --comment spec-gate-report.md \ + --summary "$GITHUB_STEP_SUMMARY" + + - name: Upsert PR comment + if: always() && github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v7 + with: + script: | + const fs = require("fs"); + let body; + try { + body = fs.readFileSync("spec-gate-report.md", "utf8"); + } catch { + body = "## Spec-gate mutation\n\nNo report produced."; + } + const marker = ""; + body = `${marker}\n${body}`; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find(c => c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Upload spec-gate artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: salt-spec-gate-results + path: | + target/umutate + spec-gate-report.md + if-no-files-found: ignore + + # ── spec-gate: nightly full sweep, sharded by pack (informational) ────────── + # Each pack stays bounded (analyze is sequential). Surfaces survivors but does + # not fail the workflow — triage into suppressions or killing tests. + spec-gate-sweep: + name: spec-gate sweep (${{ matrix.pack }}) + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 330 + strategy: + fail-fast: false + matrix: + pack: [trie, canon, endian] + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install Rust + # Lint-tolerant mutant analysis; see the spec-gate job. (flyq, PR #143) + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "" + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Install universalmutator + run: pipx install universalmutator==1.14.1 + + - name: Install comby + run: | + sudo apt-get update + sudo apt-get install -y libpcre3 libev4 + sudo curl -fsSL https://github.com/comby-tools/comby/releases/download/1.8.1/comby-1.8.1-x86_64-linux -o /usr/local/bin/comby + sudo chmod +x /usr/local/bin/comby + comby -version + + - name: Run spec-gate mutation (${{ matrix.pack }}) + run: | + python3 scripts/umutate.py run --packs ${{ matrix.pack }} \ + --output "target/umutate/${{ matrix.pack }}" + + - name: Score (informational) + run: | + python3 scripts/mutation_gate.py report \ + --results "target/umutate/${{ matrix.pack }}" \ + --suppressions mutants/suppressions.toml \ + --summary "$GITHUB_STEP_SUMMARY" || true + + - name: Upload spec-gate sweep artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: salt-spec-gate-sweep-${{ matrix.pack }} + path: target/umutate/${{ matrix.pack }} + if-no-files-found: ignore + # Full-crate runs are far too large for one hosted runner within the job # timeout, so scheduled/manual runs shard the mutant universe. Keep the # shard denominator in sync with the matrix size. @@ -196,8 +375,10 @@ jobs: filters: | hygiene: - 'mutants/suppressions.toml' + - 'mutants/operators/**' - 'scripts/mutation_*.py' - 'scripts/mutation_*.sh' + - 'scripts/umutate*' - '.cargo/mutants.toml' - 'salt/src/**' @@ -222,8 +403,30 @@ jobs: with: tool: cargo-mutants@27.1.0 - - name: Build full mutant universe - run: cargo mutants --list --package salt --colors never > universe.txt + - name: Install universalmutator + run: pipx install universalmutator==1.14.1 + + - name: Install comby + run: | + sudo apt-get update + sudo apt-get install -y libpcre3 libev4 + sudo curl -fsSL https://github.com/comby-tools/comby/releases/download/1.8.1/comby-1.8.1-x86_64-linux -o /usr/local/bin/comby + sudo chmod +x /usr/local/bin/comby + comby -version + + - name: Build the full mutant universe (both engines) + run: | + # pipefail so a failing `umutate.py plan` fails the step instead of + # grep swallowing it and the orphan check running on a partial universe. + set -o pipefail + # --package salt: match the driver scope so a stale suppression can't + # be kept alive by a same-named mutant in another crate. + # --colors never: CI forces color, which would embed ANSI codes and + # break exact suppression matching. + cargo mutants --list --package salt --colors never > universe.txt + python3 scripts/umutate.py plan \ + | grep -oE 'salt/src/[^:]+:[0-9]+:[0-9]+: .*' >> universe.txt + echo "universe: $(wc -l < universe.txt) mutants" - name: Check for orphan suppressions run: | diff --git a/mutants/operators/README.md b/mutants/operators/README.md new file mode 100644 index 0000000..7e6a05d --- /dev/null +++ b/mutants/operators/README.md @@ -0,0 +1,115 @@ +# Domain-specific mutation operators + +Custom mutation-operator packs run through +[universalmutator](https://github.com/agroce/universalmutator) + +[comby](https://comby.dev), complementing the `cargo-mutants` gate (issue +#141). `cargo-mutants` ships a fixed set of structural operators and has no +plugin API, so it never generates the mutations that matter most for a +state-commitment system. Each pack targets a specific soundness or determinism +invariant. The selection filter for adding rules: *"what soundness or +determinism invariant would a surviving mutant here violate?"* Generic +arithmetic/relational/logical operators are deliberately excluded — the +`cargo-mutants` gate already owns those. + +All packs are driven by `scripts/umutate.py` and scored by the **same** gate as +`cargo-mutants` (`scripts/mutation_gate.py`), via the +caught/missed/timeout/unviable results contract. Both engines live in one CI +workflow, `.github/workflows/mutation.yml`. + +## Layout + +Each pack is a directory `mutants/operators//` with: + +- `manifest.toml` — pack config: `name`, `description`, `targets` (source + globs), optional `match` (a regex file-content filter), the `rules` filename, + the `test_cmd` whose non-zero exit means a mutant was killed, and an optional + `build_cmd` (a `cargo check`) — a mutant that fails it is classified + *unviable* rather than counted as caught (a constant-swap operator can + reference a symbol not imported in every file). +- `.rules` — the operator rules. `PATTERN ==> REPLACEMENT` per line, `#` + for comments; the pattern is a + [comby template](https://comby.dev/docs/syntax-reference). + +| Pack | Scope | Invariant guarded | +| --- | --- | --- | +| `trie` | `trie/node_utils.rs`, `trie/trie.rs`, `constant.rs`, `types.rs`, `proof/shape.rs` | Level-base and bit-width arithmetic of the two-tier trie (`STARTING_NODE_ID`, 8/24/40 splits, 256-way fanout, ceil rounding) | +| `canon` | `proof/prover.rs`, `proof/salt_witness.rs` | Canonical form: sort/dedup on the proof path must not be removable or reversible | +| `endian` | `state/hasher.rs`, `types.rs` | Byte-order determinism of the consensus hash and 12-byte metadata codec (see issue #127) | + +## Why comby + +comby matches **structurally**, not textually, which removes a class of +regex-fragility bugs: + +- A method/path-call template (`:[recv].to_le_bytes()`, + `:[ty]::from_le_bytes(:[args])`) can only match a real call — never a + substring of an identifier such as `serialize_to_le_bytes`. +- A single-word hole `:[[recv]]` matches one identifier, so a comparator or + receiver hole cannot swallow a surrounding compound expression. + +The one case comby does **not** anchor for free is a bare constant that is a +prefix of a longer one (`TRIE_WIDTH` inside `TRIE_WIDTH_BITS`): a literal +template is not word-anchored. Pin a boundary with a trailing negated-class +hole, reproduced in the replacement: + +``` +% TRIE_WIDTH:[b~[^A-Za-z0-9_]] ==> % (TRIE_WIDTH - 1):[b] +``` + +## Running + +```bash +scripts/umutate_setup.sh # installs universalmutator + comby + +# Show what WOULD be mutated (no tests run, tree untouched): +python3 scripts/umutate.py plan + +# Full pipeline for one pack -> gate-ready results dir: +python3 scripts/umutate.py run --packs trie --output target/umutate +python3 scripts/mutation_gate.py report --results target/umutate \ + --suppressions mutants/suppressions.toml + +# Diff-scoped (the PR-gate scope: only files+lines changed vs a base): +python3 scripts/umutate.py run --diff origin/main --output target/umutate +``` + +Mutants inside `#[cfg(test)]` code are pruned automatically (mutating test +assertions is meaningless), as are — in `--diff` mode — mutants outside the +changed lines. + +## CI + +`.github/workflows/mutation.yml` runs three concerns off one file: + +- **`spec-gate`** — diff-scoped, blocking, on every PR. Most PRs change no + operator site, so it generates zero mutants and is near-instant. An + unsuppressed survivor fails the gate and is upserted as a PR comment. +- **`spec-gate-sweep`** — nightly full sweep, sharded per pack, informational + (surfaces survivors to triage; does not fail the workflow). +- **`orphan-suppressions`** — folds the umutate mutant universe into the + `cargo-mutants` universe so a suppression that no longer matches any live + mutant (either engine) is flagged. + +## Triage policy + +Every **survivor** is either: + +1. a missing test — add a killing test (preferred), or +2. an equivalent mutant — record it in `mutants/suppressions.toml` with a + justification. Use a `kind = "line"` entry whose `mutant` is the survivor's + description (from `missed.txt`/`timeout.txt`) with the leading + `file:line:col:` locator stripped: matching is scoped to `file` but + deliberately drops the line number, so the suppression keeps matching when + unrelated edits shift the code. The trade-off is that two mutants with + identical text in one file cannot be pinned apart — rely on a killing test + there instead. To exclude a whole equivalent-by-construction function, use a + `kind = "function"` entry with a `pattern` regex. + +**Unviable** (non-compiling) mutants are fine in moderation; if a rule mostly +produces unviable mutants, tighten its template. + +## Known gaps + +- Families C (field-element chunk widths), D (serialization codec config) and F + (hash diffusion ops) from issue #141 are staged for a follow-up after the + first survivor triage of A/B/E. diff --git a/mutants/operators/canon/canon.rules b/mutants/operators/canon/canon.rules new file mode 100644 index 0000000..16832a2 --- /dev/null +++ b/mutants/operators/canon/canon.rules @@ -0,0 +1,24 @@ +# Family E - canonicalization removal (issue #141), comby structural rules. +# +# The proof path sorts and dedups to reach canonical form. Deleting those calls +# (or flipping comparator direction) is a soundness mutation: duplicates or +# unordered elements should be rejected or change the root. Statement deletion +# is not in cargo-mutants' repertoire. +# +# `:[[word]]` matches a single identifier (comby's whitespace-delimited hole), so +# a receiver / comparator operand can't swallow a surrounding compound +# expression the way a bare `:[hole]` would. `:[args]` spans the (possibly +# multi-line) closure by balanced delimiters. +# comby rule format: PATTERN ==> REPLACEMENT ('#' lines are comments). + +# Delete single sort/dedup statements. +:[[recv]].sort(); ==> +:[[recv]].sort_unstable(); ==> +:[[recv]].sort_unstable_by(:[args]); ==> +:[[recv]].sort_unstable_by_key(:[args]); ==> +:[[recv]].dedup(); ==> +:[[recv]].dedup_by(:[args]); ==> +sort_unstable!(:[[m]]); ==> + +# Flip comparator direction inside sort closures. +:[[x]].cmp(:[[y]]) ==> :[[y]].cmp(:[[x]]) diff --git a/mutants/operators/canon/manifest.toml b/mutants/operators/canon/manifest.toml new file mode 100644 index 0000000..c31f548 --- /dev/null +++ b/mutants/operators/canon/manifest.toml @@ -0,0 +1,20 @@ +# Operator pack: canon (Family E, issue #141) +# +# Run through universalmutator + comby, scored by the shared gate +# (scripts/mutation_gate.py). See mutants/operators/README.md. + +name = "canon" +description = "Canonicalization removal: delete sort/dedup calls and flip comparator direction on the proof path" + +targets = [ + "salt/src/proof/prover.rs", + "salt/src/proof/salt_witness.rs", +] + +# Compile check: a mutant that fails this is unviable (e.g. a constant +# swapped for one not imported in that file), classified separately rather +# than miscounted as caught. +build_cmd = "cargo check -p salt --no-default-features --features parallel" + +rules = "canon.rules" +test_cmd = "cargo nextest run -p salt --no-default-features --features parallel" diff --git a/mutants/operators/endian/endian.rules b/mutants/operators/endian/endian.rules new file mode 100644 index 0000000..953a4a6 --- /dev/null +++ b/mutants/operators/endian/endian.rules @@ -0,0 +1,12 @@ +# Family B - endianness / byte-order (issue #141), comby structural rules. +# +# Flipping endianness silently breaks determinism and cross-platform +# reproducibility (issue #127); no generic operator flips it. Structural +# method-/path-call matching means these can only hit a real `to/from_*_bytes` +# call, never a substring of an identifier like `serialize_to_le_bytes`. +# comby rule format: PATTERN ==> REPLACEMENT. + +:[recv].to_le_bytes() ==> :[recv].to_be_bytes() +:[recv].to_be_bytes() ==> :[recv].to_le_bytes() +:[ty]::from_le_bytes(:[args]) ==> :[ty]::from_be_bytes(:[args]) +:[ty]::from_be_bytes(:[args]) ==> :[ty]::from_le_bytes(:[args]) diff --git a/mutants/operators/endian/manifest.toml b/mutants/operators/endian/manifest.toml new file mode 100644 index 0000000..b1c8297 --- /dev/null +++ b/mutants/operators/endian/manifest.toml @@ -0,0 +1,24 @@ +# Operator pack: endian (Family B, issue #141) +# +# Run through universalmutator + comby, scored by the shared gate +# (scripts/mutation_gate.py). See mutants/operators/README.md. + +name = "endian" +description = "Byte-order flips on explicit to/from_*_bytes packing (determinism / cross-platform reproducibility)" + +# Candidate source files (matched from the repo root). These rules match explicit +# `to/from_*_bytes` calls; `state/ahash/convert.rs` is intentionally not a target +# because its hash readers go through `zerocopy::transmute!` / `.convert()`, which +# has no to/from_*_bytes call to flip (tracked for a future transmute operator). +targets = [ + "salt/src/state/hasher.rs", + "salt/src/types.rs", +] + +# Compile check: a mutant that fails this is unviable (e.g. a constant +# swapped for one not imported in that file), classified separately rather +# than miscounted as caught. +build_cmd = "cargo check -p salt --no-default-features --features parallel" + +rules = "endian.rules" +test_cmd = "cargo nextest run -p salt --no-default-features --features parallel" diff --git a/mutants/operators/trie/manifest.toml b/mutants/operators/trie/manifest.toml new file mode 100644 index 0000000..a4d794c --- /dev/null +++ b/mutants/operators/trie/manifest.toml @@ -0,0 +1,23 @@ +# Operator pack: trie (Family A, issue #141) +# +# Run through universalmutator + comby, scored by the shared gate +# (scripts/mutation_gate.py). See mutants/operators/README.md. + +name = "trie" +description = "Trie-structure operators: level-index base, bit-width split, fanout shift, granularity, ceil-division" + +targets = [ + "salt/src/trie/node_utils.rs", + "salt/src/trie/trie.rs", + "salt/src/constant.rs", + "salt/src/types.rs", + "salt/src/proof/shape.rs", +] + +# Compile check: a mutant that fails this is unviable (e.g. a constant +# swapped for one not imported in that file), classified separately rather +# than miscounted as caught. +build_cmd = "cargo check -p salt --no-default-features --features parallel" + +rules = "trie.rules" +test_cmd = "cargo nextest run -p salt --no-default-features --features parallel" diff --git a/mutants/operators/trie/trie.rules b/mutants/operators/trie/trie.rules new file mode 100644 index 0000000..fa6f150 --- /dev/null +++ b/mutants/operators/trie/trie.rules @@ -0,0 +1,39 @@ +# Family A - trie-structure operators (issue #141), comby structural rules. +# +# The SALT trie's correctness reduces to "right level base and right bit-width": +# pure integer arithmetic over the STARTING_NODE_ID table and the 8/24/40 +# bit-width split. A wrong constant produces a plausible, compiling, wrong +# structure that generic operators never generate. +# +# comby rule format: PATTERN ==> REPLACEMENT ('#' lines are comments). A +# trailing `:[b~[^A-Za-z0-9_]]` hole pins a non-identifier boundary (reproduced +# in the replacement) where a bare constant is a prefix of a longer one — comby +# literals are NOT word-anchored, so `TRIE_WIDTH` would otherwise rewrite inside +# `TRIE_WIDTH_BITS`. + +# A1. Level-index off-by-one in the STARTING_NODE_ID base table. The index hole +# is lowercase-only so it targets computed levels (`level`), not the named +# MAIN_/MAX_ constants (those have dedicated rules below). +STARTING_NODE_ID[:[i~[a-z_][a-z_0-9]*] + 1] ==> STARTING_NODE_ID[:[i]] +STARTING_NODE_ID[:[i~[a-z_][a-z_0-9]*] - 1] ==> STARTING_NODE_ID[:[i]] +MAIN_TRIE_LEVELS - 1 ==> MAIN_TRIE_LEVELS +MAX_SUBTREE_LEVELS - 1 ==> MAX_SUBTREE_LEVELS - 2 + +# A2. Bit-width constant confusion (swap one named width for another real one). +<< BUCKET_SLOT_BITS ==> << BUCKET_ID_BITS +>> MIN_BUCKET_SIZE_BITS ==> >> BUCKET_SLOT_BITS +<< BUCKET_SLOT_BITS ==> << (BUCKET_SLOT_BITS - 1) + +# A3. Fanout shift direction / amount (the 256-way branching). +(parent_relative_position << TRIE_WIDTH_BITS) ==> (parent_relative_position >> TRIE_WIDTH_BITS) +relative_position >> TRIE_WIDTH_BITS ==> relative_position << TRIE_WIDTH_BITS +<< TRIE_WIDTH_BITS ==> << (TRIE_WIDTH_BITS + 1) + +# A4. Segment granularity (256 slots per leaf). +>> MIN_BUCKET_SIZE_BITS ==> >> (MIN_BUCKET_SIZE_BITS + 1) +% TRIE_WIDTH:[b~[^A-Za-z0-9_]] ==> % (TRIE_WIDTH - 1):[b] + +# A5. Ceil-division rounding in the subtree-root climb (subtree_root_level). +(capacity + MIN_BUCKET_SIZE as u64 - 1) ==> (capacity + MIN_BUCKET_SIZE as u64) +capacity > MIN_BUCKET_SIZE as u64 ==> capacity >= MIN_BUCKET_SIZE as u64 +level -= 1 ==> level += 1 diff --git a/scripts/umutate.py b/scripts/umutate.py new file mode 100755 index 0000000..c3f1471 --- /dev/null +++ b/scripts/umutate.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 +"""Custom-operator mutation driver (universalmutator + comby) for the salt crate. + +This is the engine for *custom operator packs* — the domain-specific complement +to cargo-mutants. Each pack lives under `mutants/operators//` with a +`manifest.toml` (see that dir for the schema). The engine is pack-agnostic: it +runs whatever packs exist and feeds their results to the SAME shared gate that +cargo-mutants uses (`scripts/mutation_gate.py`), via the +caught/missed/timeout/unviable file contract (the `_adapt` step maps a mutant to +its stable id). + +It uses universalmutator's `mutate` to GENERATE mutants but runs and classifies +them itself (`analyze`): a clean baseline is required first, `#[cfg(test)]` code +is excluded, and timeouts are reported separately rather than counted as caught. + +Subcommands: + + plan [--diff ] [--packs a,b] + Generate rules, resolve target files, generate the mutants, and print + what WOULD be tested (counts + each mutant's stable id). Runs no tests + and does not touch the working tree's tracked files. Use this to review + operators before a real run. + + run [--diff ] [--packs a,b] --output + Full pipeline: generate -> mutate -> analyze (run the test command per + mutant) -> write caught.txt/missed.txt/unviable.txt/timeout.txt into + , ready for `mutation_gate.py report --results `. + +In `--diff ` mode only files AND lines changed vs are mutated +(the PR-gate scope); without it, every gate site in the crate is mutated. +""" +from __future__ import annotations + +import argparse +import difflib +import fnmatch +import glob +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +OPERATORS_DIR = ROOT / "mutants" / "operators" +DEFAULT_TEST_CMD = "cargo nextest run -p salt --no-default-features --features parallel" +ANALYZE_TIMEOUT = "600" # seconds per mutant; generous for a full nextest run + + +def _run(cmd, **kw): + return subprocess.run(cmd, cwd=ROOT, text=True, capture_output=True, **kw) + + +def require_tools() -> None: + # We use universalmutator's `mutate` to generate mutants but run/score them + # ourselves (so we control the baseline check and timeout classification). + missing = [t for t in ("mutate", "comby") if shutil.which(t) is None] + if missing: + sys.exit( + f"missing required tools: {', '.join(missing)}\n" + f" mutate: `pip install universalmutator` (see scripts/umutate_setup.sh)\n" + f" comby: `paru -S comby-bin` (Arch) or `bash <(curl -sL get.comby.dev)`\n" + f"see scripts/umutate_setup.sh." + ) + + +def load_packs(selected: list[str] | None) -> list[dict]: + packs = [] + for manifest in sorted(OPERATORS_DIR.glob("*/manifest.toml")): + data = tomllib.loads(manifest.read_text()) + data["_dir"] = manifest.parent + if selected and data.get("name") not in selected: + continue + packs.append(data) + if selected: + found = {p.get("name") for p in packs} + for name in selected: + if name not in found: + sys.exit(f"no operator pack named '{name}' under {OPERATORS_DIR}") + return packs + + +def ensure_rules(pack: dict) -> Path: + """Run the pack's generator (if any) and return the rules file path.""" + pdir: Path = pack["_dir"] + rules = pdir / pack.get("rules", "rules.comby") + if "generator" in pack: + gen = pdir / pack["generator"] + r = _run([sys.executable, str(gen), "--output", str(rules)]) + if r.returncode != 0: + sys.exit(f"[{pack['name']}] generator failed:\n{r.stderr}") + if not rules.exists(): + sys.exit(f"[{pack['name']}] rules file not found: {rules}") + return rules + + +def changed_files_and_lines(base: str) -> tuple[set[str], dict[str, set[int]]]: + """Parse `git diff --unified=0 ...HEAD` into changed files and the set + of new-side line numbers per file (the lines the PR added/modified).""" + r = _run(["git", "diff", "--unified=0", "--no-color", f"{base}...HEAD", "--", + "salt/src/**"]) + if r.returncode != 0: + sys.exit(f"git diff failed:\n{r.stderr}") + files: set[str] = set() + lines: dict[str, set[int]] = {} + cur: str | None = None + hunk = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + for ln in r.stdout.splitlines(): + if ln.startswith("+++ b/"): + cur = ln[6:] + files.add(cur) + lines.setdefault(cur, set()) + elif cur and (m := hunk.match(ln)): + start, count = int(m.group(1)), int(m.group(2) or "1") + # A pure-deletion hunk has new-side `+N,0` (count == 0): it adds no + # new line, so contribute nothing. (A single-line add omits the count + # entirely -> defaulted to 1 above.) + for i in range(start, start + count): + lines[cur].add(i) + return files, lines + + +def resolve_targets(pack: dict, changed: set[str] | None, + files_glob: str | None = None) -> list[str]: + cand: set[str] = set() + for pat in pack.get("targets", []): + cand.update(glob.glob(pat, root_dir=str(ROOT), recursive=True)) + cand = {f for f in cand if f.endswith(".rs") and (ROOT / f).is_file()} + if changed is not None: + cand &= changed + if files_glob: + cand = {f for f in cand if fnmatch.fnmatch(f, files_glob)} + match = pack.get("match") + if match: + rx = re.compile(match) + cand = {f for f in cand if rx.search((ROOT / f).read_text(errors="ignore"))} + return sorted(cand) + + +def generate_mutants(pack: dict, rules: Path, src: str, mutant_dir: Path, + allowed_lines: set[int] | None) -> None: + mutant_dir.mkdir(parents=True, exist_ok=True) + if allowed_lines is not None and not allowed_lines: + return # diff mode, but nothing changed in this file + # `--only ` restricts to our pack's rules (no default language rules). + # The rules path is absolute and contains ".rules", so universalmutator's + # built-in lookup fails and it falls back to opening the file directly. + # + # We do NOT use universalmutator's `--lines` for diff scoping: in comby mode + # its line filter compares against comby's byte-offset range (not line + # numbers), so it silently drops every mutant. Instead we generate all mutants + # and prune by the actual changed line, detected the same way the adapter does. + cmd = ["mutate", src, "--only", str(rules), "--comby", "--noCheck", + "--mutantDir", str(mutant_dir)] + r = _run(cmd) + if r.returncode != 0: + sys.exit(f"[{pack['name']}] mutate failed on {src}:\n{r.stderr}\n{r.stdout}") + # Prune: drop mutants in #[cfg(test)] code (always — mutating test assertions + # is meaningless and would score test edits as production gaps) and, in diff + # mode, drop mutants outside the PR's changed lines. + test_lines = _test_line_ranges((ROOT / src).read_text(errors="ignore")) + for m in list(mutant_dir.glob("*")): + if not m.is_file(): + continue + change = _first_change(ROOT / src, m) + line = change[0] if change else None + if (line is None + or line in test_lines + or (allowed_lines is not None and line not in allowed_lines)): + m.unlink() + + +def _test_line_ranges(text: str) -> set[int]: + """1-based line numbers inside `#[cfg(test)]` / `#[test]` blocks. + + A heuristic brace counter (it does not parse strings/comments), which is + sufficient to keep operator-pack mutation off test code; the production + sites these packs target live far from test modules.""" + lines = text.splitlines() + n = len(lines) + test: set[int] = set() + i = 0 + while i < n: + s = lines[i].strip() + if s.startswith("#[cfg(test)]") or s.startswith("#[test]"): + j = i + while j < n and "{" not in lines[j]: + j += 1 + if j >= n: + test.add(i + 1) + i += 1 + continue + depth, k, started = 0, j, False + while k < n: + depth += lines[k].count("{") - lines[k].count("}") + started = started or "{" in lines[k] + if started and depth <= 0: + break + k += 1 + for ln in range(i + 1, k + 2): # 1-based, inclusive of i..k + test.add(ln) + i = k + 1 + continue + i += 1 + return test + + +def _ascii(s: str) -> str: + # comby mode rewrites the file from an ASCII-normalized source, so we must + # normalize the original the same way before diffing — otherwise non-ASCII + # chars in comments (×, ≠, …) show up as spurious "changes". + return s.encode("ascii", "ignore").decode() + + +def _first_change(src_path: Path, mutant_path: Path) -> tuple[int, str, str] | None: + """Return (1-based line, before, after) of the first differing line.""" + a = [_ascii(l) for l in src_path.read_text(errors="ignore").splitlines()] + b = [_ascii(l) for l in mutant_path.read_text(errors="ignore").splitlines()] + sm = difflib.SequenceMatcher(a=a, b=b, autojunk=False) + for tag, i1, i2, j1, j2 in sm.get_opcodes(): + if tag != "equal": + before = a[i1].strip() if i1 < len(a) else "" + after = b[j1].strip() if j1 < len(b) else "" + return (i1 + 1, before, after) + return None + + +def _adapt(pack: dict, src: str, mutant_basename: str, mutant_dir: Path, + allowed_lines: set[int] | None) -> str | None: + """Map one universalmutator mutant file to a stable, gate-format id line. + + Returns `::1: -> ` or None if the mutant + falls outside the diff scope (belt-and-suspenders alongside --lines).""" + mpath = mutant_dir / mutant_basename + if not mpath.exists(): + return None + change = _first_change(ROOT / src, mpath) + if change is None: + return None + line, before, after = change + if allowed_lines is not None and line not in allowed_lines: + return None + return f"{src}:{line}:1: {pack['name']} {before} -> {after}" + + +def _run_test(test_cmd: str) -> str: + """Run the test command from the repo root; return 'fail' (non-zero exit, the + mutant is caught), 'ok' (exit 0, the mutant survived), or 'timeout'.""" + p = subprocess.Popen(test_cmd, cwd=ROOT, shell=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True) + try: + return "fail" if p.wait(timeout=float(ANALYZE_TIMEOUT)) != 0 else "ok" + except subprocess.TimeoutExpired: + os.killpg(os.getpgid(p.pid), signal.SIGKILL) # kill the whole cargo group + p.wait() + return "timeout" + + +def _compiles(build_cmd: str) -> bool: + """True if the mutated tree builds. A mutant that fails to compile is + *unviable* (an out-of-scope symbol swap, a type error) — not a killed test — + the same distinction cargo-mutants draws. Comby operators readily produce + these: e.g. `<< BUCKET_SLOT_BITS ==> << BUCKET_ID_BITS` is unviable in a file + that doesn't import BUCKET_ID_BITS. Packs opt in by declaring `build_cmd`; + without it the check is skipped, so a build failure would instead surface as + a non-zero test exit and be miscounted as caught.""" + return subprocess.run(build_cmd, cwd=ROOT, shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode == 0 + + +def baseline_ok(test_cmd: str) -> bool: + """The test command must pass on the UNMUTATED tree; otherwise every mutant + looks 'caught' (non-zero) and the run is meaningless.""" + return _run_test(test_cmd) == "ok" + + +def analyze(pack: dict, src: str, mutant_dir: Path) -> tuple[list, list, list, list]: + """Swap each mutant into `src`, classify it as caught / survived / timed-out + / unviable, and always restore the original file. + + When the pack declares a `build_cmd`, a mutant that fails it is unviable and + its test run is skipped (it would only fail the test command's own build + phase and be miscounted as caught). Otherwise the `test_cmd` exit decides: + non-zero is caught, zero is a survivor, a hang is a timeout. We run this + ourselves rather than via `analyze_mutants` so we (a) control the baseline + and (b) separate timeouts and unviables, which it would both fold into + 'killed'.""" + src_path = ROOT / src + original = src_path.read_text() + test_cmd = pack.get("test_cmd", DEFAULT_TEST_CMD) + build_cmd = pack.get("build_cmd") + caught, survived, timed_out, unviable = [], [], [], [] + bucket = {"fail": caught, "ok": survived, "timeout": timed_out} + try: + for m in sorted(p for p in mutant_dir.glob("*") if p.is_file()): + src_path.write_text(m.read_text()) + if build_cmd and not _compiles(build_cmd): + unviable.append(m.name) + continue + bucket[_run_test(test_cmd)].append(m.name) + finally: + src_path.write_text(original) + return caught, survived, timed_out, unviable + + +def cmd_plan(args) -> int: + require_tools() + packs = load_packs(args.packs) + changed_files = changed_lines = None + if args.diff: + changed_files, changed_lines = changed_files_and_lines(args.diff) + total = 0 + with tempfile.TemporaryDirectory() as tmp: + for pack in packs: + rules = ensure_rules(pack) + targets = resolve_targets(pack, changed_files, args.files) + print(f"\n## pack '{pack['name']}' — {len(targets)} target file(s)") + for src in targets: + md = Path(tmp) / pack["name"] / src.replace("/", "_") + allowed = changed_lines.get(src) if changed_lines is not None else None + generate_mutants(pack, rules, src, md, allowed) + ids = [] + for m in sorted(md.glob("*")): + if m.is_file(): + a = _adapt(pack, src, m.name, md, allowed) + if a: + ids.append(a) + total += len(ids) + if ids: + print(f" {src}: {len(ids)} mutant(s)") + for i in ids: + print(f" {i}") + print(f"\n=== {total} mutant(s) would be tested ===") + return 0 + + +def cmd_run(args) -> int: + require_tools() + out = Path(args.output) + out.mkdir(parents=True, exist_ok=True) + caught, missed, timeouts, unviable = [], [], [], [] + packs = load_packs(args.packs) + changed_files = changed_lines = None + if args.diff: + changed_files, changed_lines = changed_files_and_lines(args.diff) + + # Baseline check, run lazily: a test command must pass on the unmutated tree + # before we trust any of its mutants, but we only pay for it once there is + # actually a mutant to analyze (most PRs change no gate, so skip it then). + baselined: set[str] = set() + + def ensure_baseline(tc: str) -> None: + if tc in baselined: + return + if not baseline_ok(tc): + sys.exit( + f"baseline failed on the unmutated tree: `{tc}`\n" + f"fix the failing tests before running mutation analysis " + f"(otherwise every mutant is falsely 'caught')." + ) + baselined.add(tc) + + with tempfile.TemporaryDirectory() as tmp: + for pack in packs: + rules = ensure_rules(pack) + for src in resolve_targets(pack, changed_files, args.files): + md = Path(tmp) / pack["name"] / src.replace("/", "_") + allowed = changed_lines.get(src) if changed_lines is not None else None + generate_mutants(pack, rules, src, md, allowed) + if not any(p.is_file() for p in md.glob("*")): + continue + ensure_baseline(pack.get("test_cmd", DEFAULT_TEST_CMD)) + c, s, t, u = analyze(pack, src, md) + for names, sink in ((c, caught), (s, missed), (t, timeouts), (u, unviable)): + for name in names: + a = _adapt(pack, src, name, md, allowed) + if a: + sink.append(a) + + def write(name, items): + (out / name).write_text("\n".join(items) + ("\n" if items else "")) + + write("caught.txt", caught) + write("missed.txt", missed) + write("timeout.txt", timeouts) + write("unviable.txt", unviable) + print(f"caught: {len(caught)} missed: {len(missed)} timeout: {len(timeouts)} " + f"unviable: {len(unviable)} -> {out}") + print(f"score with: python3 scripts/mutation_gate.py report --results {out} " + f"--suppressions mutants/suppressions.toml") + return 0 + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + + def common(sp): + sp.add_argument("--diff", metavar="BASE", help="scope to files+lines changed vs BASE") + sp.add_argument("--packs", type=lambda s: s.split(","), default=None, + help="comma-separated pack names (default: all)") + sp.add_argument("--files", metavar="GLOB", default=None, + help="further restrict target files to those matching GLOB") + + pp = sub.add_parser("plan", help="show what would be mutated; run no tests") + common(pp) + pp.set_defaults(func=cmd_plan) + + pr = sub.add_parser("run", help="run the full pipeline and write gate results") + common(pr) + pr.add_argument("--output", required=True, help="results dir for mutation_gate.py") + pr.set_defaults(func=cmd_run) + + args = p.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/umutate_setup.sh b/scripts/umutate_setup.sh new file mode 100755 index 0000000..296f989 --- /dev/null +++ b/scripts/umutate_setup.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Install the custom-operator mutation toolchain: universalmutator + comby. +# +# These back the operator packs under mutants/operators/ (scripts/umutate.py), +# the domain-specific complement to cargo-mutants. comby is intentionally NOT +# pip-managed: its prebuilt binary links against libev/libpcre sonames. +set -euo pipefail + +# --- universalmutator (`mutate`, `analyze_mutants`) --------------------------- +if ! command -v mutate >/dev/null 2>&1; then + echo "Installing universalmutator into a venv..." + VENV="${HOME}/.local/share/umutate-venv" + python3 -m venv "$VENV" + "$VENV/bin/pip" install --quiet universalmutator + mkdir -p "${HOME}/.local/bin" + for s in mutate analyze_mutants; do ln -sf "$VENV/bin/$s" "${HOME}/.local/bin/$s"; done + echo " (ensure ~/.local/bin is on PATH)" +else + echo "universalmutator already installed: $(command -v mutate)" +fi + +# --- comby (structural matcher used by every operator pack) ------------------- +if ! command -v comby >/dev/null 2>&1; then + echo "Installing comby..." + if command -v paru >/dev/null 2>&1; then + paru -S --needed comby-bin || paru -S --needed comby + else + # The prebuilt comby links against libev/libpcre; install them first on + # apt-based systems (e.g. Ubuntu CI). + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update && sudo apt-get install -y libpcre3 libev4 + fi + bash <(curl -sL get.comby.dev) || { + echo "comby install failed; see https://github.com/comby-tools/comby/releases" >&2 + exit 1 + } + fi +else + echo "comby already installed: $(command -v comby)" +fi + +echo +echo "Done. Verify with: command -v mutate analyze_mutants comby"