From 752bf2cff629fe3c560a2210c007e903029d6f10 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Tue, 15 Sep 2026 18:27:58 -0700 Subject: [PATCH] Guard mutation runs against differential under-selection The default differential run can select fewer covered sites than exist after a function is added or removed, forcing a manual --mutate-all rerun. Add mutate-file.sh (auto-reruns when Selected < Covered) and clean-builds.sh (removes worker-copy bloat), and document them plus the intrinsic upper-bound Gherkin survivors in the architect cheat-sheet. By specifier. --- docs/architect-startup.md | 23 +++++++++---- swarmforge/scripts/clean-builds.sh | 16 +++++++++ swarmforge/scripts/mutate-file.sh | 53 ++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 7 deletions(-) create mode 100755 swarmforge/scripts/clean-builds.sh create mode 100755 swarmforge/scripts/mutate-file.sh diff --git a/docs/architect-startup.md b/docs/architect-startup.md index 683f998..3f68d83 100644 --- a/docs/architect-startup.md +++ b/docs/architect-startup.md @@ -20,11 +20,13 @@ the last run. If you add tests to kill survivors but the SOURCE is unchanged, a plain re-run reports `Killed: 0, Survived: 0, Uncovered: 0` and silently skips the survivors. -**Always pass `--mutate-all` when re-running mutation on a file that already has -a manifest** (especially right after adding hardening tests). New files (no -manifest) run fully on the first pass. - -Canonical invocation (run from the component dir so `npm test` works): +Prefer `swarmforge/scripts/mutate-file.sh`, which detects differential +under-selection automatically (`Selected < Covered`) and reruns once with +`--mutate-all`. Run it from the component directory: +```bash +../swarmforge/scripts/mutate-file.sh src/.js --max-workers 8 +``` +Manual fallback when you must run the binary directly: ```bash node ../psf-memo-client/node_modules/mutate4javascript/bin/mutate4javascript.js \ src/.js --max-workers 8 --mutate-all @@ -44,6 +46,14 @@ bb gherkin-mutator \ Parse survivors with a small python one-liner over `d['results']` (filter `Status == 'survived'`). +### Equivalent upper-bound survivors (no project filter hook) +APS `gherkin-mutator` exposes no project-specific mutation-filter hook, so +upper-bound steps such as `the store was read at most entries` +leave survivors when the bound is mutated upward (a larger bound can never +fail). Treat these as intrinsic equivalents, document them in the review +summary, and do not chase them. Prefer exact counts or independently-tied +fixture values when a bound must itself be mutatable. + ## Long runs - Mutation and gherkin-mutator runs take 60-180s each. Use `--max-workers 8` / `--workers 8` and `--status-interval` so progress is visible. @@ -56,8 +66,7 @@ Stale `tmp/acceptance` (LevelDB dirs) and `target/mutation-workers` can bloat to hang. `architect-startup.sh` now flags any of these over 100MB as `[FAIL]`. Clean them before any mutation run: ```bash -rm -rf psf-memo-db/tmp/acceptance psf-memo-db/target/mutation-workers \ - psf-memo-client/tmp/acceptance psf-memo-client/target/mutation-workers +swarmforge/scripts/clean-builds.sh ``` This cut a notifications-query mutation run from 20+ minutes to ~2 minutes. diff --git a/swarmforge/scripts/clean-builds.sh b/swarmforge/scripts/clean-builds.sh new file mode 100755 index 0000000..b4f2925 --- /dev/null +++ b/swarmforge/scripts/clean-builds.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Remove gitignored build directories that bloat mutate4javascript worker +# copies and make mutation runs appear to hang. +# +# Usage: clean-builds.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +for component in psf-memo-client psf-memo-db psf-memo-indexer; do + rm -rf "$ROOT/$component/tmp/acceptance" \ + "$ROOT/$component/target/mutation-workers" +done + +echo "clean-builds: removed tmp/acceptance and target/mutation-workers for all components" diff --git a/swarmforge/scripts/mutate-file.sh b/swarmforge/scripts/mutate-file.sh new file mode 100755 index 0000000..46981c3 --- /dev/null +++ b/swarmforge/scripts/mutate-file.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Run mutate4javascript on one source file and guard against silent +# differential under-selection. +# +# The default differential run can select fewer covered mutations than exist +# after a function is added or removed. When Selected < Covered, this wrapper +# reruns once with --mutate-all so new mutations are not skipped. +# +# Run from the component directory (so the tool's `npm test` baseline works): +# swarmforge/scripts/mutate-file.sh src/.js [options...] +set -uo pipefail + +if [ "$#" -lt 1 ]; then + echo "usage: mutate-file.sh [mutate4javascript options...]" >&2 + exit 2 +fi + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +BIN="${MUTATE4JS_BIN:-$ROOT/psf-memo-client/node_modules/mutate4javascript/bin/mutate4javascript.js}" + +if [ ! -f "$BIN" ]; then + echo "mutate-file: cannot find mutate4javascript at $BIN" >&2 + exit 1 +fi + +SOURCE="$1" +shift + +already_all=0 +for arg in "$@"; do + [ "$arg" = "--mutate-all" ] && already_all=1 +done + +mkdir -p "$ROOT/tmp" +OUT="$(mktemp "$ROOT/tmp/mutate-file.XXXXXX")" +trap 'rm -f "$OUT"' EXIT + +node "$BIN" "$SOURCE" "$@" 2>&1 | tee "$OUT" +status=${PIPESTATUS[0]} + +if [ "$already_all" -eq 0 ]; then + covered="$(grep -oE 'Covered mutation sites: [0-9]+' "$OUT" | grep -oE '[0-9]+' | tail -1)" + selected="$(grep -oE 'Selected mutation sites: [0-9]+' "$OUT" | grep -oE '[0-9]+' | tail -1)" + if [ -n "${covered:-}" ] && [ -n "${selected:-}" ] && [ "$selected" -lt "$covered" ]; then + echo "" >&2 + echo "mutate-file: selected $selected of $covered covered sites (differential under-selection); rerunning with --mutate-all" >&2 + node "$BIN" "$SOURCE" "$@" --mutate-all + status=$? + fi +fi + +exit "$status"