mutation-table: a control run of the unmutated snapshot gates the fan-out (SNAPSHOT RED)
A test that cannot run twice (create_dir("target/fresh-marker").unwrap())
passes the baseline and then fails in every mutant's container, so every
mutant read CAUGHT and the job exited GREEN (Soul F1). The baseline's test
loop is now one function, `unmutated`, run again from the committed snapshot
in a --control container before any mutant; red or empty there aborts with
SNAPSHOT RED, exit 2, no rows counted. Mock case E3 (oncetest.sh) fails if
the control is removed. Reproducer crate: tools/fixtures/mutation-probe/fresh.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens
parent
e5bdc386df
commit
b4942b7571
6 changed files with 73 additions and 72 deletions
|
|
@ -1,18 +0,0 @@
|
|||
SYNTHETIC: hand-written in the format jest prints; not captured from a run.
|
||||
FAIL src/add.test.js
|
||||
● Test suite failed to run
|
||||
|
||||
Jest encountered an unexpected token
|
||||
|
||||
SyntaxError: /work/src/add.js: Unexpected token (1:11)
|
||||
|
||||
> 1 | add = BROKEN BROKEN
|
||||
| ^
|
||||
|
||||
at Parser._raise (node_modules/@babel/parser/lib/index.js:571:17)
|
||||
|
||||
Test Suites: 1 failed, 1 total
|
||||
Tests: 0 total
|
||||
Snapshots: 0 total
|
||||
Time: 0.298 s
|
||||
Ran all test suites.
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
SYNTHETIC: hand-written in the format jest prints; not captured from a run.
|
||||
FAIL src/add.test.js
|
||||
add
|
||||
✓ adds (2 ms)
|
||||
✕ carries (3 ms)
|
||||
○ skipped subtracts
|
||||
|
||||
● add › carries
|
||||
|
||||
expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 10
|
||||
Received: 9
|
||||
|
||||
4 | test('carries', () => {
|
||||
> 5 | expect(add(5, 5)).toBe(10);
|
||||
| ^
|
||||
|
||||
at Object.<anonymous> (src/add.test.js:5:21)
|
||||
|
||||
Test Suites: 1 failed, 1 total
|
||||
Tests: 1 failed, 1 skipped, 1 passed, 3 total
|
||||
Snapshots: 0 total
|
||||
Time: 0.412 s
|
||||
Ran all test suites.
|
||||
6
tools/fixtures/mutation-mock/mutants-i.txt
Normal file
6
tools/fixtures/mutation-mock/mutants-i.txt
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
=== untested-equiv
|
||||
file: untested.txt
|
||||
--- before
|
||||
1
|
||||
--- after
|
||||
2
|
||||
6
tools/fixtures/mutation-mock/oncetest.sh
Normal file
6
tools/fixtures/mutation-mock/oncetest.sh
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
# Mock test that can run only once per tree, like create_dir("target/fresh-marker").unwrap():
|
||||
# green in the baseline, red from the snapshot, whatever the mutation. See tools/fixtures/mutation-probe/fresh.
|
||||
if mkdir once-marker 2>/dev/null; then
|
||||
echo "test tests::f_fresh ... ok"; echo "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s"; exit 0
|
||||
fi
|
||||
echo "test tests::f_fresh ... FAILED"; echo "test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s"; exit 101
|
||||
|
|
@ -66,7 +66,11 @@
|
|||
# Before any mutant, every distinct test command and BUILD run UNMUTATED. If one
|
||||
# fails or a test command runs zero tests the job aborts with `BASELINE RED` or
|
||||
# `BASELINE RAN ZERO TESTS`, because a red or empty baseline would make every
|
||||
# mutant look killed or every one survive.
|
||||
# mutant look killed or every one survive. Then the same unmutated run repeats in
|
||||
# a fresh container from the snapshot (the state every mutant starts from); red
|
||||
# there aborts with `SNAPSHOT RED` and no row counted: a test that cannot run
|
||||
# twice (it creates a directory, claims a port) passes the baseline and fails
|
||||
# every mutant, which would read as every mutant CAUGHT.
|
||||
# Exit 0 means every mutant was CAUGHT or UNVIABLE and nothing else happened.
|
||||
#
|
||||
# DELETION LINE: this runner replaces the per-agent runmut/mutrun/strykrun
|
||||
|
|
@ -76,7 +80,7 @@
|
|||
#
|
||||
# Modes (this one file, three places): no flag = launcher on the workstation;
|
||||
# --host = the driver on the Yggdrasil host, started by ygg-verify.sh RUN_ON_HOST=1;
|
||||
# --baseline / --mutant N = the work inside a container.
|
||||
# --baseline / --control / --mutant N = the work inside a container.
|
||||
set -euo pipefail
|
||||
|
||||
SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" # ygg-verify.sh carries only lines tagged with its per-run nonce
|
||||
|
|
@ -84,7 +88,7 @@ SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" # ygg-verify.sh carries only lin
|
|||
OUTCOMES=(CAUGHT SURVIVED UNVIABLE TIMEOUT NOAPPLY ERROR); CODE0=10
|
||||
|
||||
# ---- launcher (workstation) -------------------------------------------------
|
||||
if [ "${1:-}" != --host ] && [ "${1:-}" != --baseline ] && [ "${1:-}" != --mutant ]; then
|
||||
if [ "${1:-}" != --host ] && [ "${1:-}" != --baseline ] && [ "${1:-}" != --mutant ] && [ "${1:-}" != --control ]; then
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
repo=${1:?usage: mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]}
|
||||
rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.}
|
||||
|
|
@ -139,7 +143,14 @@ if [ "$1" = --host ]; then
|
|||
t0=$(date +%s)
|
||||
sudo docker commit "$base" "$snap" >/dev/null || { echo "$SUMTAG: docker commit failed"; exit 2; }
|
||||
sudo docker rm -f "$base" >/dev/null
|
||||
echo "$SUMTAG: SNAPSHOT committed in $(( $(date +%s) - t0 ))s; each mutant runs in a fresh container from it"
|
||||
echo "$SUMTAG: SNAPSHOT committed in $(( $(date +%s) - t0 ))s; a control run of the unmutated snapshot comes first, then each mutant in a fresh container from it"
|
||||
# A test that passed once and cannot pass again (it creates a directory, claims a port, appends to a
|
||||
# fixture) is green in the baseline and red in every mutant's container, which would read as a fully
|
||||
# killed suite. The control is the baseline's tests run once more from the snapshot, unmutated.
|
||||
# shellcheck disable=SC2086
|
||||
drun --rm --name "$YGG_CNAME-ctl" "${common[@]}" -w /src ${DOCKER_ARGS:-} "$snap" bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --control"
|
||||
rc=$?
|
||||
[ "$rc" = 0 ] || { echo "$SUMTAG: the unmutated snapshot is red though the baseline was green, so no mutant result would mean anything; no rows counted"; exit "$rc"; }
|
||||
|
||||
declare -A tally; for o in "${OUTCOMES[@]}"; do tally[$o]=0; done
|
||||
for n in $(seq 1 "$count"); do
|
||||
|
|
@ -173,6 +184,38 @@ tmo=${MUT_TIMEOUT:-600}
|
|||
# run <command> <logfile>: prints the exit status; 124 means timeout
|
||||
run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; }
|
||||
|
||||
# unmutated <BASELINE|SNAPSHOT> <count>: run every distinct test command, then BUILD, on the tree as it
|
||||
# stands. A red or empty run aborts the job (exit 2) under that name: a red baseline would make every
|
||||
# mutant look killed, an empty one every mutant survive, and a SNAPSHOT that is red where the BASELINE
|
||||
# was green (a test that cannot run twice) would do the first again.
|
||||
unmutated() {
|
||||
local what=$1 count=$2 n t rc scan tests; declare -A based
|
||||
for n in $(seq 1 "$count"); do
|
||||
t=$(cat "$M/$n/test"); t=${t:-$TEST}
|
||||
[ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; }
|
||||
[ -z "${based[$t]:-}" ] || continue
|
||||
based[$t]=1
|
||||
rc=$(run "$t" "$M/base.log")
|
||||
scan=$(tp_scan "$M/base.log")
|
||||
tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}')
|
||||
if [ "$rc" != 0 ]; then
|
||||
echo "$SUMTAG: $what RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2
|
||||
fi
|
||||
if [ "$tests" -eq 0 ]; then
|
||||
echo "$SUMTAG: $what RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2
|
||||
fi
|
||||
done
|
||||
# BUILD runs unmutated too. A mistyped or missing build command fails every
|
||||
# mutant, and a table of all-UNVIABLE rows reads as a fully killed suite.
|
||||
if [ -n "${BUILD:-}" ]; then
|
||||
rc=$(run "$BUILD" "$M/base.log")
|
||||
if [ "$rc" != 0 ]; then
|
||||
echo "$SUMTAG: $what RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD"
|
||||
tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "$1" = --baseline ]; then
|
||||
# The revision lives in this container's own filesystem, so `docker commit` keeps it.
|
||||
if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "$SUMTAG: cannot copy $SEED into $SRC"; exit 2; }; fi
|
||||
|
|
@ -205,37 +248,20 @@ if [ "$1" = --baseline ]; then
|
|||
fi
|
||||
count=$(cat "$M/count")
|
||||
echo "$SUMTAG: START $count mutants; per-run timeout ${tmo}s; whole-job timeout ${JOB_TIMEOUT:-unknown}s; one row per mutant follows as it finishes, so a timeout keeps the finished rows"
|
||||
declare -A based
|
||||
for n in $(seq 1 "$count"); do
|
||||
t=$(cat "$M/$n/test"); t=${t:-$TEST}
|
||||
[ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; }
|
||||
[ -z "${based[$t]:-}" ] || continue
|
||||
based[$t]=1
|
||||
rc=$(run "$t" "$M/base.log")
|
||||
scan=$(tp_scan "$M/base.log")
|
||||
tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}')
|
||||
if [ "$rc" != 0 ]; then
|
||||
echo "$SUMTAG: BASELINE RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2
|
||||
fi
|
||||
if [ "$tests" -eq 0 ]; then
|
||||
echo "$SUMTAG: BASELINE RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2
|
||||
fi
|
||||
done
|
||||
# BUILD runs unmutated too. A mistyped or missing build command fails every
|
||||
# mutant, and a table of all-UNVIABLE rows reads as a fully killed suite.
|
||||
if [ -n "${BUILD:-}" ]; then
|
||||
rc=$(run "$BUILD" "$M/base.log")
|
||||
if [ "$rc" != 0 ]; then
|
||||
echo "$SUMTAG: BASELINE RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD"
|
||||
tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2
|
||||
fi
|
||||
fi
|
||||
unmutated BASELINE "$count"
|
||||
# A baseline that rewrote a tracked file would hand every mutant a changed tree
|
||||
# and make `git diff` blind to the mutation; untracked and ignored output stays.
|
||||
git -C "$SRC" reset -q --hard 2>/dev/null
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$1" = --control ]; then
|
||||
# One unmutated run from the committed snapshot, the state every mutant starts from.
|
||||
cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
|
||||
unmutated SNAPSHOT "$(cat "$M/count")"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --mutant <n>
|
||||
n=${2:?mutant number}
|
||||
cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
|
||||
|
|
|
|||
|
|
@ -112,6 +112,12 @@ mt_k=$(mktemp); MT_KILLED=$mt_k mt mutants-c.txt 'bash test.sh'; rm -f "$mt_k"
|
|||
[ "$mt_rc" = 124 ] || { echo "FAIL mutation E2: exit $mt_rc, want 124"; fail=1; }
|
||||
mtcheck E2-killed "JOB KILLED by the watchdog before mutant 1 of 2" "!CAUGHT"
|
||||
mtclean E2
|
||||
# E3 (Soul, F1): a test that cannot run twice is green in the baseline and red in every mutant's
|
||||
# container; the control run of the unmutated snapshot must abort the job, not report a killed suite
|
||||
mt mutants-i.txt 'bash oncetest.sh'
|
||||
[ "$mt_rc" = 2 ] || { echo "FAIL mutation E3: exit $mt_rc, want 2"; fail=1; }
|
||||
mtcheck E3-snapshot-red "SNAPSHOT RED (exit 101) for: bash oncetest.sh" "no rows counted" "!CAUGHT" "!MUTATION"
|
||||
mtclean E3
|
||||
# F: LF before-text matches a CRLF file
|
||||
mt mutants-f.txt 'bash test.sh'
|
||||
mtcheck F-crlf "crlf-multiline | CAUGHT | tests::d_win"
|
||||
|
|
|
|||
Loading…
Reference in a new issue