diff --git a/tools/fixtures/logs/jest-load-failure.log b/tools/fixtures/logs/jest-load-failure.log deleted file mode 100644 index c04a4d2..0000000 --- a/tools/fixtures/logs/jest-load-failure.log +++ /dev/null @@ -1,18 +0,0 @@ -SYNTHETIC: hand-written in the format jest prints; not captured from a run. - FAIL src/add.test.js - ● Test suite failed to run - - Jest encountered an unexpected token - - SyntaxError: /work/src/add.js: Unexpected token (1:11) - - > 1 | add = BROKEN BROKEN - | ^ - - at Parser._raise (node_modules/@babel/parser/lib/index.js:571:17) - -Test Suites: 1 failed, 1 total -Tests: 0 total -Snapshots: 0 total -Time: 0.298 s -Ran all test suites. diff --git a/tools/fixtures/logs/jest-red.log b/tools/fixtures/logs/jest-red.log deleted file mode 100644 index 10a5b07..0000000 --- a/tools/fixtures/logs/jest-red.log +++ /dev/null @@ -1,25 +0,0 @@ -SYNTHETIC: hand-written in the format jest prints; not captured from a run. - FAIL src/add.test.js - add - ✓ adds (2 ms) - ✕ carries (3 ms) - ○ skipped subtracts - - ● add › carries - - expect(received).toBe(expected) // Object.is equality - - Expected: 10 - Received: 9 - - 4 | test('carries', () => { - > 5 | expect(add(5, 5)).toBe(10); - | ^ - - at Object. (src/add.test.js:5:21) - -Test Suites: 1 failed, 1 total -Tests: 1 failed, 1 skipped, 1 passed, 3 total -Snapshots: 0 total -Time: 0.412 s -Ran all test suites. diff --git a/tools/fixtures/mutation-mock/mutants-i.txt b/tools/fixtures/mutation-mock/mutants-i.txt new file mode 100644 index 0000000..c355428 --- /dev/null +++ b/tools/fixtures/mutation-mock/mutants-i.txt @@ -0,0 +1,6 @@ +=== untested-equiv +file: untested.txt +--- before +1 +--- after +2 diff --git a/tools/fixtures/mutation-mock/oncetest.sh b/tools/fixtures/mutation-mock/oncetest.sh new file mode 100644 index 0000000..e1f91e1 --- /dev/null +++ b/tools/fixtures/mutation-mock/oncetest.sh @@ -0,0 +1,6 @@ +# Mock test that can run only once per tree, like create_dir("target/fresh-marker").unwrap(): +# green in the baseline, red from the snapshot, whatever the mutation. See tools/fixtures/mutation-probe/fresh. +if mkdir once-marker 2>/dev/null; then + echo "test tests::f_fresh ... ok"; echo "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s"; exit 0 +fi +echo "test tests::f_fresh ... FAILED"; echo "test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s"; exit 101 diff --git a/tools/mutation-table.sh b/tools/mutation-table.sh index 7674e45..9e19efd 100644 --- a/tools/mutation-table.sh +++ b/tools/mutation-table.sh @@ -66,7 +66,11 @@ # Before any mutant, every distinct test command and BUILD run UNMUTATED. If one # fails or a test command runs zero tests the job aborts with `BASELINE RED` or # `BASELINE RAN ZERO TESTS`, because a red or empty baseline would make every -# mutant look killed or every one survive. +# mutant look killed or every one survive. Then the same unmutated run repeats in +# a fresh container from the snapshot (the state every mutant starts from); red +# there aborts with `SNAPSHOT RED` and no row counted: a test that cannot run +# twice (it creates a directory, claims a port) passes the baseline and fails +# every mutant, which would read as every mutant CAUGHT. # Exit 0 means every mutant was CAUGHT or UNVIABLE and nothing else happened. # # DELETION LINE: this runner replaces the per-agent runmut/mutrun/strykrun @@ -76,7 +80,7 @@ # # Modes (this one file, three places): no flag = launcher on the workstation; # --host = the driver on the Yggdrasil host, started by ygg-verify.sh RUN_ON_HOST=1; -# --baseline / --mutant N = the work inside a container. +# --baseline / --control / --mutant N = the work inside a container. set -euo pipefail SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" # ygg-verify.sh carries only lines tagged with its per-run nonce @@ -84,7 +88,7 @@ SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" # ygg-verify.sh carries only lin OUTCOMES=(CAUGHT SURVIVED UNVIABLE TIMEOUT NOAPPLY ERROR); CODE0=10 # ---- launcher (workstation) ------------------------------------------------- -if [ "${1:-}" != --host ] && [ "${1:-}" != --baseline ] && [ "${1:-}" != --mutant ]; then +if [ "${1:-}" != --host ] && [ "${1:-}" != --baseline ] && [ "${1:-}" != --mutant ] && [ "${1:-}" != --control ]; then here=$(cd "$(dirname "$0")" && pwd) repo=${1:?usage: mutation-table.sh [workdir]} rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.} @@ -139,7 +143,14 @@ if [ "$1" = --host ]; then t0=$(date +%s) sudo docker commit "$base" "$snap" >/dev/null || { echo "$SUMTAG: docker commit failed"; exit 2; } sudo docker rm -f "$base" >/dev/null - echo "$SUMTAG: SNAPSHOT committed in $(( $(date +%s) - t0 ))s; each mutant runs in a fresh container from it" + echo "$SUMTAG: SNAPSHOT committed in $(( $(date +%s) - t0 ))s; a control run of the unmutated snapshot comes first, then each mutant in a fresh container from it" + # A test that passed once and cannot pass again (it creates a directory, claims a port, appends to a + # fixture) is green in the baseline and red in every mutant's container, which would read as a fully + # killed suite. The control is the baseline's tests run once more from the snapshot, unmutated. + # shellcheck disable=SC2086 + drun --rm --name "$YGG_CNAME-ctl" "${common[@]}" -w /src ${DOCKER_ARGS:-} "$snap" bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --control" + rc=$? + [ "$rc" = 0 ] || { echo "$SUMTAG: the unmutated snapshot is red though the baseline was green, so no mutant result would mean anything; no rows counted"; exit "$rc"; } declare -A tally; for o in "${OUTCOMES[@]}"; do tally[$o]=0; done for n in $(seq 1 "$count"); do @@ -173,6 +184,38 @@ tmo=${MUT_TIMEOUT:-600} # run : prints the exit status; 124 means timeout run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; } +# unmutated : run every distinct test command, then BUILD, on the tree as it +# stands. A red or empty run aborts the job (exit 2) under that name: a red baseline would make every +# mutant look killed, an empty one every mutant survive, and a SNAPSHOT that is red where the BASELINE +# was green (a test that cannot run twice) would do the first again. +unmutated() { + local what=$1 count=$2 n t rc scan tests; declare -A based + for n in $(seq 1 "$count"); do + t=$(cat "$M/$n/test"); t=${t:-$TEST} + [ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; } + [ -z "${based[$t]:-}" ] || continue + based[$t]=1 + rc=$(run "$t" "$M/base.log") + scan=$(tp_scan "$M/base.log") + tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}') + if [ "$rc" != 0 ]; then + echo "$SUMTAG: $what RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 + fi + if [ "$tests" -eq 0 ]; then + echo "$SUMTAG: $what RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2 + fi + done + # BUILD runs unmutated too. A mistyped or missing build command fails every + # mutant, and a table of all-UNVIABLE rows reads as a fully killed suite. + if [ -n "${BUILD:-}" ]; then + rc=$(run "$BUILD" "$M/base.log") + if [ "$rc" != 0 ]; then + echo "$SUMTAG: $what RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD" + tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 + fi + fi +} + if [ "$1" = --baseline ]; then # The revision lives in this container's own filesystem, so `docker commit` keeps it. if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "$SUMTAG: cannot copy $SEED into $SRC"; exit 2; }; fi @@ -205,37 +248,20 @@ if [ "$1" = --baseline ]; then fi count=$(cat "$M/count") echo "$SUMTAG: START $count mutants; per-run timeout ${tmo}s; whole-job timeout ${JOB_TIMEOUT:-unknown}s; one row per mutant follows as it finishes, so a timeout keeps the finished rows" - declare -A based - for n in $(seq 1 "$count"); do - t=$(cat "$M/$n/test"); t=${t:-$TEST} - [ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; } - [ -z "${based[$t]:-}" ] || continue - based[$t]=1 - rc=$(run "$t" "$M/base.log") - scan=$(tp_scan "$M/base.log") - tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}') - if [ "$rc" != 0 ]; then - echo "$SUMTAG: BASELINE RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 - fi - if [ "$tests" -eq 0 ]; then - echo "$SUMTAG: BASELINE RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2 - fi - done - # BUILD runs unmutated too. A mistyped or missing build command fails every - # mutant, and a table of all-UNVIABLE rows reads as a fully killed suite. - if [ -n "${BUILD:-}" ]; then - rc=$(run "$BUILD" "$M/base.log") - if [ "$rc" != 0 ]; then - echo "$SUMTAG: BASELINE RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD" - tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 - fi - fi + unmutated BASELINE "$count" # A baseline that rewrote a tracked file would hand every mutant a changed tree # and make `git diff` blind to the mutation; untracked and ignored output stays. git -C "$SRC" reset -q --hard 2>/dev/null exit 0 fi +if [ "$1" = --control ]; then + # One unmutated run from the committed snapshot, the state every mutant starts from. + cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; } + unmutated SNAPSHOT "$(cat "$M/count")" + exit 0 +fi + # --mutant n=${2:?mutant number} cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; } diff --git a/tools/test-tools.sh b/tools/test-tools.sh index 3149243..d08b979 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -112,6 +112,12 @@ mt_k=$(mktemp); MT_KILLED=$mt_k mt mutants-c.txt 'bash test.sh'; rm -f "$mt_k" [ "$mt_rc" = 124 ] || { echo "FAIL mutation E2: exit $mt_rc, want 124"; fail=1; } mtcheck E2-killed "JOB KILLED by the watchdog before mutant 1 of 2" "!CAUGHT" mtclean E2 +# E3 (Soul, F1): a test that cannot run twice is green in the baseline and red in every mutant's +# container; the control run of the unmutated snapshot must abort the job, not report a killed suite +mt mutants-i.txt 'bash oncetest.sh' +[ "$mt_rc" = 2 ] || { echo "FAIL mutation E3: exit $mt_rc, want 2"; fail=1; } +mtcheck E3-snapshot-red "SNAPSHOT RED (exit 101) for: bash oncetest.sh" "no rows counted" "!CAUGHT" "!MUTATION" +mtclean E3 # F: LF before-text matches a CRLF file mt mutants-f.txt 'bash test.sh' mtcheck F-crlf "crlf-multiline | CAUGHT | tests::d_win"