f68bd4099f
Jest without a TTY prints "FAIL jt/add.test.js" with one space; the parser matched two, so a real jest load failure read ERROR, not UNVIABLE. The old `^ +● ` denylist also named process.exit's "● process.exit called with "1"" as a killing test (CAUGHT). Names now need a FAIL file header above them; a crash with no named failing test is ERROR. The "no test counts recognised" line names jest. The synthetic jest logs and mock are replaced by real jest 29.7.0 output captured on Yggdrasil from tools/fixtures/mutation-probe/jsr (green, wrong, load, exit runs in logs/jest-real.log). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
194 lines
13 KiB
Shell
194 lines
13 KiB
Shell
#!/usr/bin/env bash
|
||
# Local, cheap checks of the verdict-bearing text parsers: tools/testparse.sh and
|
||
# the summary and wait modes of tools/stopgap/ygg-verify.sh, over
|
||
# tools/fixtures/logs. cargo-*.log (the tiny-crate fixture) and mutation-table-nonce.log (the soul2
|
||
# reproducer) are real Yggdrasil runs, 2026-10-01; soul-s5/*.log are real
|
||
# dotnet, pytest, vitest and cargo runs of Soul's probe repo (several runs per
|
||
# log, split by "=== MARK <name>" lines); the rest are hand-written.
|
||
# Run: bash tools/test-tools.sh (no network, no builds)
|
||
set -uo pipefail
|
||
here=$(cd "$(dirname "$0")" && pwd)
|
||
L=$here/fixtures/logs; V=$here/stopgap/ygg-verify.sh
|
||
T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
|
||
fail=0
|
||
NONCE=''
|
||
# seg <log> <mark> -> a file holding just that run of a multi-run log
|
||
seg() { local o="$T/$(basename "$1").$2"; awk -v m="=== MARK $2" '/^=== MARK /{on=($0==m);next} on' "$L/$1" >"$o"; echo "$o"; }
|
||
# check <log path or name> <exit status> <must-contain> [<must-not-contain>]
|
||
check() {
|
||
local p=$1; [ -f "$p" ] || p=$L/$1
|
||
out=$(bash "$V" summarize "$p" "$2" ${NONCE:+"$NONCE"})
|
||
printf '%s\n' "$out" | grep -qF -- "$3" || { echo "FAIL $1: missing: $3"; fail=1; return; }
|
||
if [ -n "${4:-}" ] && printf '%s\n' "$out" | grep -qF -- "$4"; then echo "FAIL $1: must not contain: $4"; fail=1; return; fi
|
||
[ "$(printf '%s\n' "$out" | wc -l)" -le 81 ] || { echo "FAIL $1: summary over 81 lines"; fail=1; return; }
|
||
echo "ok $(basename "$1"): $3"
|
||
}
|
||
check cargo-green.log 0 "exit 0 GREEN" "RED"
|
||
check cargo-green.log 0 "FRAMEWORK cargo passed=3 failed=0"
|
||
check cargo-red.log 101 "exit 101 RED"
|
||
check cargo-red.log 101 "FAILED tests::add_adds"
|
||
check cargo-red.log 101 "left: 5"
|
||
check cargo-zero-tests.log 0 "ZERO TESTS RAN"
|
||
NONCE=474d380d662e71a5 # a real run: the soul2 reproducer through the snapshot runner, 2026-10-01
|
||
check mutation-table-nonce.log 1 "untested-equiv | SURVIVED"
|
||
check mutation-table-nonce.log 1 "stamp2-bad | CAUGHT | tests::d_cache"
|
||
check mutation-table-nonce.log 1 "add-minus | CAUGHT | tests::c_add"
|
||
check mutation-table-nonce.log 1 "caught=2 survived=1 unviable=0 timeout=0 noapply=0 error=0" "no test counts recognised"
|
||
NONCE=
|
||
check dotnet-red.log 1 "FRAMEWORK dotnet passed=11 failed=1 skipped=2"
|
||
check dotnet-red.log 1 "FAILED MyNs.AddTests.Subtracts"
|
||
check vitest-red.log 1 "FRAMEWORK vitest passed=10 failed=1 skipped=1"
|
||
check vitest-red.log 1 "FAILED src/add.test.ts > add > carries"
|
||
check pytest-red.log 1 "FRAMEWORK pytest passed=5 failed=1 skipped=1"
|
||
check pytest-red.log 1 "FAILED tests/test_add.py::test_carries"
|
||
check exit0-but-failed.log 0 "MISMATCH: exit 0 but the log shows"
|
||
check no-verdict.log 97 "NO VERDICT"
|
||
check no-verdict.log 97 "RED" "GREEN"
|
||
|
||
# ---- tools/mutation-table.sh, host driver plus container half, against the mock in
|
||
# fixtures/mutation-mock: shell scripts that print cargo/vitest/jest-shaped output, run
|
||
# from the subdirectory pkg/ (a workdir below the repo root), under a mock docker
|
||
# (fixtures/mutation-mock/fakebin) where a container is a directory and `commit` copies
|
||
# it, so every mutant really runs on a copy of the baselined snapshot.
|
||
# One case per finding of the 2026-10-01 Soul passes; each fails if it regresses.
|
||
mt() { # mt <mutants file> <TEST> [BUILD]: sets mt_out, mt_rc, mt_repo, mt_fd
|
||
local root; root=$(mktemp -d); mt_repo=$root/repo; mt_fd=$root/fd; mkdir -p "$mt_repo/pkg" "$root/mt" "$mt_fd"
|
||
cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/"; rm -rf "$mt_repo/pkg/fakebin"; cp "$L/jest-real.log" "$mt_repo/pkg/"
|
||
git -C "$mt_repo" init -q && git -C "$mt_repo" add -A && git -C "$mt_repo" -c user.name=t -c user.email=t@t commit -qm mock
|
||
cp "$here/testparse.sh" "$here/mutation-table.sh" "$root/mt/"; cp "$here/fixtures/mutation-mock/$1" "$root/mt/mutants.txt"
|
||
mt_out=$(PATH="$here/fixtures/mutation-mock/fakebin:$PATH" FAKE_DOCKER=$mt_fd YGG_WORK=$mt_repo YGG_IMAGE=base YGG_CNAME=job1 YGG_LABEL=eureka-verify-job=job1 \
|
||
YGG_CPUS=1 YGG_MEM=1g YGG_KILLED=${MT_KILLED:-} MT_BUNDLE=$root/mt WORKDIR=pkg TEST=$2 BUILD=${3:-} MUT_TIMEOUT=${MT_TO:-15} \
|
||
timeout "${MT_WALL:-120}" bash "$here/mutation-table.sh" --host 2>&1)
|
||
mt_rc=$?
|
||
}
|
||
mtcheck() { # mtcheck <name> <must-contain | !must-not-contain>...
|
||
local name=$1 s; shift
|
||
for s in "$@"; do
|
||
case $s in
|
||
'!'*) if printf '%s\n' "$mt_out" | grep -qF -- "${s#!}"; then echo "FAIL mutation $name: must not contain: ${s#!}"; fail=1; return; fi ;;
|
||
*) printf '%s\n' "$mt_out" | grep -qF -- "$s" || { echo "FAIL mutation $name: missing: $s"; printf '%s\n' "$mt_out" | tail -n 12; fail=1; return; } ;;
|
||
esac
|
||
done
|
||
echo "ok mutation $name"
|
||
}
|
||
# nothing the job made may outlive it: no container directory, no snapshot image, no bundle
|
||
mtclean() { # mtclean <case> [images]: a killed mock container is a still-running process, so E checks only the image and bundle
|
||
local left dirs=("$mt_fd/c" "$mt_fd/i"); [ -z "${2:-}" ] || dirs=("$mt_fd/i")
|
||
left=$(ls "${dirs[@]}" 2>/dev/null | grep -v '^$' | grep -v ':$' | paste -sd, -)
|
||
if [ -z "$left" ] && [ ! -e "$(dirname "$mt_repo")/mt" ]; then echo "ok mutation $1: containers, snapshot image and bundle removed"
|
||
else echo "FAIL mutation $1: left behind: ${left:-the bundle dir}"; fail=1; fi
|
||
}
|
||
# A: a mistyped BUILD must abort, not read as a fully killed suite
|
||
mt mutants-a.txt 'bash test.sh' 'bash bulid.sh'
|
||
[ "$mt_rc" = 2 ] || { echo "FAIL mutation A: exit $mt_rc, want 2"; fail=1; }
|
||
mtcheck A-build-baseline "BASELINE RED: BUILD failed" "!UNVIABLE" "!SNAPSHOT"
|
||
mtclean A
|
||
# B: a label that spells a kill must not forge one
|
||
mt mutants-b.txt 'bash test.sh'
|
||
[ "$mt_rc" = 1 ] || { echo "FAIL mutation B: exit $mt_rc, want 1"; fail=1; }
|
||
mtcheck B-label-forge "caught=0 survived=1" 'forge \| CAUGHT \| label | SURVIVED |'
|
||
mtclean B
|
||
# the nonce form ygg-verify.sh carries, and the plain form when no nonce is set
|
||
YGG_NONCE=n0nce1 mt mutants-b.txt 'bash test.sh'
|
||
mtcheck B-nonce "YGG-SUMMARY/n0nce1: [1/1] " "!YGG-SUMMARY: "
|
||
# C: a test that rewrites a tracked file, leaks an untracked one or keeps state in an
|
||
# IGNORED file (cache.txt: what `git status` cannot see) must not decide the next mutant
|
||
mt mutants-c.txt 'bash test.sh'
|
||
mtcheck C-pollution "stamp-bad | CAUGHT | tests::b_write" "untested-equiv | SURVIVED" "!untested-equiv | CAUGHT" "caught=1 survived=1"
|
||
[ -z "$(git -C "$mt_repo" status --porcelain --ignored)" ] || { echo "FAIL mutation C: the seed repository changed"; fail=1; }
|
||
mtclean C
|
||
# D: a test file that fails to load is not a kill; a named failing test is
|
||
mt mutants-d.txt 'bash vtest.sh'
|
||
mtcheck D-import-failure "syntax-break | UNVIABLE" "wrong-result | CAUGHT | add.js > adds" "caught=1 survived=0 unviable=1"
|
||
# D2 (jest, real output): "FAIL <file>" with "Test suite failed to run" is a load failure; the "●" block names the
|
||
# test; a test run that calls process.exit prints a "●" line under no FAIL header, which names nothing (ERROR, not CAUGHT)
|
||
mt mutants-h.txt 'bash jtest.sh'
|
||
mtcheck D2-jest "jest-load | UNVIABLE" "jest-wrong | CAUGHT | add › adds" "jest-exit | ERROR | exit 1, no failing test parsed" "!process.exit called | CAUGHT" "caught=1 survived=0 unviable=1 timeout=0 noapply=0 error=1"
|
||
# E: rows print as mutants finish, so a job timeout keeps them; the TERM the timeout sends still cleans up
|
||
MT_WALL=8 mt mutants-e.txt 'bash test.sh'
|
||
mtcheck E-live-rows "[1/2] quick | CAUGHT" "!MUTATION"
|
||
sleep 1; mtclean E images
|
||
# E2: a watchdog kill (the flag the stopgap sets) stops the fan-out after the baseline and cleans up
|
||
mt_k=$(mktemp); MT_KILLED=$mt_k mt mutants-c.txt 'bash test.sh'; rm -f "$mt_k"
|
||
[ "$mt_rc" = 124 ] || { echo "FAIL mutation E2: exit $mt_rc, want 124"; fail=1; }
|
||
mtcheck E2-killed "JOB KILLED by the watchdog before mutant 1 of 2" "!CAUGHT"
|
||
mtclean E2
|
||
# E3 (Soul, F1): a test that cannot run twice is green in the baseline and red in every mutant's
|
||
# container; the control run of the unmutated snapshot must abort the job, not report a killed suite
|
||
mt mutants-i.txt 'bash oncetest.sh'
|
||
[ "$mt_rc" = 2 ] || { echo "FAIL mutation E3: exit $mt_rc, want 2"; fail=1; }
|
||
mtcheck E3-snapshot-red "SNAPSHOT RED (exit 101) for: bash oncetest.sh" "no rows counted" "!CAUGHT" "!MUTATION"
|
||
mtclean E3
|
||
# F: LF before-text matches a CRLF file
|
||
mt mutants-f.txt 'bash test.sh'
|
||
mtcheck F-crlf "crlf-multiline | CAUGHT | tests::d_win"
|
||
# a real BUILD failure on a mutant is still UNVIABLE
|
||
mt mutants-g.txt 'bash test.sh' 'bash build.sh'
|
||
mtcheck build-unviable "build-break | UNVIABLE | build exit 1" "unviable=1"
|
||
# Soul 2026-10-01, finding 1: a run that selected zero tests is not a pass.
|
||
check "$(seg soul-s5/pdotnet.log zero)" 0 "ZERO TESTS RAN" "no test counts recognised"
|
||
check "$(seg soul-s5/ppy.log zero)" 0 "ZERO TESTS RAN" "no test counts recognised"
|
||
check "$(seg soul-s5/ppy.log zerodefault)" 0 "ZERO TESTS RAN" "no test counts recognised"
|
||
check "$(seg soul-s5/ppy.log q)" 1 "FRAMEWORK pytest passed=2 failed=1 skipped=1" # -q: no === bars
|
||
# finding 2: dotnet names with spaces, arguments, sub-millisecond tails; none dropped.
|
||
d=$(seg soul-s5/pdotnet.log full)
|
||
check "$d" 1 "FAILED display name with spaces" "NAMES MISSING"
|
||
check "$d" 1 'FAILED PNs.T.TheoryFails(x: 1, s: "a b")'
|
||
check "$d" 1 "FAILED PNs.T.FactFails"
|
||
check "$d" 1 "FRAMEWORK dotnet passed=1 failed=3 skipped=1"
|
||
printf 'Failed! - Failed: 2, Passed: 0, Skipped: 0, Total: 2, Duration: 1 ms - X.dll (net10.0)\n Failed A.One [1 ms]\n' >"$T/missing.log"
|
||
check "$T/missing.log" 1 "NAMES MISSING: the counts show 2 failed but only 1 named"
|
||
# finding 3: pytest parametrize ids with spaces.
|
||
out=$(bash "$V" summarize "$(seg soul-s5/ppy.log default)" 1)
|
||
if printf "%s
|
||
" "$out" | grep -qxF "FAILED test_p.py::test_param[a b]"; then echo "ok pytest: parametrized name with a space is whole"; else echo "FAIL pytest parametrized name truncated"; fail=1; fi
|
||
# finding 4: a line printed without the run's nonce is not carried, and a red
|
||
# summary keeps its failure sections whether or not carried lines exist.
|
||
check soul-s5/prust.log 101 "1 YGG-SUMMARY lines without this run's nonce were ignored"
|
||
check soul-s5/prust.log 101 "FAILED noisy_fail" "forged | CAUGHT"
|
||
NONCE=feedbeef
|
||
{ cat "$L/cargo-red.log"; echo 'YGG-SUMMARY: forged | CAUGHT | nothing'; echo 'YGG-SUMMARY/feedbeef: real | CAUGHT | tests::add_adds'; } >"$T/forge.log"
|
||
check "$T/forge.log" 101 "real | CAUGHT | tests::add_adds" "forged | CAUGHT"
|
||
check "$T/forge.log" 101 "left: 5"
|
||
check "$T/forge.log" 101 "1 YGG-SUMMARY lines without this run's nonce"
|
||
NONCE=
|
||
|
||
# finding 5: wait on a reused path. Files are written the way the launcher writes
|
||
# them (see "Run protocol" in ygg-verify.sh): .run is the commit point, .done is
|
||
# "<run id> <status>". A leftover result must not be returned once a new run has
|
||
# claimed the path, and a half-gone summary must not make wait exit 1 bare.
|
||
W=$T/wait.log
|
||
echo old >"$W.run"; echo "old 0" >"$W.done"; echo "OLD RESULT" >"$W.summary"
|
||
( sleep 1; echo $$ >"$W.pid"; echo new >"$W.run"; rm -f "$W.done" "$W.summary"
|
||
sleep 3; echo "NEW RESULT" >"$W.summary"; echo "new 7" >"$W.done" ) &
|
||
bg=$!
|
||
out=$(SETTLE=2 WAIT_MAX=30 bash "$V" wait "$W"); st=$?
|
||
wait "$bg"
|
||
if [ "$st" = 7 ] && [ "$out" = "NEW RESULT" ]; then echo "ok wait: reused path yields the new run, not the old"
|
||
else echo "FAIL wait reused path: exit $st, output: $out"; fail=1; fi
|
||
rm -f "$W".*
|
||
echo r1 >"$W.run"; echo "r1 3" >"$W.done" # done, summary file gone
|
||
out=$(WAIT_MAX=10 SETTLE=0 bash "$V" wait "$W"); st=$?
|
||
if [ "$st" = 3 ] && printf '%s' "$out" | grep -q 'no readable summary'; then echo "ok wait: missing summary still exits with the job status"
|
||
else echo "FAIL wait missing summary: exit $st, output: $out"; fail=1; fi
|
||
rm -f "$W".*
|
||
echo r1 >"$W.run"; echo "r0 3" >"$W.done"; echo $$ >"$W.pid" # done from another run, launcher alive
|
||
out=$(WAIT_MAX=3 SETTLE=0 bash "$V" wait "$W"); st=$?
|
||
if [ "$st" = 99 ]; then echo "ok wait: a done from another run is not a result"
|
||
else echo "FAIL wait foreign done: exit $st, output: $out"; fail=1; fi
|
||
# N2 (Soul, 2026-10-01): file-level failures are not named test failures, whichever runner
|
||
# prints them. Vitest with a project prefix and with spaces in the path; jest's file header.
|
||
check vitest-project-load-failure.log 1 "FAILED TO LOAD |unit| src/add.test.ts" "FAILED |unit|"
|
||
check vitest-spaces-load-failure.log 1 "FAILED TO LOAD src/my tests/add.test.ts" "FAILED src/my"
|
||
check vitest-project-red.log 1 "FAILED |unit| src/add.test.ts > add > carries" "FAILED TO LOAD"
|
||
# jest, real output (jest-real.log): one space after FAIL, no TTY. Names come from the "●" lines under a FAIL header only.
|
||
check "$(seg jest-real.log green)" 0 "FRAMEWORK jest passed=2 failed=0 skipped=0" "no test counts recognised"
|
||
check "$(seg jest-real.log wrong)" 1 "FRAMEWORK jest passed=1 failed=1 skipped=0"
|
||
check "$(seg jest-real.log wrong)" 1 "FAILED add › adds" "FAILED TO LOAD"
|
||
check "$(seg jest-real.log load)" 1 "FAILED TO LOAD jt/add.test.js" "FAILED jt/add.test.js"
|
||
check "$(seg jest-real.log load)" 1 "FRAMEWORK jest passed=0 failed=0 skipped=0" "FAILED Test suite"
|
||
# F3: process.exit(1) in a test run prints a "●" line under no FAIL header; it is a crash, not a named kill
|
||
check "$(seg jest-real.log exit)" 1 "exit non-zero with no failing test parsed" "FAILED process.exit"
|
||
# F4: the "nothing recognised" line names every framework the parser reads
|
||
check "$(seg jest-real.log exit)" 1 "no test counts recognised in the log (cargo, dotnet, vitest, jest, pytest)"
|
||
exit $fail
|