Eureka
d9e45c8740
Truncating writers (`>`, Unity -logFile) rewrite below any offset, so an offset turned a stale success into a silent hang. The option, the scan branch, its docs and its fixtures are gone; the header states the fresh-log-path rule. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
228 lines
15 KiB
Shell
228 lines
15 KiB
Shell
#!/usr/bin/env bash
|
||
# Local, cheap checks of the verdict-bearing text parsers: tools/testparse.sh and
|
||
# the summary and wait modes of tools/stopgap/ygg-verify.sh, over
|
||
# tools/fixtures/logs. cargo-*.log (the tiny-crate fixture) and mutation-table-nonce.log (the soul2
|
||
# reproducer) are real Yggdrasil runs, 2026-10-01; soul-s5/*.log are real
|
||
# dotnet, pytest, vitest and cargo runs of Soul's probe repo (several runs per
|
||
# log, split by "=== MARK <name>" lines); the rest are hand-written.
|
||
# Run: bash tools/test-tools.sh (no network, no builds)
|
||
set -uo pipefail
|
||
here=$(cd "$(dirname "$0")" && pwd)
|
||
L=$here/fixtures/logs; V=$here/stopgap/ygg-verify.sh
|
||
T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
|
||
fail=0
|
||
NONCE=''
|
||
# seg <log> <mark> -> a file holding just that run of a multi-run log
|
||
seg() { local o="$T/$(basename "$1").$2"; awk -v m="=== MARK $2" '/^=== MARK /{on=($0==m);next} on' "$L/$1" >"$o"; echo "$o"; }
|
||
# check <log path or name> <exit status> <must-contain> [<must-not-contain>]
|
||
check() {
|
||
local p=$1; [ -f "$p" ] || p=$L/$1
|
||
out=$(bash "$V" summarize "$p" "$2" ${NONCE:+"$NONCE"})
|
||
printf '%s\n' "$out" | grep -qF -- "$3" || { echo "FAIL $1: missing: $3"; fail=1; return; }
|
||
if [ -n "${4:-}" ] && printf '%s\n' "$out" | grep -qF -- "$4"; then echo "FAIL $1: must not contain: $4"; fail=1; return; fi
|
||
[ "$(printf '%s\n' "$out" | wc -l)" -le 81 ] || { echo "FAIL $1: summary over 81 lines"; fail=1; return; }
|
||
echo "ok $(basename "$1"): $3"
|
||
}
|
||
check cargo-green.log 0 "exit 0 GREEN" "RED"
|
||
check cargo-green.log 0 "FRAMEWORK cargo passed=3 failed=0"
|
||
check cargo-red.log 101 "exit 101 RED"
|
||
check cargo-red.log 101 "FAILED tests::add_adds"
|
||
check cargo-red.log 101 "left: 5"
|
||
check cargo-zero-tests.log 0 "ZERO TESTS RAN"
|
||
NONCE=474d380d662e71a5 # a real run: the soul2 reproducer through the snapshot runner, 2026-10-01
|
||
check mutation-table-nonce.log 1 "untested-equiv | SURVIVED"
|
||
check mutation-table-nonce.log 1 "stamp2-bad | CAUGHT | tests::d_cache"
|
||
check mutation-table-nonce.log 1 "add-minus | CAUGHT | tests::c_add"
|
||
check mutation-table-nonce.log 1 "caught=2 survived=1 unviable=0 timeout=0 noapply=0 error=0" "no test counts recognised"
|
||
NONCE=
|
||
check dotnet-red.log 1 "FRAMEWORK dotnet passed=11 failed=1 skipped=2"
|
||
check dotnet-red.log 1 "FAILED MyNs.AddTests.Subtracts"
|
||
check vitest-red.log 1 "FRAMEWORK vitest passed=10 failed=1 skipped=1"
|
||
check vitest-red.log 1 "FAILED src/add.test.ts > add > carries"
|
||
check pytest-red.log 1 "FRAMEWORK pytest passed=5 failed=1 skipped=1"
|
||
check pytest-red.log 1 "FAILED tests/test_add.py::test_carries"
|
||
check exit0-but-failed.log 0 "MISMATCH: exit 0 but the log shows"
|
||
check no-verdict.log 97 "NO VERDICT"
|
||
check no-verdict.log 97 "RED" "GREEN"
|
||
|
||
# ---- tools/mutation-table.sh, host driver plus container half, against the mock in
|
||
# fixtures/mutation-mock: shell scripts that print cargo/vitest/jest-shaped output, run
|
||
# from the subdirectory pkg/ (a workdir below the repo root), under a mock docker
|
||
# (fixtures/mutation-mock/fakebin) where a container is a directory and `commit` copies
|
||
# it, so every mutant really runs on a copy of the baselined snapshot.
|
||
# One case per finding of the 2026-10-01 Soul passes; each fails if it regresses.
|
||
mt() { # mt <mutants file> <TEST> [BUILD]: sets mt_out, mt_rc, mt_repo, mt_fd
|
||
local root; root=$(mktemp -d); mt_repo=$root/repo; mt_fd=$root/fd; mkdir -p "$mt_repo/pkg" "$root/mt" "$mt_fd"
|
||
cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/"; rm -rf "$mt_repo/pkg/fakebin"; cp "$L/jest-real.log" "$mt_repo/pkg/"
|
||
git -C "$mt_repo" init -q && git -C "$mt_repo" add -A && git -C "$mt_repo" -c user.name=t -c user.email=t@t commit -qm mock
|
||
cp "$here/testparse.sh" "$here/mutation-table.sh" "$root/mt/"; cp "$here/fixtures/mutation-mock/$1" "$root/mt/mutants.txt"
|
||
mt_out=$(PATH="$here/fixtures/mutation-mock/fakebin:$PATH" FAKE_DOCKER=$mt_fd YGG_WORK=$mt_repo YGG_IMAGE=base YGG_CNAME=job1 YGG_LABEL=eureka-verify-job=job1 \
|
||
YGG_CPUS=1 YGG_MEM=1g YGG_KILLED=${MT_KILLED:-} MT_BUNDLE=$root/mt WORKDIR=pkg TEST=$2 BUILD=${3:-} MUT_TIMEOUT=${MT_TO:-15} \
|
||
timeout "${MT_WALL:-120}" bash "$here/mutation-table.sh" --host 2>&1)
|
||
mt_rc=$?
|
||
}
|
||
mtcheck() { # mtcheck <name> <must-contain | !must-not-contain>...
|
||
local name=$1 s; shift
|
||
for s in "$@"; do
|
||
case $s in
|
||
'!'*) if printf '%s\n' "$mt_out" | grep -qF -- "${s#!}"; then echo "FAIL mutation $name: must not contain: ${s#!}"; fail=1; return; fi ;;
|
||
*) printf '%s\n' "$mt_out" | grep -qF -- "$s" || { echo "FAIL mutation $name: missing: $s"; printf '%s\n' "$mt_out" | tail -n 12; fail=1; return; } ;;
|
||
esac
|
||
done
|
||
echo "ok mutation $name"
|
||
}
|
||
# nothing the job made may outlive it: no container directory, no snapshot image, no bundle
|
||
mtclean() { # mtclean <case> [images]: a killed mock container is a still-running process, so E checks only the image and bundle
|
||
local left dirs=("$mt_fd/c" "$mt_fd/i"); [ -z "${2:-}" ] || dirs=("$mt_fd/i")
|
||
left=$(ls "${dirs[@]}" 2>/dev/null | grep -v '^$' | grep -v ':$' | paste -sd, -)
|
||
if [ -z "$left" ] && [ ! -e "$(dirname "$mt_repo")/mt" ]; then echo "ok mutation $1: containers, snapshot image and bundle removed"
|
||
else echo "FAIL mutation $1: left behind: ${left:-the bundle dir}"; fail=1; fi
|
||
}
|
||
# A: a mistyped BUILD must abort, not read as a fully killed suite
|
||
mt mutants-a.txt 'bash test.sh' 'bash bulid.sh'
|
||
[ "$mt_rc" = 2 ] || { echo "FAIL mutation A: exit $mt_rc, want 2"; fail=1; }
|
||
mtcheck A-build-baseline "BASELINE RED: BUILD failed" "!UNVIABLE" "!SNAPSHOT"
|
||
mtclean A
|
||
# B: a label that spells a kill must not forge one
|
||
mt mutants-b.txt 'bash test.sh'
|
||
[ "$mt_rc" = 1 ] || { echo "FAIL mutation B: exit $mt_rc, want 1"; fail=1; }
|
||
mtcheck B-label-forge "caught=0 survived=1" 'forge \| CAUGHT \| label | SURVIVED |'
|
||
mtclean B
|
||
# the nonce form ygg-verify.sh carries, and the plain form when no nonce is set
|
||
YGG_NONCE=n0nce1 mt mutants-b.txt 'bash test.sh'
|
||
mtcheck B-nonce "YGG-SUMMARY/n0nce1: [1/1] " "!YGG-SUMMARY: "
|
||
# C: a test that rewrites a tracked file, leaks an untracked one or keeps state in an
|
||
# IGNORED file (cache.txt: what `git status` cannot see) must not decide the next mutant
|
||
mt mutants-c.txt 'bash test.sh'
|
||
mtcheck C-pollution "stamp-bad | CAUGHT | tests::b_write" "untested-equiv | SURVIVED" "!untested-equiv | CAUGHT" "caught=1 survived=1"
|
||
[ -z "$(git -C "$mt_repo" status --porcelain --ignored)" ] || { echo "FAIL mutation C: the seed repository changed"; fail=1; }
|
||
mtclean C
|
||
# D: a test file that fails to load is not a kill; a named failing test is
|
||
mt mutants-d.txt 'bash vtest.sh'
|
||
mtcheck D-import-failure "syntax-break | UNVIABLE" "wrong-result | CAUGHT | add.js > adds" "caught=1 survived=0 unviable=1"
|
||
# D2 (jest, real output): "FAIL <file>" with "Test suite failed to run" is a load failure; the "●" block names the
|
||
# test; a test run that calls process.exit prints a "●" line under no FAIL header, which names nothing (ERROR, not CAUGHT)
|
||
mt mutants-h.txt 'bash jtest.sh'
|
||
mtcheck D2-jest "jest-load | UNVIABLE" "jest-wrong | CAUGHT | add › adds" "jest-exit | ERROR | exit 1, no failing test parsed" "!process.exit called | CAUGHT" "caught=1 survived=0 unviable=1 timeout=0 noapply=0 error=1"
|
||
# E: rows print as mutants finish, so a job timeout keeps them; the TERM the timeout sends still cleans up
|
||
MT_WALL=8 mt mutants-e.txt 'bash test.sh'
|
||
mtcheck E-live-rows "[1/2] quick | CAUGHT" "!MUTATION"
|
||
sleep 1; mtclean E images
|
||
# E2: a watchdog kill (the flag the stopgap sets) stops the fan-out after the baseline and cleans up
|
||
mt_k=$(mktemp); MT_KILLED=$mt_k mt mutants-c.txt 'bash test.sh'; rm -f "$mt_k"
|
||
[ "$mt_rc" = 124 ] || { echo "FAIL mutation E2: exit $mt_rc, want 124"; fail=1; }
|
||
mtcheck E2-killed "JOB KILLED by the watchdog before mutant 1 of 2" "!CAUGHT"
|
||
mtclean E2
|
||
# E3 (Soul, F1): a test that cannot run twice is green in the baseline and red in every mutant's
|
||
# container; the control run of the unmutated snapshot must abort the job, not report a killed suite
|
||
mt mutants-i.txt 'bash oncetest.sh'
|
||
[ "$mt_rc" = 2 ] || { echo "FAIL mutation E3: exit $mt_rc, want 2"; fail=1; }
|
||
mtcheck E3-snapshot-red "SNAPSHOT RED (exit 101) for: bash oncetest.sh" "no rows counted" "!CAUGHT" "!MUTATION"
|
||
mtclean E3
|
||
# F: LF before-text matches a CRLF file
|
||
mt mutants-f.txt 'bash test.sh'
|
||
mtcheck F-crlf "crlf-multiline | CAUGHT | tests::d_win"
|
||
# a real BUILD failure on a mutant is still UNVIABLE
|
||
mt mutants-g.txt 'bash test.sh' 'bash build.sh'
|
||
mtcheck build-unviable "build-break | UNVIABLE | build exit 1" "unviable=1"
|
||
# Soul 2026-10-01, finding 1: a run that selected zero tests is not a pass.
|
||
check "$(seg soul-s5/pdotnet.log zero)" 0 "ZERO TESTS RAN" "no test counts recognised"
|
||
check "$(seg soul-s5/ppy.log zero)" 0 "ZERO TESTS RAN" "no test counts recognised"
|
||
check "$(seg soul-s5/ppy.log zerodefault)" 0 "ZERO TESTS RAN" "no test counts recognised"
|
||
check "$(seg soul-s5/ppy.log q)" 1 "FRAMEWORK pytest passed=2 failed=1 skipped=1" # -q: no === bars
|
||
# finding 2: dotnet names with spaces, arguments, sub-millisecond tails; none dropped.
|
||
d=$(seg soul-s5/pdotnet.log full)
|
||
check "$d" 1 "FAILED display name with spaces" "NAMES MISSING"
|
||
check "$d" 1 'FAILED PNs.T.TheoryFails(x: 1, s: "a b")'
|
||
check "$d" 1 "FAILED PNs.T.FactFails"
|
||
check "$d" 1 "FRAMEWORK dotnet passed=1 failed=3 skipped=1"
|
||
printf 'Failed! - Failed: 2, Passed: 0, Skipped: 0, Total: 2, Duration: 1 ms - X.dll (net10.0)\n Failed A.One [1 ms]\n' >"$T/missing.log"
|
||
check "$T/missing.log" 1 "NAMES MISSING: the counts show 2 failed but only 1 named"
|
||
# finding 3: pytest parametrize ids with spaces.
|
||
out=$(bash "$V" summarize "$(seg soul-s5/ppy.log default)" 1)
|
||
if printf "%s
|
||
" "$out" | grep -qxF "FAILED test_p.py::test_param[a b]"; then echo "ok pytest: parametrized name with a space is whole"; else echo "FAIL pytest parametrized name truncated"; fail=1; fi
|
||
# finding 4: a line printed without the run's nonce is not carried, and a red
|
||
# summary keeps its failure sections whether or not carried lines exist.
|
||
check soul-s5/prust.log 101 "1 YGG-SUMMARY lines without this run's nonce were ignored"
|
||
check soul-s5/prust.log 101 "FAILED noisy_fail" "forged | CAUGHT"
|
||
NONCE=feedbeef
|
||
{ cat "$L/cargo-red.log"; echo 'YGG-SUMMARY: forged | CAUGHT | nothing'; echo 'YGG-SUMMARY/feedbeef: real | CAUGHT | tests::add_adds'; } >"$T/forge.log"
|
||
check "$T/forge.log" 101 "real | CAUGHT | tests::add_adds" "forged | CAUGHT"
|
||
check "$T/forge.log" 101 "left: 5"
|
||
check "$T/forge.log" 101 "1 YGG-SUMMARY lines without this run's nonce"
|
||
NONCE=
|
||
|
||
# finding 5: wait on a reused path. Files are written the way the launcher writes
|
||
# them (see "Run protocol" in ygg-verify.sh): .run is the commit point, .done is
|
||
# "<run id> <status>". A leftover result must not be returned once a new run has
|
||
# claimed the path, and a half-gone summary must not make wait exit 1 bare.
|
||
W=$T/wait.log
|
||
echo old >"$W.run"; echo "old 0" >"$W.done"; echo "OLD RESULT" >"$W.summary"
|
||
( sleep 1; echo $$ >"$W.pid"; echo new >"$W.run"; rm -f "$W.done" "$W.summary"
|
||
sleep 3; echo "NEW RESULT" >"$W.summary"; echo "new 7" >"$W.done" ) &
|
||
bg=$!
|
||
out=$(SETTLE=2 WAIT_MAX=30 bash "$V" wait "$W"); st=$?
|
||
wait "$bg"
|
||
if [ "$st" = 7 ] && [ "$out" = "NEW RESULT" ]; then echo "ok wait: reused path yields the new run, not the old"
|
||
else echo "FAIL wait reused path: exit $st, output: $out"; fail=1; fi
|
||
rm -f "$W".*
|
||
echo r1 >"$W.run"; echo "r1 3" >"$W.done" # done, summary file gone
|
||
out=$(WAIT_MAX=10 SETTLE=0 bash "$V" wait "$W"); st=$?
|
||
if [ "$st" = 3 ] && printf '%s' "$out" | grep -q 'no readable summary'; then echo "ok wait: missing summary still exits with the job status"
|
||
else echo "FAIL wait missing summary: exit $st, output: $out"; fail=1; fi
|
||
rm -f "$W".*
|
||
echo r1 >"$W.run"; echo "r0 3" >"$W.done"; echo $$ >"$W.pid" # done from another run, launcher alive
|
||
out=$(WAIT_MAX=3 SETTLE=0 bash "$V" wait "$W"); st=$?
|
||
if [ "$st" = 99 ]; then echo "ok wait: a done from another run is not a result"
|
||
else echo "FAIL wait foreign done: exit $st, output: $out"; fail=1; fi
|
||
# N2 (Soul, 2026-10-01): file-level failures are not named test failures, whichever runner
|
||
# prints them. Vitest with a project prefix and with spaces in the path; jest's file header.
|
||
check vitest-project-load-failure.log 1 "FAILED TO LOAD |unit| src/add.test.ts" "FAILED |unit|"
|
||
check vitest-spaces-load-failure.log 1 "FAILED TO LOAD src/my tests/add.test.ts" "FAILED src/my"
|
||
check vitest-project-red.log 1 "FAILED |unit| src/add.test.ts > add > carries" "FAILED TO LOAD"
|
||
# jest, real output (jest-real.log): one space after FAIL, no TTY. Names come from the "●" lines under a FAIL header only.
|
||
check "$(seg jest-real.log green)" 0 "FRAMEWORK jest passed=2 failed=0 skipped=0" "no test counts recognised"
|
||
check "$(seg jest-real.log wrong)" 1 "FRAMEWORK jest passed=1 failed=1 skipped=0"
|
||
check "$(seg jest-real.log wrong)" 1 "FAILED add › adds" "FAILED TO LOAD"
|
||
check "$(seg jest-real.log load)" 1 "FAILED TO LOAD jt/add.test.js" "FAILED jt/add.test.js"
|
||
check "$(seg jest-real.log load)" 1 "FRAMEWORK jest passed=0 failed=0 skipped=0" "FAILED Test suite"
|
||
# F3: process.exit(1) in a test run prints a "●" line under no FAIL header; it is a crash, not a named kill
|
||
check "$(seg jest-real.log exit)" 1 "exit non-zero with no failing test parsed" "FAILED process.exit"
|
||
# F4: the "nothing recognised" line names every framework the parser reads
|
||
check "$(seg jest-real.log exit)" 1 "no test counts recognised in the log (cargo, dotnet, vitest, jest, pytest)"
|
||
|
||
# wait-on.sh: pid, log and file forms, and the vanished-process exit
|
||
W=$here/wait-on.sh
|
||
wo() { local rc; WAIT_MAX=${WM:-4} bash "$W" "$@" >"$T/wo.out" 2>&1; rc=$?; echo "$rc"; }
|
||
wchk() { if [ "$3" = "$2" ]; then echo "ok wait-on $1: exit $3"; else echo "FAIL wait-on $1: exit $3, want $2"; fail=1; fi; }
|
||
sleep 3 & sp=$!; printf 'one\ntwo\n' >"$T/w.log"
|
||
wchk "pid finishes" 0 "$(WM=10 wo pid $sp "$T/w.log")"
|
||
sleep 20 & sp=$!
|
||
r=$(WM=2 wo pid $sp); wchk "pid still running" 99 "$r"
|
||
grep -q '^RUNNING ' "$T/wo.out" || { echo "FAIL wait-on: no RUNNING line"; fail=1; }
|
||
kill $sp 2>/dev/null; wait $sp 2>/dev/null
|
||
r=$(WM=10 wo pid $sp "$T/w.log"); wchk "killed pid is done" 0 "$r"
|
||
grep -q two "$T/wo.out" || { echo "FAIL wait-on pid: no log tail"; fail=1; }
|
||
: >"$T/l.log"; ( sleep 2; echo "step 1" >>"$T/l.log"; echo "all done ok" >>"$T/l.log" ) &
|
||
r=$(WM=10 wo log "$T/l.log" 'done ok'); wchk "log gains its done line" 0 "$r"
|
||
grep -q 'all done ok' "$T/wo.out" || { echo "FAIL wait-on log: no tail"; fail=1; }
|
||
r=$(WM=2 wo log "$T/l.log" 'never-appears'); wchk "log never matches" 99 "$r"
|
||
printf 'partial\n' >"$T/v.log"; sleep 1 & vp=$!; wait $vp
|
||
r=$(WAIT_PID=$vp WM=10 wo log "$T/v.log" 'FINISHED'); wchk "vanished pid, no completion" 98 "$r"
|
||
printf 'FINISHED\n' >>"$T/v.log"
|
||
r=$(WAIT_PID=$vp WM=10 wo log "$T/v.log" 'FINISHED'); wchk "vanished pid, completion logged" 0 "$r"
|
||
r=$(WM=2 wo log "$T/none.log" 'x'); wchk "log missing, no pid, short cap" 99 "$r"
|
||
( sleep 2; echo ready >"$T/f.txt" ) &
|
||
r=$(WM=10 wo file "$T/f.txt"); wchk "file appears" 0 "$r"
|
||
: >"$T/e.txt"; r=$(WM=2 wo file "$T/e.txt"); wchk "file empty is not done" 99 "$r"
|
||
# F2: the pid form reports GONE, never anything that reads as success
|
||
sleep 1 & gp=$!; wait $gp
|
||
r=$(WM=4 wo pid $gp); wchk "pid gone" 0 "$r"
|
||
grep -q '^GONE (no completion evidence)' "$T/wo.out" || { echo "FAIL wait-on pid: no GONE line"; fail=1; }
|
||
# F4: an invalid ERE is a usage error, not a forever RUNNING
|
||
r=$(WM=2 wo log "$T/l.log" '(unclosed'); wchk "invalid regex" 3 "$r"
|
||
grep -q 'not a valid extended regex' "$T/wo.out" || { echo "FAIL wait-on: no regex message"; fail=1; }
|
||
r=$(wo bogus); wchk "bad usage" 2 "$r"
|
||
wait
|
||
exit $fail
|