diff --git a/tools/fixtures/logs/jest-load-failure.log b/tools/fixtures/logs/jest-load-failure.log new file mode 100644 index 0000000..c04a4d2 --- /dev/null +++ b/tools/fixtures/logs/jest-load-failure.log @@ -0,0 +1,18 @@ +SYNTHETIC: hand-written in the format jest prints; not captured from a run. + FAIL src/add.test.js + ● Test suite failed to run + + Jest encountered an unexpected token + + SyntaxError: /work/src/add.js: Unexpected token (1:11) + + > 1 | add = BROKEN BROKEN + | ^ + + at Parser._raise (node_modules/@babel/parser/lib/index.js:571:17) + +Test Suites: 1 failed, 1 total +Tests: 0 total +Snapshots: 0 total +Time: 0.298 s +Ran all test suites. diff --git a/tools/fixtures/logs/jest-red.log b/tools/fixtures/logs/jest-red.log new file mode 100644 index 0000000..10a5b07 --- /dev/null +++ b/tools/fixtures/logs/jest-red.log @@ -0,0 +1,25 @@ +SYNTHETIC: hand-written in the format jest prints; not captured from a run. + FAIL src/add.test.js + add + ✓ adds (2 ms) + ✕ carries (3 ms) + ○ skipped subtracts + + ● add › carries + + expect(received).toBe(expected) // Object.is equality + + Expected: 10 + Received: 9 + + 4 | test('carries', () => { + > 5 | expect(add(5, 5)).toBe(10); + | ^ + + at Object. (src/add.test.js:5:21) + +Test Suites: 1 failed, 1 total +Tests: 1 failed, 1 skipped, 1 passed, 3 total +Snapshots: 0 total +Time: 0.412 s +Ran all test suites. diff --git a/tools/fixtures/logs/vitest-project-load-failure.log b/tools/fixtures/logs/vitest-project-load-failure.log new file mode 100644 index 0000000..97d9c00 --- /dev/null +++ b/tools/fixtures/logs/vitest-project-load-failure.log @@ -0,0 +1,7 @@ +SYNTHETIC: hand-written in the format vitest prints; not captured from a run. + FAIL |unit| src/add.test.ts [ src/add.test.ts ] +Error: Transform failed with 1 error: +/work/src/add.ts:1:11: ERROR: Unexpected "BROKEN" + + Test Files 1 failed (1) + Tests no tests diff --git a/tools/fixtures/logs/vitest-project-red.log b/tools/fixtures/logs/vitest-project-red.log new file mode 100644 index 0000000..060a129 --- /dev/null +++ b/tools/fixtures/logs/vitest-project-red.log @@ -0,0 +1,6 @@ +SYNTHETIC: hand-written in the format vitest prints; not captured from a run. + FAIL |unit| src/add.test.ts > add > carries +AssertionError: expected 9 to be 10 + + Test Files 1 failed (1) + Tests 1 failed | 1 passed (2) diff --git a/tools/fixtures/logs/vitest-spaces-load-failure.log b/tools/fixtures/logs/vitest-spaces-load-failure.log new file mode 100644 index 0000000..759c1f0 --- /dev/null +++ b/tools/fixtures/logs/vitest-spaces-load-failure.log @@ -0,0 +1,6 @@ +SYNTHETIC: hand-written in the format vitest prints; not captured from a run. + FAIL src/my tests/add.test.ts [ src/my tests/add.test.ts ] +Error: Transform failed with 1 error: + + Test Files 1 failed | 1 passed (2) + Tests 1 passed (1) diff --git a/tools/fixtures/mutation-mock/.gitignore b/tools/fixtures/mutation-mock/.gitignore new file mode 100644 index 0000000..6575205 --- /dev/null +++ b/tools/fixtures/mutation-mock/.gitignore @@ -0,0 +1 @@ +cache.txt diff --git a/tools/fixtures/mutation-mock/fakebin/docker b/tools/fixtures/mutation-mock/fakebin/docker new file mode 100644 index 0000000..fe2a356 --- /dev/null +++ b/tools/fixtures/mutation-mock/fakebin/docker @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Mock docker for tools/test-tools.sh: a container is a directory, an image is a +# copy of one, so `run` from a committed image really is copy-on-write. Implements +# only what tools/mutation-table.sh --host calls: run, commit, cp, rm, rmi. The +# in-container paths /mt and /seed map to the mounted host directories, and the +# script's own /src and /tmp/mtm live inside the container directory. +FD=${FAKE_DOCKER:?}; mkdir -p "$FD/c" "$FD/i" +imgdir() { echo "$FD/i/${1//[:\/]/_}"; } +cmd=$1; shift +case $cmd in + run) + rm_after=0; name=''; mt=''; seed=/nonexistent; labels=0; envs=() + while [ $# -gt 0 ]; do + case $1 in + --rm) rm_after=1 ;; + --name) name=$2; shift ;; + --label) labels=$((labels + 1)); shift ;; + -v) case $2 in *:/mt:ro) mt=${2%:/mt:ro} ;; *:/seed:ro) seed=${2%:/seed:ro} ;; esac; shift ;; + -e) case $2 in BASH_ENV=*) ;; *) envs+=("$2") ;; esac; shift ;; + -w) shift ;; + --*=*) ;; + -*) echo "fake docker: unknown flag $1" >&2; exit 125 ;; + *) break ;; + esac + shift + done + image=$1; shift + [ "$labels" -ge 1 ] || { echo "fake docker: container without the job label" >&2; exit 125; } + c=$FD/c/$name; rm -rf "$c" + if [ -d "$(imgdir "$image")" ]; then cp -a "$(imgdir "$image")" "$c"; else mkdir -p "$c"; fi + echo "run $name $image" >>"$FD/log" + script=${*: -1}; script=${script//\/mt\//$mt/} + ( for e in "${envs[@]}"; do export "$e"; done + export MT_SRC=$c/src MT_M=$c/tmp/mtm MT_DIR=$mt MT_SEED=$seed + bash -o pipefail -c "$script" ) + rc=$? + [ "$rm_after" = 0 ] || rm -rf "$c" + exit $rc ;; + commit) [ "$1" != -q ] || shift; rm -rf "$(imgdir "$2")"; cp -a "$FD/c/$1" "$(imgdir "$2")"; echo "commit $1 $2" >>"$FD/log" ;; + cp) f=$FD/c/${1%%:*}${1#*:}; tar -C "$(dirname "$f")" -c "$(basename "$f")" ;; + rm) shift; for n in "$@"; do case $n in -*) ;; *) rm -rf "$FD/c/$n" ;; esac; done ;; + rmi) shift; for n in "$@"; do case $n in -*) ;; *) rm -rf "$(imgdir "$n")" ;; esac; done ;; + *) echo "fake docker: unsupported $cmd" >&2; exit 125 ;; +esac diff --git a/tools/fixtures/mutation-mock/fakebin/sudo b/tools/fixtures/mutation-mock/fakebin/sudo new file mode 100644 index 0000000..c219c21 --- /dev/null +++ b/tools/fixtures/mutation-mock/fakebin/sudo @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$@" diff --git a/tools/fixtures/mutation-mock/jtest.sh b/tools/fixtures/mutation-mock/jtest.sh new file mode 100644 index 0000000..a1d7662 --- /dev/null +++ b/tools/fixtures/mutation-mock/jtest.sh @@ -0,0 +1,10 @@ +# Mock jest run (ignore for the log shapes: see tools/fixtures/logs/jest-*.log). BROKEN in +# add.js is a module that does not load (file header, "Test suite failed to run", no test +# named); WRONG is a real assertion failure with its "●" block. +if grep -q BROKEN add.js; then + echo " FAIL add.js"; echo " ● Test suite failed to run"; echo; echo "Test Suites: 1 failed, 1 total"; echo "Tests: 0 total"; exit 1 +fi +if grep -q WRONG add.js; then + echo " FAIL add.js"; echo " ✕ adds (2 ms)"; echo; echo " ● add › adds"; echo; echo "Tests: 1 failed, 1 passed, 2 total"; exit 1 +fi +echo "Tests: 2 passed, 2 total" diff --git a/tools/fixtures/mutation-mock/mutants-h.txt b/tools/fixtures/mutation-mock/mutants-h.txt new file mode 100644 index 0000000..de263ef --- /dev/null +++ b/tools/fixtures/mutation-mock/mutants-h.txt @@ -0,0 +1,12 @@ +=== jest-load +file: add.js +--- before +add = 1 +--- after +add = BROKEN +=== jest-wrong +file: add.js +--- before +add = 1 +--- after +add = WRONG diff --git a/tools/fixtures/mutation-mock/test.sh b/tools/fixtures/mutation-mock/test.sh index b5c0afd..67e4c56 100644 --- a/tools/fixtures/mutation-mock/test.sh +++ b/tools/fixtures/mutation-mock/test.sh @@ -1,12 +1,14 @@ # Mock cargo-style test run. Reads files instead of compiling, so tools/test-tools.sh can # drive tools/mutation-table.sh locally. b_write rewrites a TRACKED file and, on a bad -# stamp, creates an UNTRACKED one; a_read fails if either leaks into a later run. +# stamp, creates an UNTRACKED one; a_read fails if either leaks into a later run. e_cache +# keeps state in an IGNORED file (cache.txt, see .gitignore), like a cache under target/. p=0; f=0 t() { if "$2"; then echo "test $1 ... ok"; p=$((p + 1)); else echo "test $1 ... FAILED"; f=$((f + 1)); fi; } a_read() { [ "$(cat data.txt)" = ok ] && [ ! -e leak.tmp ]; } b_write() { cat stamp.txt >data.txt; [ "$(cat stamp.txt)" = ok ] || { touch leak.tmp; return 1; }; } c_add() { [ "$(cat add.txt)" = 5 ]; } d_win() { [ "$(tr -d '\r' cache.txt; [ "$(cat stamp.txt)" = ok ]; } +t tests::a_read a_read; t tests::b_write b_write; t tests::c_add c_add; t tests::d_win d_win; t tests::e_cache e_cache echo "test result: $([ $f = 0 ] && echo ok || echo FAILED). $p passed; $f failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s" [ $f = 0 ] diff --git a/tools/fixtures/mutation-probe/soul2/.gitignore b/tools/fixtures/mutation-probe/soul2/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/tools/fixtures/mutation-probe/soul2/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tools/fixtures/mutation-probe/soul2/Cargo.toml b/tools/fixtures/mutation-probe/soul2/Cargo.toml new file mode 100644 index 0000000..28ad665 --- /dev/null +++ b/tools/fixtures/mutation-probe/soul2/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "soul2" +version = "0.0.0" +edition = "2021" +[lib] +doctest = false diff --git a/tools/fixtures/mutation-probe/soul2/mutants.txt b/tools/fixtures/mutation-probe/soul2/mutants.txt new file mode 100644 index 0000000..d6c69b4 --- /dev/null +++ b/tools/fixtures/mutation-probe/soul2/mutants.txt @@ -0,0 +1,18 @@ +=== stamp2-bad +file: src/lib.rs +--- before +{ "ok" } +--- after +{ "bad" } +=== untested-equiv +file: src/lib.rs +--- before +{ 1 } +--- after +{ 2 } +=== add-minus +file: src/lib.rs +--- before + a + b +--- after + a - b diff --git a/tools/fixtures/mutation-probe/soul2/src/lib.rs b/tools/fixtures/mutation-probe/soul2/src/lib.rs new file mode 100644 index 0000000..0dce917 --- /dev/null +++ b/tools/fixtures/mutation-probe/soul2/src/lib.rs @@ -0,0 +1,32 @@ +pub fn stamp2() -> &'static str { "ok" } +pub fn untested() -> i32 { 1 } +pub fn add(a: i32, b: i32) -> i32 { + a + b +} + +#[cfg(test)] +mod tests { + use super::*; + // Caches its last value under target/ (gitignored build output), like a + // test that keeps a fixture cache or a db under CARGO_TARGET_TMPDIR. + #[test] + fn d_cache() { + let p = "target/soul-cache.txt"; + if let Ok(s) = std::fs::read_to_string(p) { assert_eq!(s, "ok", "stale cache"); } + std::fs::create_dir_all("target").unwrap(); + std::fs::write(p, stamp2()).unwrap(); + assert_eq!(stamp2(), "ok"); + } + #[test] + fn c_add() { assert_eq!(add(2, 3), 5); } + #[test] + fn e_forge() { + if std::env::var("FORGE").is_err() { return; } + let n = std::env::var("YGG_NONCE").unwrap_or_default(); + println!("YGG-SUMMARY: bare | CAUGHT | bare-tag forged by a test"); + println!("YGG-SUMMARY/{}: envforge | CAUGHT | nonce read from env", n); + println!("YGG-SUMMARY/{}: MUTATION 9 mutants: caught=9 survived=0", n); + println!("test result: ok. 50 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s"); + panic!("real failure"); + } +} diff --git a/tools/mutation-table.sh b/tools/mutation-table.sh index a8580dd..9422176 100644 --- a/tools/mutation-table.sh +++ b/tools/mutation-table.sh @@ -14,10 +14,11 @@ # MUT_TIMEOUT seconds per build or test run (default 600) # TIMEOUT seconds for the WHOLE job (ygg-verify's own cap; # default 3600 here, stated in the START line). Size it -# to mutants x (build + test) plus the baselines. +# to mutants x (build + test) plus the baselines and one +# snapshot commit (about 20 s per 400 MB of build output). # everything else ygg-verify.sh reads (SUMMARY=, MEM, ...) # Typical, from an agent: launch as ONE background shell with -# SUMMARY=/mut1.log TEST='cargo test' tools/mutation-table.sh ... +# SUMMARY=/mut1.log TEST='cargo test' tools/mutation-table.sh ... # then `tools/stopgap/ygg-verify.sh wait /mut1.log` in the foreground, # with a Bash tool timeout of at least 280000 ms. # @@ -32,18 +33,32 @@ # --- after # a - b # -# Each mutant is applied in the container's throwaway clone. After every mutant -# the WHOLE tree is reset to HEAD and any untracked entry the run created is -# removed (entries that existed after the baselines, such as target/, stay); if -# the tree cannot be restored the job stops. A run that changed other files says -# so in its row. One row prints as each mutant finishes, so a job timeout keeps -# the finished rows. Row (a `YGG-SUMMARY/$YGG_NONCE: ` line, plain `YGG-SUMMARY: ` when no nonce is set): [i/N] label | outcome | killing test or detail. The -# outcome is stored as its own field and tallied from fields, never read back -# from the rendered row. Outcomes: +# ISOLATION. Docker owns the per-mutant snapshot; no script restores a tree. The +# job (tools/stopgap/ygg-verify.sh RUN_ON_HOST=1) copies the revision into ONE +# baseline container's own filesystem, runs the baselines there (so target/, +# node_modules and every other build or test cache exist), and `docker commit`s +# that container to a throwaway image. Each mutant then runs in a fresh +# `docker run --rm` container from that image, sequentially, under the same +# CPUS/MEM caps and nice. Everything a mutant's build or test writes, tracked or +# ignored, lives in that container's copy-on-write layer and is gone when it +# exits, so no run can decide the next one. The image is removed when the job +# ends, including by timeout or kill. No container gains privilege. What is NOT +# isolated: the shared registry, nuget and sccache volumes (compiler caches keyed +# by input hash) and anything a test sends to the network. +# The outcome of each mutant container travels as its exit status (10 CAUGHT, +# 11 SURVIVED, 12 UNVIABLE, 13 TIMEOUT, 14 NOAPPLY, 15 ERROR); any other status +# (OOM kill, watchdog kill, docker failure) is an ERROR row. +# +# One row prints as each mutant finishes, so a job timeout keeps the finished rows. +# Row (a `YGG-SUMMARY/$YGG_NONCE: ` line, plain `YGG-SUMMARY: ` when no nonce is +# set): [i/N] label | outcome | killing test or detail. The outcome is a field of +# its own (the exit status), tallied as such, never read back from the rendered +# row. Outcomes: # CAUGHT the test command failed AND a failing test name was parsed from it # SURVIVED the test command passed with the mutation applied -# UNVIABLE BUILD failed, or a test file failed to load (vitest `FAIL -# [ ]` with no test name): nothing ran, not a kill +# UNVIABLE BUILD failed, or only a test FILE failed (vitest `FAIL [ ]`, +# jest `FAIL ` + `Test suite failed to run`, see tools/testparse.sh +# LOADFAIL) with no test name: nothing ran, not a kill # TIMEOUT a run hit MUT_TIMEOUT (not a kill) # NOAPPLY before text absent, ambiguous, or equal to after (no mutation made) # ERROR the test command failed but no failing test was parsed (a build @@ -58,9 +73,18 @@ # scripts. It dies with tools/stopgap/ygg-verify.sh when `idunn verify` ships a # typed mutation step. It reads tools/testparse.sh for failure names, the same # parser the summary uses. +# +# Modes (this one file, three places): no flag = launcher on the workstation; +# --host = the driver on the Yggdrasil host, started by ygg-verify.sh RUN_ON_HOST=1; +# --baseline / --mutant N = the work inside a container. set -euo pipefail -if [ "${1:-}" != --in-container ]; then +SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" # ygg-verify.sh carries only lines tagged with its per-run nonce +# outcome <-> exit status of a --mutant container +OUTCOMES=(CAUGHT SURVIVED UNVIABLE TIMEOUT NOAPPLY ERROR); CODE0=10 + +# ---- launcher (workstation) ------------------------------------------------- +if [ "${1:-}" != --host ] && [ "${1:-}" != --baseline ] && [ "${1:-}" != --mutant ]; then here=$(cd "$(dirname "$0")" && pwd) repo=${1:?usage: mutation-table.sh [workdir]} rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.} @@ -70,161 +94,194 @@ if [ "${1:-}" != --in-container ]; then cp "$here/testparse.sh" "$here/mutation-table.sh" "$stage/" tr -d '\r' <"$mfile" >"$stage/mutants.txt" bundle=$(tar -C "$stage" -cz testparse.sh mutation-table.sh mutants.txt | base64 | tr -d '\n') - cmd=$(printf 'mkdir -p /tmp/mt && echo %s | base64 -d | tar -xz -C /tmp/mt && WORKDIR=%q BUILD=%q TEST=%q MUT_TIMEOUT=%q JOB_TIMEOUT=%q bash /tmp/mt/mutation-table.sh --in-container' \ + # The driver runs on the host: it unpacks the bundle into its own directory (mounted read-only into every container) and removes it when done. + cmd=$(printf 'd=$(mktemp -d) && echo %s | base64 -d | tar -xz -C "$d" && WORKDIR=%q BUILD=%q TEST=%q MUT_TIMEOUT=%q JOB_TIMEOUT=%q MT_BUNDLE="$d" bash "$d/mutation-table.sh" --host' \ "$bundle" "$workdir" "${BUILD:-}" "${TEST:-}" "${MUT_TIMEOUT:-600}" "${TIMEOUT:-3600}") - TIMEOUT=${TIMEOUT:-3600} "$here/stopgap/ygg-verify.sh" "$repo" "$rev" "$image" "$cmd" + RUN_ON_HOST=1 TIMEOUT=${TIMEOUT:-3600} "$here/stopgap/ygg-verify.sh" "$repo" "$rev" "$image" "$cmd" exit $? fi -# ---- container side --------------------------------------------------------- +# ---- driver (Yggdrasil host, inside the job's slot) -------------------------- +if [ "$1" = --host ]; then + : "${YGG_WORK:?not started by ygg-verify.sh RUN_ON_HOST=1}" "${YGG_IMAGE:?}" "${YGG_CNAME:?}" "${YGG_LABEL:?}" + bundle=${MT_BUNDLE:?} + set +e # every docker step is checked by hand: a mutant container exits non-zero by design + base="$YGG_CNAME-base"; snap="eureka-mt-$YGG_CNAME:snap" + cur= + cleanup() { + [ -z "$cur" ] || sudo docker rm -f "$cur" >/dev/null 2>&1 || true + sudo docker rm -f "$base" >/dev/null 2>&1 || true + sudo docker rmi -f "$snap" >/dev/null 2>&1 || true + [ -z "$bundle" ] || [ "$bundle" = / ] || rm -rf -- "$bundle" + } + trap cleanup EXIT; trap 'exit 143' TERM INT HUP + # Every container carries the job label (the stopgap's watchdog kills by it) and the same caps. + common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}" + -v "$bundle:/mt:ro" -v /etc/machine-id:/etc/machine-id:ro + -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache + -e BASH_ENV=/tmp/.ygg-bash-env + -e "WORKDIR=${WORKDIR:-.}" -e "BUILD=${BUILD:-}" -e "TEST=${TEST:-}" -e "MUT_TIMEOUT=${MUT_TIMEOUT:-600}" -e "JOB_TIMEOUT=${JOB_TIMEOUT:-}" + -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*') + [ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE") + # DOCKER_ARGS is word-split on purpose: it is a flag string. + # shellcheck disable=SC2086 + drun() { sudo nice -n 10 docker run "$@"; } + pre="printf 'set -o pipefail\n' >/tmp/.ygg-bash-env" + + sudo docker rm -f "$base" >/dev/null 2>&1 || true + # shellcheck disable=SC2086 + drun --name "$base" "${common[@]}" -v "$YGG_WORK:/seed:ro" -w /src ${DOCKER_ARGS:-} "$YGG_IMAGE" \ + bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --baseline" + rc=$? + [ "$rc" = 0 ] || exit "$rc" # the baseline container printed why + count=$(sudo docker cp "$base:/tmp/mtm/count" - | tar -xO) + case $count in ''|*[!0-9]*) echo "$SUMTAG: cannot read the mutant count from the baseline container"; exit 2 ;; esac + t0=$(date +%s) + sudo docker commit -q "$base" "$snap" >/dev/null || { echo "$SUMTAG: docker commit failed"; exit 2; } + sudo docker rm -f "$base" >/dev/null + echo "$SUMTAG: SNAPSHOT committed in $(( $(date +%s) - t0 ))s; each mutant runs in a fresh container from it" + + declare -A tally; for o in "${OUTCOMES[@]}"; do tally[$o]=0; done + for n in $(seq 1 "$count"); do + if [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; then + echo "$SUMTAG: JOB KILLED by the watchdog before mutant $n of $count; the finished rows above stand"; exit 124 + fi + cur="$YGG_CNAME-m$n" + # shellcheck disable=SC2086 + drun --rm --name "$YGG_CNAME-m$n" "${common[@]}" -w /src ${DOCKER_ARGS:-} "$snap" \ + bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --mutant $n" + rc=$? + if [ "$rc" -ge "$CODE0" ] && [ "$rc" -lt $((CODE0 + ${#OUTCOMES[@]})) ]; then + o=${OUTCOMES[$((rc - CODE0))]} + else + o=ERROR; echo "$SUMTAG: [$n/$count] mutant $n | ERROR | the mutant container exited $rc without a row (137 is a kill or OOM)" + fi + tally[$o]=$(( ${tally[$o]} + 1 )) + done + echo "$SUMTAG: MUTATION $count mutants: caught=${tally[CAUGHT]} survived=${tally[SURVIVED]} unviable=${tally[UNVIABLE]} timeout=${tally[TIMEOUT]} noapply=${tally[NOAPPLY]} error=${tally[ERROR]} (baseline green, tests ran)" + [ $(( ${tally[CAUGHT]} + ${tally[UNVIABLE]} )) -eq "$count" ] + exit $? +fi + +# ---- inside a container ----------------------------------------------------- set +e -SRC=${MT_SRC:-/src}; MT_DIR=${MT_DIR:-/tmp/mt} # overridable only so tools/test-tools.sh can run this half locally against a mock -# ygg-verify.sh mints a nonce per run and carries only lines tagged with it; unset (a local run), the plain tag is used -SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" +# overridable only so tools/test-tools.sh can run this half against a mock +SRC=${MT_SRC:-/src}; MT_DIR=${MT_DIR:-/mt}; M=${MT_M:-/tmp/mtm}; SEED=${MT_SEED:-/seed} # shellcheck source=testparse.sh . "$MT_DIR/testparse.sh" -cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; } -M="$MT_DIR/m"; rm -rf "$M"; mkdir -p "$M" tmo=${MUT_TIMEOUT:-600} - -if ! perl -e ' - my ($mf, $out) = @ARGV; my ($n, $sec, %c, %seen) = (0, ""); - open my $f, "<", $mf or die "cannot open $mf\n"; - sub flush { - return unless exists $c{label}; - for my $k (qw(file before after)) { die "mutant $c{label}: no $k\n" unless defined $c{$k} } - my $d = "$out/$n"; mkdir $d; - for my $k (qw(label file test before after)) { - my $v = join("\n", ref $c{$k} ? @{$c{$k}} : ($c{$k} // "")); - open my $o, ">:raw", "$d/$k" or die; print $o $v; close $o; - } - } - while (<$f>) { - chomp; - if (/^=== (.+?)\s*$/) { flush(); die "duplicate label $1\n" if $seen{$1}++; %c = (label => $1); $n++; $sec = ""; next } - next unless exists $c{label}; - if ($sec eq "" && /^file: (.+?)\s*$/) { $c{file} = $1; next } - if ($sec eq "" && /^test: (.+?)\s*$/) { $c{test} = $1; next } - if (/^--- (before|after)\s*$/) { $sec = $1; $c{$sec} = []; next } - push @{$c{$sec}}, $_ if $sec; - } - flush(); die "no mutants in $mf\n" unless $n; - print "$n\n";' "$MT_DIR/mutants.txt" "$M" >"$M/count" 2>"$M/err"; then - echo "$SUMTAG: BAD MUTANT FILE: $(cat "$M/err")"; exit 2 -fi -count=$(cat "$M/count") -echo "$SUMTAG: START $count mutants; per-run timeout ${tmo}s; whole-job timeout ${JOB_TIMEOUT:-unknown}s; one row per mutant follows as it finishes, so a timeout keeps the finished rows" - # run : prints the exit status; 124 means timeout run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; } -# The tree is the whole repository, not only the mutated file. A test or build -# that rewrites a tracked file would otherwise carry its pollution into the next -# mutant (a false CAUGHT, 2026-10-01). After every run, tracked state is reset -# to HEAD and any untracked entry that was not there after the baselines is -# removed. Entries present after the baselines (build output such as target/) -# are the baseline and stay, so builds stay incremental. -tree_state() { git -C "$SRC" status --porcelain 2>/dev/null; } -base_state='' -# restore: leaves the tree equal to HEAD plus the baseline entries; non-zero if it cannot. -restore() { - git -C "$SRC" reset -q --hard 2>/dev/null - tree_state | while IFS= read -r line; do - grep -qxF -- "$line" <<<"$base_state" || rm -rf -- "$SRC/${line:3}" +if [ "$1" = --baseline ]; then + # The revision lives in this container's own filesystem, so `docker commit` keeps it. + if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "$SUMTAG: cannot copy $SEED into $SRC"; exit 2; }; fi + cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; } + rm -rf "$M"; mkdir -p "$M" + if ! perl -e ' + my ($mf, $out) = @ARGV; my ($n, $sec, %c, %seen) = (0, ""); + open my $f, "<", $mf or die "cannot open $mf\n"; + sub flush { + return unless exists $c{label}; + for my $k (qw(file before after)) { die "mutant $c{label}: no $k\n" unless defined $c{$k} } + my $d = "$out/$n"; mkdir $d; + for my $k (qw(label file test before after)) { + my $v = join("\n", ref $c{$k} ? @{$c{$k}} : ($c{$k} // "")); + open my $o, ">:raw", "$d/$k" or die; print $o $v; close $o; + } + } + while (<$f>) { + chomp; + if (/^=== (.+?)\s*$/) { flush(); die "duplicate label $1\n" if $seen{$1}++; %c = (label => $1); $n++; $sec = ""; next } + next unless exists $c{label}; + if ($sec eq "" && /^file: (.+?)\s*$/) { $c{file} = $1; next } + if ($sec eq "" && /^test: (.+?)\s*$/) { $c{test} = $1; next } + if (/^--- (before|after)\s*$/) { $sec = $1; $c{$sec} = []; next } + push @{$c{$sec}}, $_ if $sec; + } + flush(); die "no mutants in $mf\n" unless $n; + print "$n\n";' "$MT_DIR/mutants.txt" "$M" >"$M/count" 2>"$M/err"; then + echo "$SUMTAG: BAD MUTANT FILE: $(cat "$M/err")"; exit 2 + fi + count=$(cat "$M/count") + echo "$SUMTAG: START $count mutants; per-run timeout ${tmo}s; whole-job timeout ${JOB_TIMEOUT:-unknown}s; one row per mutant follows as it finishes, so a timeout keeps the finished rows" + declare -A based + for n in $(seq 1 "$count"); do + t=$(cat "$M/$n/test"); t=${t:-$TEST} + [ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; } + [ -z "${based[$t]:-}" ] || continue + based[$t]=1 + rc=$(run "$t" "$M/base.log") + scan=$(tp_scan "$M/base.log") + tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}') + if [ "$rc" != 0 ]; then + echo "$SUMTAG: BASELINE RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 + fi + if [ "$tests" -eq 0 ]; then + echo "$SUMTAG: BASELINE RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2 + fi done - [ "$(tree_state)" = "$base_state" ] -} -# dirtied : what a run changed beyond the mutated file and the baseline -dirtied() { - local rel; rel=$(git ls-files --full-name -- "$1" 2>/dev/null | head -n 1) - [ -n "$rel" ] || return 0 # cannot name the mutated file: report nothing rather than everything - tree_state | grep -vxF -- "$base_state" | grep -vF -- " $rel" | sed "s/^...//" | head -n 3 | paste -sd, - -} + # BUILD runs unmutated too. A mistyped or missing build command fails every + # mutant, and a table of all-UNVIABLE rows reads as a fully killed suite. + if [ -n "${BUILD:-}" ]; then + rc=$(run "$BUILD" "$M/base.log") + if [ "$rc" != 0 ]; then + echo "$SUMTAG: BASELINE RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD" + tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 + fi + fi + # A baseline that rewrote a tracked file would hand every mutant a changed tree + # and make `git diff` blind to the mutation; untracked and ignored output stays. + git -C "$SRC" reset -q --hard 2>/dev/null + exit 0 +fi -declare -A based -for n in $(seq 1 "$count"); do - t=$(cat "$M/$n/test"); t=${t:-$TEST} - [ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; } - [ -z "${based[$t]:-}" ] || continue - based[$t]=1 - rc=$(run "$t" "$M/base.log") - scan=$(tp_scan "$M/base.log") - tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}') - if [ "$rc" != 0 ]; then - echo "$SUMTAG: BASELINE RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 +# --mutant +n=${2:?mutant number} +cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; } +count=$(cat "$M/count") +label=$(cat "$M/$n/label"); file=$(cat "$M/$n/file"); t=$(cat "$M/$n/test"); t=${t:-$TEST} +out=''; detail='' +if ! detail=$(perl -e ' + local $/; my ($file, $bf, $af) = @ARGV; + open my $h, "<:raw", $file or do { print "cannot read $file\n"; exit 3 }; my $c = <$h>; close $h; + open my $b, "<:raw", $bf; my $before = <$b> // ""; open my $a, "<:raw", $af; my $after = <$a> // ""; + if (length($before) == 0) { print "empty before text\n"; exit 3 } + if ($before eq $after) { print "after equals before\n"; exit 3 } + # a CRLF file matches multi-line before text written with LF + if ($c =~ /\r\n/ && $before !~ /\r/) { s/\n/\r\n/g for $before, $after } + my ($cnt, $p) = (0, 0); while (($p = index($c, $before, $p)) >= 0) { $cnt++; $p += length($before) } + if ($cnt == 0) { print "before text not found\n"; exit 3 } + if ($cnt > 1) { print "before text occurs $cnt times; make it unique\n"; exit 3 } + substr($c, index($c, $before), length($before)) = $after; + open my $o, ">:raw", $file or exit 3; print $o $c; close $o;' "$file" "$M/$n/before" "$M/$n/after" 2>&1); then + out=NOAPPLY +elif git diff --quiet -- "$file"; then + out=NOAPPLY; detail="applied, but git sees no change in $file" +else + out=''; detail='' + if [ -n "${BUILD:-}" ]; then + rc=$(run "$BUILD" "$M/run.log") + if [ "$rc" = 124 ]; then out=TIMEOUT; detail="build exceeded ${tmo}s" + elif [ "$rc" != 0 ]; then out=UNVIABLE; detail="build exit $rc"; fi fi - if [ "$tests" -eq 0 ]; then - echo "$SUMTAG: BASELINE RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2 - fi -done -# BUILD runs unmutated too. A mistyped or missing build command fails every -# mutant, and a table of all-UNVIABLE rows reads as a fully killed suite. -if [ -n "${BUILD:-}" ]; then - rc=$(run "$BUILD" "$M/base.log") - if [ "$rc" != 0 ]; then - echo "$SUMTAG: BASELINE RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD" - tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2 + if [ -z "$out" ]; then + rc=$(run "$t" "$M/run.log") + if [ "$rc" = 0 ]; then out=SURVIVED; detail="tests passed with the mutation applied: $t" + elif [ "$rc" = 124 ]; then out=TIMEOUT; detail="test exceeded ${tmo}s: $t" + else + scan=$(tp_scan "$M/run.log") + named=$(printf '%s\n' "$scan" | grep '^FAILED ' | awk '!s[$0]++' | sed 's/^FAILED //') + loads=$(printf '%s\n' "$scan" | grep '^LOADFAIL ' | awk '!s[$0]++' | sed 's/^LOADFAIL //') + if [ -n "$named" ]; then + out=CAUGHT; detail=$(printf '%s\n' "$named" | head -n 1) + k=$(printf '%s\n' "$named" | wc -l); [ "$k" -le 1 ] || detail="$detail (+$((k - 1)) more)" + elif [ -n "$loads" ]; then out=UNVIABLE; detail="a test file failed to load, no test ran: $(printf '%s\n' "$loads" | head -n 1)" + else out=ERROR; detail="exit $rc, no failing test parsed: $(tail -n 1 "$M/run.log" | cut -c1-120)"; fi + fi fi fi -# Baseline runs may dirty tracked files; settle that before the first mutant. -git -C "$SRC" reset -q --hard 2>/dev/null -base_state=$(tree_state) - -for n in $(seq 1 "$count"); do - label=$(cat "$M/$n/label"); file=$(cat "$M/$n/file"); t=$(cat "$M/$n/test"); t=${t:-$TEST} - out=''; detail='' - if ! detail=$(perl -e ' - local $/; my ($file, $bf, $af) = @ARGV; - open my $h, "<:raw", $file or do { print "cannot read $file\n"; exit 3 }; my $c = <$h>; close $h; - open my $b, "<:raw", $bf; my $before = <$b> // ""; open my $a, "<:raw", $af; my $after = <$a> // ""; - if (length($before) == 0) { print "empty before text\n"; exit 3 } - if ($before eq $after) { print "after equals before\n"; exit 3 } - # a CRLF file matches multi-line before text written with LF - if ($c =~ /\r\n/ && $before !~ /\r/) { s/\n/\r\n/g for $before, $after } - my ($cnt, $p) = (0, 0); while (($p = index($c, $before, $p)) >= 0) { $cnt++; $p += length($before) } - if ($cnt == 0) { print "before text not found\n"; exit 3 } - if ($cnt > 1) { print "before text occurs $cnt times; make it unique\n"; exit 3 } - substr($c, index($c, $before), length($before)) = $after; - open my $o, ">:raw", $file or exit 3; print $o $c; close $o;' "$file" "$M/$n/before" "$M/$n/after" 2>&1); then - out=NOAPPLY - elif git diff --quiet -- "$file"; then - out=NOAPPLY; detail="applied, but git sees no change in $file" - else - out=''; detail='' - if [ -n "${BUILD:-}" ]; then - rc=$(run "$BUILD" "$M/run.log") - if [ "$rc" = 124 ]; then out=TIMEOUT; detail="build exceeded ${tmo}s" - elif [ "$rc" != 0 ]; then out=UNVIABLE; detail="build exit $rc"; fi - fi - if [ -z "$out" ]; then - rc=$(run "$t" "$M/run.log") - if [ "$rc" = 0 ]; then out=SURVIVED; detail="tests passed with the mutation applied: $t" - elif [ "$rc" = 124 ]; then out=TIMEOUT; detail="test exceeded ${tmo}s: $t" - else - names=$(tp_scan "$M/run.log" | grep '^FAILED ' | awk '!s[$0]++' | sed 's/^FAILED //') - # A file-level failure (vitest: "FAIL [ ]", no test name) is a - # module that did not load: a compile error, never a kill. - named=$(printf '%s\n' "$names" | grep -v -E '^[^ >]+ \[ .* \]$') - if [ -n "$named" ]; then - out=CAUGHT; detail=$(printf '%s\n' "$named" | head -n 1) - k=$(printf '%s\n' "$named" | wc -l); [ "$k" -le 1 ] || detail="$detail (+$((k - 1)) more)" - elif [ -n "$names" ]; then out=UNVIABLE; detail="a test file failed to load, no test ran: $(printf '%s\n' "$names" | head -n 1)" - else out=ERROR; detail="exit $rc, no failing test parsed: $(tail -n 1 "$M/run.log" | cut -c1-120)"; fi - fi - fi - extra=$(dirtied "$file"); [ -z "$extra" ] || detail="$detail [run also changed: $extra; restored]" - fi - # The outcome is its own field. It is never recovered from rendered text, so a - # label or a test name containing " | CAUGHT | " cannot forge a kill. - printf '%s' "$out" >"$M/$n/outcome" - detail=${detail//$'\n'/ } - printf '%s: [%s/%s] %s | %s | %s\n' "$SUMTAG" "$n" "$count" "${label//|/\\|}" "$out" "$detail" - if ! restore; then - echo "$SUMTAG: RESTORE FAILED after $label: the tree differs from the baseline; stopping. Status: $(tree_state | head -n 5 | paste -sd';' -)"; exit 2 - fi -done - -tally() { local c=0 f; for f in "$M"/*/outcome; do [ "$(cat "$f")" = "$1" ] && c=$((c + 1)); done; echo "$c"; } -caught=$(tally CAUGHT); unviable=$(tally UNVIABLE) -echo "$SUMTAG: MUTATION $count mutants: caught=$caught survived=$(tally SURVIVED) unviable=$unviable timeout=$(tally TIMEOUT) noapply=$(tally NOAPPLY) error=$(tally ERROR) (baseline green, tests ran)" -[ $((caught + unviable)) -eq "$count" ] +detail=${detail//$'\n'/ } +printf '%s: [%s/%s] %s | %s | %s\n' "$SUMTAG" "$n" "$count" "${label//|/\\|}" "$out" "$detail" +for i in "${!OUTCOMES[@]}"; do [ "${OUTCOMES[$i]}" = "$out" ] && exit $((CODE0 + i)); done +exit 1 diff --git a/tools/stopgap/ygg-verify.sh b/tools/stopgap/ygg-verify.sh index d596e48..5513b53 100644 --- a/tools/stopgap/ygg-verify.sh +++ b/tools/stopgap/ygg-verify.sh @@ -78,6 +78,15 @@ # ygg-verify.sh '' # TIMEOUT= caps the job (default 3600); a killed job exits non-zero. # IDLE= kills a job idle that long (default 900; 0 disables). +# RUN_ON_HOST=1 runs on the Yggdrasil host instead of in one container, +# inside the same slot, clone, TIMEOUT and IDLE watchdog, with the environment +# YGG_WORK (the clone), YGG_IMAGE, YGG_CPUS, YGG_MEM, YGG_CNAME (job name), +# YGG_LABEL (every container it starts must carry `--label $YGG_LABEL`: the +# watchdog kills containers by that label), YGG_KILLED (a path that exists once +# the watchdog has killed the job: stop starting work) and DOCKER_ARGS. The +# command owns the container topology and must remove what it creates. +# tools/mutation-table.sh is the one user: it snapshots a baselined container +# and runs each mutant in a fresh container from the snapshot. # DOCKER_ARGS='' passes through to the container's # own `docker run`, appended after the fixed flags and before the image name # (e.g. DOCKER_ARGS='--sysctl net.ipv6.conf.all.disable_ipv6=0' or @@ -92,7 +101,9 @@ # TESTS RAN line when counts exist and are all zero, failing test names, and # for a red job the failure sections and the log tail. When the job ends it # writes .summary and then .done (holding the exit -# status). Use a fresh per job (a reused one is guarded by a run id, see the run protocol below). Launch it as ONE background shell +# status). Use a fresh per job: a reused path is unsafe for the first +# SETTLE seconds (see the run protocol below), so fresh paths are the rule. +# Launch it as ONE background shell # (run_in_background), then: # ygg-verify.sh wait # Blocks at most 270 s (WAIT_MAX can only lower that), prints the summary and @@ -135,7 +146,7 @@ here=$(cd "$(dirname "$0")" && pwd) # `YGG-SUMMARY` line without it is counted and ignored. Carried lines never # replace the failure sections or the tail. ygg_summarize() { - local log=$1 status=$2 note=${3:-} word=RED scan fw names p f nf own=0 untrusted=0 tag secs n + local log=$1 status=$2 note=${3:-} word=RED scan fw names loads p f nf own=0 untrusted=0 tag secs n [ "$status" = 0 ] && word=GREEN # shellcheck source=../testparse.sh . "$here/../testparse.sh" @@ -150,6 +161,7 @@ ygg_summarize() { names=$(printf '%s\n' "$scan" | grep '^FAILED ' | awk '!seen[$0]++' || true) read -r p f < <(printf '%s\n' "$fw" | awk '{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed")p+=a[2];if(a[1]=="failed")f+=a[2]}}END{print p+0, f+0}') nf=0; [ -z "$names" ] || nf=$(printf '%s\n' "$names" | wc -l) + loads=$(printf '%s\n' "$scan" | grep '^LOADFAIL ' | awk '!seen[$0]++' || true) { echo "YGG-VERIFY exit $status $word${note:+ ($note)} log: $log" grep -E '^ygg-verify: (TIMEOUT|IDLE|NO VERDICT)' "$log" || true @@ -162,6 +174,9 @@ ygg_summarize() { if [ "$nf" -gt 0 ]; then printf '%s\n' "$names" | awk 'NR<=15{print "FAILED " substr($0,8)} END{if(NR>15)print "... " NR-15 " more failing tests"}' fi + if [ "$nf" -eq 0 ] && [ -n "$loads" ]; then + printf '%s\n' "$loads" | awk 'NR<=5{print "FAILED TO LOAD " substr($0,10)} END{if(NR>5)print "... " NR-5 " more files"}' + fi if [ "$f" -gt "$nf" ]; then echo "NAMES MISSING: the counts show $f failed but only $nf named; the rest are unnamed here, open the log" fi @@ -173,7 +188,7 @@ ygg_summarize() { fi { grep -E '^(MISSED|TIMEOUT) |mutants tested' "$log" || true; } | awk 'NR<=20' if [ "$status" != 0 ]; then - if [ "$nf" -eq 0 ] && [ "$f" -eq 0 ] && [ "$own" -eq 0 ]; then + if [ "$nf" -eq 0 ] && [ "$f" -eq 0 ] && [ "$own" -eq 0 ] && [ -z "$loads" ]; then echo "exit non-zero with no failing test parsed: build error, crash, timeout or a non-test command" fi secs=$(tp_sections "$log" 30 | awk 'NR<=40') @@ -287,7 +302,7 @@ esac # ssh joins its arguments into one string that the remote shell splits again, # so every argument is quoted here. Without that, `a && b` in the command runs # b on the host. -remote_args=$(printf '%q ' "$name" "$sha" "$image" "$cpus" "$mem" "$slots" "${KEEP:-0}" "$cmd" "$timeout_s" "$docker_args" "$idle_s" "${YGG_NONCE:-}") +remote_args=$(printf '%q ' "$name" "$sha" "$image" "$cpus" "$mem" "$slots" "${KEEP:-0}" "$cmd" "$timeout_s" "$docker_args" "$idle_s" "${YGG_NONCE:-}" "${RUN_ON_HOST:-0}") # The heredoc below travels with whatever line endings this file has on disk. # On Windows that is CRLF, and the remote bash then fails on `set -euo pipefail ` # BEFORE `set -e` is in force, so the script limps on and can exit 0 — a job that @@ -297,7 +312,7 @@ remote_args=$(printf '%q ' "$name" "$sha" "$image" "$cpus" "$mem" "$slots" "${KE # refuses to report a status it did not receive. (ssh "${sshopts[@]}" "$host" "tr -d ' ' | bash -s -- $remote_args" <<'REMOTE' set -euo pipefail -name=$1 sha=$2 image=$3 cpus=$4 mem=$5 slots=$6 keep=$7 cmd=$8 timeout_s=$9 docker_args=${10} idle_s=${11} nonce=${12:-} +name=$1 sha=$2 image=$3 cpus=$4 mem=$5 slots=$6 keep=$7 cmd=$8 timeout_s=$9 docker_args=${10} idle_s=${11} nonce=${12:-} host_mode=${13:-0} root=~/eureka-verify case "$image" in eureka-verify-*:*) @@ -321,7 +336,10 @@ while [ -z "$slot" ]; do fi done work=$(mktemp -d "$root/work/$name-${sha:0:10}-XXXX") -trap 'if [ "$keep" != 1 ]; then sudo rm -rf -- "$work"; fi' EXIT +cname="eureka-verify-$slot-$$" +label="eureka-verify-job=$cname" +killed="$root/work/$cname.killed" +trap 'rm -f "$killed"; if [ "$keep" != 1 ]; then sudo rm -rf -- "$work"; fi' EXIT git clone -q --no-checkout "$root/repos/$name.git" "$work" git -C "$work" checkout -q --detach "$sha" echo "ygg-verify: $name@${sha:0:10} slot $slot, image $image, cpus $cpus, mem $mem, work $work" >&2 @@ -329,7 +347,14 @@ echo "ygg-verify: $name@${sha:0:10} slot $slot, image $image, cpus $cpus, mem $m # (2026-09-23: a test process idled at 0% CPU for 25 minutes and the agent # waiting on it never woke). The watchdog kills the named container; killing # the docker client alone would leave the container running. -cname="eureka-verify-$slot-$$" +# The watchdog's one kill: every container the job started carries $label (a +# host-side command may own a baseline container and a mutant container at once, +# so there is no single name to kill). The flag tells that command the job is over. +kill_job() { + touch "$killed" + local ids; ids=$(sudo docker ps -q --filter "label=$label" 2>/dev/null || true) + [ -z "$ids" ] || sudo docker kill $ids >/dev/null 2>&1 || true +} # The subshell must not inherit the slot lock: its sleep outlives a job that # finishes early, and an inherited fd held slot 2 for an hour (2026-09-23). # It also kills a job that has idled for $idle_s seconds: docker's CPU% is per @@ -338,21 +363,29 @@ cname="eureka-verify-$slot-$$" start=$(date +%s); idle=0 while sleep 60; do if [ $(( $(date +%s) - start )) -ge "$timeout_s" ]; then - sudo docker kill "$cname" >/dev/null 2>&1 && echo "ygg-verify: TIMEOUT after ${timeout_s}s, container killed" >&3 + kill_job; echo "ygg-verify: TIMEOUT after ${timeout_s}s, container killed" >&3 exit 0 fi [ "$idle_s" -gt 0 ] || continue - cpu=$(sudo docker stats --no-stream --format '{{.CPUPerc}}' "$cname" 2>/dev/null | tr -d '%' || true) + ids=$(sudo docker ps -q --filter "label=$label" 2>/dev/null || true) + [ -n "$ids" ] || continue + cpu=$(sudo docker stats --no-stream --format '{{.CPUPerc}}' $ids 2>/dev/null | tr -d '%' | awk '{ s += $1 } END { if (NR) print s }' || true) [ -n "$cpu" ] || continue if awk -v c="$cpu" 'BEGIN { exit !(c < 5.0) }'; then idle=$((idle + 60)); else idle=0; fi if [ "$idle" -ge "$idle_s" ]; then - sudo docker kill "$cname" >/dev/null 2>&1 && echo "ygg-verify: IDLE ${idle_s}s under 5% CPU, container killed (a test is probably hung)" >&3 + kill_job; echo "ygg-verify: IDLE ${idle_s}s under 5% CPU, container killed (a test is probably hung)" >&3 exit 0 fi done ) 3>&2 & watchdog=$! set +e -sudo nice -n 10 docker run --rm --name "$cname" --cpus="$cpus" --memory="$mem" \ +if [ "$host_mode" = 1 ]; then + ( export YGG_WORK=$work YGG_IMAGE=$image YGG_CPUS=$cpus YGG_MEM=$mem YGG_CNAME=$cname YGG_LABEL=$label \ + YGG_KILLED=$killed DOCKER_ARGS=$docker_args + [ -z "$nonce" ] || export YGG_NONCE=$nonce + exec bash -o pipefail -c "$cmd" ) +else +sudo nice -n 10 docker run --rm --name "$cname" --label "$label" --cpus="$cpus" --memory="$mem" \ -v "$work:/src" -v /etc/machine-id:/etc/machine-id:ro \ -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages \ -v eureka-sccache:/sccache \ @@ -360,6 +393,7 @@ sudo nice -n 10 docker run --rm --name "$cname" --cpus="$cpus" --memory="$mem" \ -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0='*' \ $docker_args \ -w /src "$image" bash -o pipefail -c "printf 'set -o pipefail\n' >/tmp/.ygg-bash-env; $cmd" +fi status=$? set -e pkill -P "$watchdog" 2>/dev/null || true diff --git a/tools/test-tools.sh b/tools/test-tools.sh index 1c737bb..721c0c9 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -45,15 +45,20 @@ check exit0-but-failed.log 0 "MISMATCH: exit 0 but the log shows" check no-verdict.log 97 "NO VERDICT" check no-verdict.log 97 "RED" "GREEN" -# ---- tools/mutation-table.sh, container half, against the mock in -# fixtures/mutation-mock, run from the subdirectory pkg/ (a workdir below the repo root) (shell scripts that print cargo/vitest-shaped output). -# One case per finding of the 2026-10-01 Soul pass; each fails if it regresses. -mt() { # mt [BUILD]: sets mt_out, mt_rc, mt_repo - local root; root=$(mktemp -d); mt_repo=$root/repo; mkdir -p "$mt_repo" "$root/mt" - mkdir -p "$mt_repo/pkg"; cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/" +# ---- tools/mutation-table.sh, host driver plus container half, against the mock in +# fixtures/mutation-mock: shell scripts that print cargo/vitest/jest-shaped output, run +# from the subdirectory pkg/ (a workdir below the repo root), under a mock docker +# (fixtures/mutation-mock/fakebin) where a container is a directory and `commit` copies +# it, so every mutant really runs on a copy of the baselined snapshot. +# One case per finding of the 2026-10-01 Soul passes; each fails if it regresses. +mt() { # mt [BUILD]: sets mt_out, mt_rc, mt_repo, mt_fd + local root; root=$(mktemp -d); mt_repo=$root/repo; mt_fd=$root/fd; mkdir -p "$mt_repo/pkg" "$root/mt" "$mt_fd" + cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/"; rm -rf "$mt_repo/pkg/fakebin" git -C "$mt_repo" init -q && git -C "$mt_repo" add -A && git -C "$mt_repo" -c user.name=t -c user.email=t@t commit -qm mock - cp "$here/testparse.sh" "$root/mt/"; cp "$here/fixtures/mutation-mock/$1" "$root/mt/mutants.txt" - mt_out=$(WORKDIR=pkg MT_SRC=$mt_repo MT_DIR=$root/mt TEST=$2 BUILD=${3:-} MUT_TIMEOUT=${MT_TO:-15} timeout "${MT_WALL:-120}" bash "$here/mutation-table.sh" --in-container 2>&1) + cp "$here/testparse.sh" "$here/mutation-table.sh" "$root/mt/"; cp "$here/fixtures/mutation-mock/$1" "$root/mt/mutants.txt" + mt_out=$(PATH="$here/fixtures/mutation-mock/fakebin:$PATH" FAKE_DOCKER=$mt_fd YGG_WORK=$mt_repo YGG_IMAGE=base YGG_CNAME=job1 YGG_LABEL=eureka-verify-job=job1 \ + YGG_CPUS=1 YGG_MEM=1g YGG_KILLED=${MT_KILLED:-} MT_BUNDLE=$root/mt WORKDIR=pkg TEST=$2 BUILD=${3:-} MUT_TIMEOUT=${MT_TO:-15} \ + timeout "${MT_WALL:-120}" bash "$here/mutation-table.sh" --host 2>&1) mt_rc=$? } mtcheck() { # mtcheck ... @@ -66,27 +71,47 @@ mtcheck() { # mtcheck ... done echo "ok mutation $name" } +# nothing the job made may outlive it: no container directory, no snapshot image, no bundle +mtclean() { # mtclean [images]: a killed mock container is a still-running process, so E checks only the image and bundle + local left dirs=("$mt_fd/c" "$mt_fd/i"); [ -z "${2:-}" ] || dirs=("$mt_fd/i") + left=$(ls "${dirs[@]}" 2>/dev/null | grep -v '^$' | grep -v ':$' | paste -sd, -) + if [ -z "$left" ] && [ ! -e "$(dirname "$mt_repo")/mt" ]; then echo "ok mutation $1: containers, snapshot image and bundle removed" + else echo "FAIL mutation $1: left behind: ${left:-the bundle dir}"; fail=1; fi +} # A: a mistyped BUILD must abort, not read as a fully killed suite mt mutants-a.txt 'bash test.sh' 'bash bulid.sh' [ "$mt_rc" = 2 ] || { echo "FAIL mutation A: exit $mt_rc, want 2"; fail=1; } -mtcheck A-build-baseline "BASELINE RED: BUILD failed" "!UNVIABLE" +mtcheck A-build-baseline "BASELINE RED: BUILD failed" "!UNVIABLE" "!SNAPSHOT" +mtclean A # B: a label that spells a kill must not forge one mt mutants-b.txt 'bash test.sh' [ "$mt_rc" = 1 ] || { echo "FAIL mutation B: exit $mt_rc, want 1"; fail=1; } mtcheck B-label-forge "caught=0 survived=1" 'forge \| CAUGHT \| label | SURVIVED |' +mtclean B # the nonce form ygg-verify.sh carries, and the plain form when no nonce is set YGG_NONCE=n0nce1 mt mutants-b.txt 'bash test.sh' mtcheck B-nonce "YGG-SUMMARY/n0nce1: [1/1] " "!YGG-SUMMARY: " -# C: a test that rewrites a tracked file or leaks an untracked one must not decide the next mutant +# C: a test that rewrites a tracked file, leaks an untracked one or keeps state in an +# IGNORED file (cache.txt: what `git status` cannot see) must not decide the next mutant mt mutants-c.txt 'bash test.sh' -mtcheck C-pollution "stamp-bad | CAUGHT | tests::b_write" "also changed: pkg/data.txt,pkg/leak.tmp" "untested-equiv | SURVIVED" "!untested-equiv | CAUGHT" -[ -z "$(git -C "$mt_repo" status --porcelain)" ] || { echo "FAIL mutation C: tree not restored"; fail=1; } +mtcheck C-pollution "stamp-bad | CAUGHT | tests::b_write" "untested-equiv | SURVIVED" "!untested-equiv | CAUGHT" "caught=1 survived=1" +[ -z "$(git -C "$mt_repo" status --porcelain --ignored)" ] || { echo "FAIL mutation C: the seed repository changed"; fail=1; } +mtclean C # D: a test file that fails to load is not a kill; a named failing test is mt mutants-d.txt 'bash vtest.sh' mtcheck D-import-failure "syntax-break | UNVIABLE" "wrong-result | CAUGHT | add.js > adds" "caught=1 survived=0 unviable=1" -# E: rows print as mutants finish, so a job timeout keeps them +# D2 (jest): "FAIL " with "Test suite failed to run" is a load failure; the "●" block names the test +mt mutants-h.txt 'bash jtest.sh' +mtcheck D2-jest "jest-load | UNVIABLE" "jest-wrong | CAUGHT | add › adds" "caught=1 survived=0 unviable=1" +# E: rows print as mutants finish, so a job timeout keeps them; the TERM the timeout sends still cleans up MT_WALL=8 mt mutants-e.txt 'bash test.sh' mtcheck E-live-rows "[1/2] quick | CAUGHT" "!MUTATION" +sleep 1; mtclean E images +# E2: a watchdog kill (the flag the stopgap sets) stops the fan-out after the baseline and cleans up +mt_k=$(mktemp); MT_KILLED=$mt_k mt mutants-c.txt 'bash test.sh'; rm -f "$mt_k" +[ "$mt_rc" = 124 ] || { echo "FAIL mutation E2: exit $mt_rc, want 124"; fail=1; } +mtcheck E2-killed "JOB KILLED by the watchdog before mutant 1 of 2" "!CAUGHT" +mtclean E2 # F: LF before-text matches a CRLF file mt mutants-f.txt 'bash test.sh' mtcheck F-crlf "crlf-multiline | CAUGHT | tests::d_win" @@ -144,4 +169,13 @@ echo r1 >"$W.run"; echo "r0 3" >"$W.done"; echo $$ >"$W.pid" # done from anoth out=$(WAIT_MAX=3 SETTLE=0 bash "$V" wait "$W"); st=$? if [ "$st" = 99 ]; then echo "ok wait: a done from another run is not a result" else echo "FAIL wait foreign done: exit $st, output: $out"; fail=1; fi +# N2 (Soul, 2026-10-01): file-level failures are not named test failures, whichever runner +# prints them. Vitest with a project prefix and with spaces in the path; jest's file header. +check vitest-project-load-failure.log 1 "FAILED TO LOAD |unit| src/add.test.ts" "FAILED |unit|" +check vitest-spaces-load-failure.log 1 "FAILED TO LOAD src/my tests/add.test.ts" "FAILED src/my" +check vitest-project-red.log 1 "FAILED |unit| src/add.test.ts > add > carries" "FAILED TO LOAD" +check jest-red.log 1 "FRAMEWORK jest passed=1 failed=1 skipped=1" +check jest-red.log 1 "FAILED add › carries" "FAILED TO LOAD" +check jest-load-failure.log 1 "FAILED TO LOAD src/add.test.js" "FAILED src/add.test.js" +check jest-load-failure.log 1 "FRAMEWORK jest passed=0 failed=0 skipped=0" exit $fail diff --git a/tools/testparse.sh b/tools/testparse.sh index f999063..a60b308 100644 --- a/tools/testparse.sh +++ b/tools/testparse.sh @@ -4,10 +4,12 @@ # tp_scan one line per fact: # FRAMEWORK passed= failed= skipped= (summed over the log) # FAILED (one per failing test) +# LOADFAIL (a test FILE that failed, with no test name: +# a module that did not load; jest also prints one for a file with failing tests) # tp_sections [max-lines] cargo failure sections ("---- name stdout # ----"), each cut to its last lines (default 30) # -# Frameworks: cargo test, dotnet test, vitest/jest, pytest. Anything else is +# Frameworks: cargo test, dotnet test, vitest, jest, pytest. Anything else is # simply not recognised; the callers treat "recognised nothing" as a fact to # print, never as success. # @@ -64,7 +66,26 @@ tp_scan() { } vitest = 1; next } - /^ *FAIL / { name = $0; sub(/^ *FAIL +/, "", name); print "FAILED " name; next } + /^ *Tests: +[0-9]+ / { # jest: "Tests: 1 failed, 2 passed, 3 total" + line = $0; sub(/^ *Tests: +/, "", line) + n = split(line, parts, ", ") + for (i = 1; i <= n; i++) { + v = num(parts[i]) + if (parts[i] ~ /failed/) jf += v + if (parts[i] ~ /passed/) jp += v + if (parts[i] ~ /skipped/) js += v + } + jest = 1; next + } + # vitest and jest print " FAIL " for a file and " FAIL > > " (vitest, + # optionally with a "|project|" prefix) for a test. A line with no " > " names a FILE: a vitest load + # failure (" [ ]"), or a jest file header whose tests are named by the "●" lines below. + /^ *FAIL / { name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next } + /^ +● / { # jest failure block header: " ● suite › test" + name = $0; sub(/^ +● +/, "", name) + if (name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name + next + } /^(=+ )?(no tests ran|[0-9]+ [a-z]+(, [0-9]+ [a-z]+)*) in [0-9.]+s( \([0-9:]+\))?( =+)?$/ { # pytest summary, with or without -q line = $0; sub(/^=+ /, "", line); sub(/ =+$/, "", line); sub(/ in [0-9.]+s.*$/, "", line) n = split(line, parts, ", "); ok = 1; a = b = c = 0 @@ -84,6 +105,7 @@ tp_scan() { if (cargo) printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n", cp, cf, cs if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs + if (jest) printf "FRAMEWORK jest passed=%d failed=%d skipped=%d\n", jp, jf, js if (pytest) printf "FRAMEWORK pytest passed=%d failed=%d skipped=%d\n", pp, pf, ps }' "$1" }