Eureka/tools/mutation-table.sh
Meta Vens 60a09dc17c Tools: ygg-verify summary and wait modes, mutation-table runner, shared test parser
SUMMARY=<log> keeps the full log and prints a bounded verdict summary;
wait <log> blocks at most 270 s and returns it once. mutation-table.sh runs
hand mutants in one verify job and returns a kill table. testparse.sh is the
one parser for both. Fixtures: real cargo and mutation logs from Yggdrasil
(tiny-crate), plus hand-written dotnet, vitest, pytest, exit-0-but-failed and
no-verdict logs. Every commit here builds; verified live on Yggdrasil.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-01 15:09:07 +02:00

167 lines
8.5 KiB
Shell

#!/usr/bin/env bash
# Hand-mutant kill table: run a list of mutants against a revision on
# Yggdrasil, in ONE verify job, and get one table back.
#
# tools/mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]
# workdir directory inside the repo where the commands run (e.g.
# packages/cultcache-rs; CultLib has no Cargo.toml at its root).
# env TEST default test command for mutants with no `test:` line
# BUILD optional compile-only command; when it fails the mutant
# is UNVIABLE. Without it a mutant that does not compile
# has no failing test and reads ERROR, never CAUGHT.
# MUT_TIMEOUT seconds per build or test run (default 600)
# everything ygg-verify.sh reads (SUMMARY=<log>, TIMEOUT, MEM, ...)
# Typical, from an agent: launch as ONE background shell with
# SUMMARY=<scratch>/mut1.log TEST='cargo test' tools/mutation-table.sh ...
# then `tools/stopgap/ygg-verify.sh wait <scratch>/mut1.log` in the foreground.
#
# Mutants file (plain text; the text before the first `===` is ignored):
# === label-unique-in-the-file
# file: src/lib.rs (relative to workdir)
# test: cargo test add_ (optional; else $TEST)
# --- before
# a + b (exact text, may span lines; must occur ONCE)
# --- after
# a - b
#
# Each mutant is applied in the container's throwaway clone and reverted from
# git before the next, so no tree you care about is touched. The output is one
# row per mutant: label | outcome | killing test or detail. Outcomes:
# CAUGHT the test command failed AND a failing test name was parsed from it
# SURVIVED the test command passed with the mutation applied
# UNVIABLE BUILD failed
# TIMEOUT a run hit MUT_TIMEOUT (not a kill)
# NOAPPLY before text absent, ambiguous, or equal to after (no mutation made)
# ERROR the test command failed but no failing test was parsed (a build
# error without BUILD, a crash): not a kill
# Before any mutant, every distinct test command runs UNMUTATED. If one fails or
# runs zero tests the job aborts with `BASELINE RED`, because a red or empty
# baseline would make every mutant look killed or every one survive.
# Exit 0 means every mutant was CAUGHT or UNVIABLE and nothing else happened.
#
# DELETION LINE: this runner replaces the per-agent runmut/mutrun/strykrun
# scripts. It dies with tools/stopgap/ygg-verify.sh when `idunn verify` ships a
# typed mutation step. It reads tools/testparse.sh for failure names, the same
# parser the summary uses.
set -euo pipefail
if [ "${1:-}" != --in-container ]; then
here=$(cd "$(dirname "$0")" && pwd)
repo=${1:?usage: mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]}
rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.}
[ -r "$mfile" ] || { echo "mutation-table: cannot read $mfile" >&2; exit 2; }
stage=$(mktemp -d)
trap 'rm -rf "$stage"' EXIT
cp "$here/testparse.sh" "$here/mutation-table.sh" "$stage/"
tr -d '\r' <"$mfile" >"$stage/mutants.txt"
bundle=$(tar -C "$stage" -cz testparse.sh mutation-table.sh mutants.txt | base64 | tr -d '\n')
cmd=$(printf 'mkdir -p /tmp/mt && echo %s | base64 -d | tar -xz -C /tmp/mt && WORKDIR=%q BUILD=%q TEST=%q MUT_TIMEOUT=%q bash /tmp/mt/mutation-table.sh --in-container' \
"$bundle" "$workdir" "${BUILD:-}" "${TEST:-}" "${MUT_TIMEOUT:-600}")
"$here/stopgap/ygg-verify.sh" "$repo" "$rev" "$image" "$cmd"
exit $?
fi
# ---- container side ---------------------------------------------------------
set +e
# shellcheck source=testparse.sh
. /tmp/mt/testparse.sh
cd "/src/${WORKDIR:-.}" || { echo "YGG-SUMMARY: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
M=/tmp/mt/m; rm -rf "$M"; mkdir -p "$M"
tmo=${MUT_TIMEOUT:-600}
if ! perl -e '
my ($mf, $out) = @ARGV; my ($n, $sec, %c, %seen) = (0, "");
open my $f, "<", $mf or die "cannot open $mf\n";
sub flush {
return unless exists $c{label};
for my $k (qw(file before after)) { die "mutant $c{label}: no $k\n" unless defined $c{$k} }
my $d = "$out/$n"; mkdir $d;
for my $k (qw(label file test before after)) {
my $v = join("\n", ref $c{$k} ? @{$c{$k}} : ($c{$k} // ""));
open my $o, ">:raw", "$d/$k" or die; print $o $v; close $o;
}
}
while (<$f>) {
chomp;
if (/^=== (.+?)\s*$/) { flush(); die "duplicate label $1\n" if $seen{$1}++; %c = (label => $1); $n++; $sec = ""; next }
next unless exists $c{label};
if ($sec eq "" && /^file: (.+?)\s*$/) { $c{file} = $1; next }
if ($sec eq "" && /^test: (.+?)\s*$/) { $c{test} = $1; next }
if (/^--- (before|after)\s*$/) { $sec = $1; $c{$sec} = []; next }
push @{$c{$sec}}, $_ if $sec;
}
flush(); die "no mutants in $mf\n" unless $n;
print "$n\n";' /tmp/mt/mutants.txt "$M" >"$M/count" 2>"$M/err"; then
echo "YGG-SUMMARY: BAD MUTANT FILE: $(cat "$M/err")"; exit 2
fi
count=$(cat "$M/count")
# run <command> <logfile>: prints the exit status; 124 means timeout
run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; }
declare -A based
for n in $(seq 1 "$count"); do
t=$(cat "$M/$n/test"); t=${t:-$TEST}
[ -n "$t" ] || { echo "YGG-SUMMARY: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; }
[ -z "${based[$t]:-}" ] || continue
based[$t]=1
rc=$(run "$t" "$M/base.log")
scan=$(tp_scan "$M/base.log")
tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}')
if [ "$rc" != 0 ]; then
echo "YGG-SUMMARY: BASELINE RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed 's/^/YGG-SUMMARY: /'; exit 2
fi
if [ "$tests" -eq 0 ]; then
echo "YGG-SUMMARY: BASELINE RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2
fi
done
: >"$M/rows"
for n in $(seq 1 "$count"); do
label=$(cat "$M/$n/label"); file=$(cat "$M/$n/file"); t=$(cat "$M/$n/test"); t=${t:-$TEST}
out=''; detail=''
if ! detail=$(perl -e '
local $/; my ($file, $bf, $af) = @ARGV;
open my $h, "<:raw", $file or do { print "cannot read $file\n"; exit 3 }; my $c = <$h>; close $h;
open my $b, "<:raw", $bf; my $before = <$b> // ""; open my $a, "<:raw", $af; my $after = <$a> // "";
if (length($before) == 0) { print "empty before text\n"; exit 3 }
if ($before eq $after) { print "after equals before\n"; exit 3 }
my ($cnt, $p) = (0, 0); while (($p = index($c, $before, $p)) >= 0) { $cnt++; $p += length($before) }
if ($cnt == 0) { print "before text not found\n"; exit 3 }
if ($cnt > 1) { print "before text occurs $cnt times; make it unique\n"; exit 3 }
substr($c, index($c, $before), length($before)) = $after;
open my $o, ">:raw", $file or exit 3; print $o $c; close $o;' "$file" "$M/$n/before" "$M/$n/after" 2>&1); then
out=NOAPPLY
elif git diff --quiet -- "$file"; then
out=NOAPPLY; detail="applied, but git sees no change in $file"
else
out=''
if [ -n "${BUILD:-}" ]; then
rc=$(run "$BUILD" "$M/run.log")
if [ "$rc" = 124 ]; then out=TIMEOUT; detail="build exceeded ${tmo}s"
elif [ "$rc" != 0 ]; then out=UNVIABLE; detail="build exit $rc"; fi
fi
if [ -z "$out" ]; then
rc=$(run "$t" "$M/run.log")
if [ "$rc" = 0 ]; then out=SURVIVED; detail="tests passed with the mutation applied: $t"
elif [ "$rc" = 124 ]; then out=TIMEOUT; detail="test exceeded ${tmo}s: $t"
else
names=$(tp_scan "$M/run.log" | grep '^FAILED ' | awk '!s[$0]++' | sed 's/^FAILED //')
if [ -n "$names" ]; then out=CAUGHT; detail=$(printf '%s\n' "$names" | head -n 1); k=$(printf '%s\n' "$names" | wc -l); [ "$k" -le 1 ] || detail="$detail (+$((k - 1)) more)"
else out=ERROR; detail="exit $rc, no failing test parsed: $(tail -n 1 "$M/run.log" | cut -c1-120)"; fi
fi
fi
fi
printf '%s | %s | %s\n' "$label" "$out" "$detail" >>"$M/rows"
git checkout -q -- "$file"
if ! git diff --quiet -- "$file"; then echo "YGG-SUMMARY: RESTORE FAILED for $file after $label; stopping"; cat "$M/rows" | sed 's/^/YGG-SUMMARY: /'; exit 2; fi
done
tally() { grep -c " | $1 | " "$M/rows" || true; }
bad=$(grep -vc -E ' \| (CAUGHT|UNVIABLE) \| ' "$M/rows" || true)
{
echo "MUTATION $count mutants: caught=$(tally CAUGHT) survived=$(tally SURVIVED) unviable=$(tally UNVIABLE) timeout=$(tally TIMEOUT) noapply=$(tally NOAPPLY) error=$(tally ERROR) (baseline green, tests ran)"
grep -v ' | CAUGHT | ' "$M/rows"
grep ' | CAUGHT | ' "$M/rows"
} | sed 's/^/YGG-SUMMARY: /'
[ "$bad" -eq 0 ]