SUMMARY=<log> keeps the full log and prints a bounded verdict summary; wait <log> blocks at most 270 s and returns it once. mutation-table.sh runs hand mutants in one verify job and returns a kill table. testparse.sh is the one parser for both. Fixtures: real cargo and mutation logs from Yggdrasil (tiny-crate), plus hand-written dotnet, vitest, pytest, exit-0-but-failed and no-verdict logs. Every commit here builds; verified live on Yggdrasil. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
167 lines
8.5 KiB
Shell
167 lines
8.5 KiB
Shell
#!/usr/bin/env bash
|
|
# Hand-mutant kill table: run a list of mutants against a revision on
|
|
# Yggdrasil, in ONE verify job, and get one table back.
|
|
#
|
|
# tools/mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]
|
|
# workdir directory inside the repo where the commands run (e.g.
|
|
# packages/cultcache-rs; CultLib has no Cargo.toml at its root).
|
|
# env TEST default test command for mutants with no `test:` line
|
|
# BUILD optional compile-only command; when it fails the mutant
|
|
# is UNVIABLE. Without it a mutant that does not compile
|
|
# has no failing test and reads ERROR, never CAUGHT.
|
|
# MUT_TIMEOUT seconds per build or test run (default 600)
|
|
# everything ygg-verify.sh reads (SUMMARY=<log>, TIMEOUT, MEM, ...)
|
|
# Typical, from an agent: launch as ONE background shell with
|
|
# SUMMARY=<scratch>/mut1.log TEST='cargo test' tools/mutation-table.sh ...
|
|
# then `tools/stopgap/ygg-verify.sh wait <scratch>/mut1.log` in the foreground.
|
|
#
|
|
# Mutants file (plain text; the text before the first `===` is ignored):
|
|
# === label-unique-in-the-file
|
|
# file: src/lib.rs (relative to workdir)
|
|
# test: cargo test add_ (optional; else $TEST)
|
|
# --- before
|
|
# a + b (exact text, may span lines; must occur ONCE)
|
|
# --- after
|
|
# a - b
|
|
#
|
|
# Each mutant is applied in the container's throwaway clone and reverted from
|
|
# git before the next, so no tree you care about is touched. The output is one
|
|
# row per mutant: label | outcome | killing test or detail. Outcomes:
|
|
# CAUGHT the test command failed AND a failing test name was parsed from it
|
|
# SURVIVED the test command passed with the mutation applied
|
|
# UNVIABLE BUILD failed
|
|
# TIMEOUT a run hit MUT_TIMEOUT (not a kill)
|
|
# NOAPPLY before text absent, ambiguous, or equal to after (no mutation made)
|
|
# ERROR the test command failed but no failing test was parsed (a build
|
|
# error without BUILD, a crash): not a kill
|
|
# Before any mutant, every distinct test command runs UNMUTATED. If one fails or
|
|
# runs zero tests the job aborts with `BASELINE RED`, because a red or empty
|
|
# baseline would make every mutant look killed or every one survive.
|
|
# Exit 0 means every mutant was CAUGHT or UNVIABLE and nothing else happened.
|
|
#
|
|
# DELETION LINE: this runner replaces the per-agent runmut/mutrun/strykrun
|
|
# scripts. It dies with tools/stopgap/ygg-verify.sh when `idunn verify` ships a
|
|
# typed mutation step. It reads tools/testparse.sh for failure names, the same
|
|
# parser the summary uses.
|
|
set -euo pipefail
|
|
|
|
if [ "${1:-}" != --in-container ]; then
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
repo=${1:?usage: mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]}
|
|
rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.}
|
|
[ -r "$mfile" ] || { echo "mutation-table: cannot read $mfile" >&2; exit 2; }
|
|
stage=$(mktemp -d)
|
|
trap 'rm -rf "$stage"' EXIT
|
|
cp "$here/testparse.sh" "$here/mutation-table.sh" "$stage/"
|
|
tr -d '\r' <"$mfile" >"$stage/mutants.txt"
|
|
bundle=$(tar -C "$stage" -cz testparse.sh mutation-table.sh mutants.txt | base64 | tr -d '\n')
|
|
cmd=$(printf 'mkdir -p /tmp/mt && echo %s | base64 -d | tar -xz -C /tmp/mt && WORKDIR=%q BUILD=%q TEST=%q MUT_TIMEOUT=%q bash /tmp/mt/mutation-table.sh --in-container' \
|
|
"$bundle" "$workdir" "${BUILD:-}" "${TEST:-}" "${MUT_TIMEOUT:-600}")
|
|
"$here/stopgap/ygg-verify.sh" "$repo" "$rev" "$image" "$cmd"
|
|
exit $?
|
|
fi
|
|
|
|
# ---- container side ---------------------------------------------------------
|
|
set +e
|
|
# shellcheck source=testparse.sh
|
|
. /tmp/mt/testparse.sh
|
|
cd "/src/${WORKDIR:-.}" || { echo "YGG-SUMMARY: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
|
|
M=/tmp/mt/m; rm -rf "$M"; mkdir -p "$M"
|
|
tmo=${MUT_TIMEOUT:-600}
|
|
|
|
if ! perl -e '
|
|
my ($mf, $out) = @ARGV; my ($n, $sec, %c, %seen) = (0, "");
|
|
open my $f, "<", $mf or die "cannot open $mf\n";
|
|
sub flush {
|
|
return unless exists $c{label};
|
|
for my $k (qw(file before after)) { die "mutant $c{label}: no $k\n" unless defined $c{$k} }
|
|
my $d = "$out/$n"; mkdir $d;
|
|
for my $k (qw(label file test before after)) {
|
|
my $v = join("\n", ref $c{$k} ? @{$c{$k}} : ($c{$k} // ""));
|
|
open my $o, ">:raw", "$d/$k" or die; print $o $v; close $o;
|
|
}
|
|
}
|
|
while (<$f>) {
|
|
chomp;
|
|
if (/^=== (.+?)\s*$/) { flush(); die "duplicate label $1\n" if $seen{$1}++; %c = (label => $1); $n++; $sec = ""; next }
|
|
next unless exists $c{label};
|
|
if ($sec eq "" && /^file: (.+?)\s*$/) { $c{file} = $1; next }
|
|
if ($sec eq "" && /^test: (.+?)\s*$/) { $c{test} = $1; next }
|
|
if (/^--- (before|after)\s*$/) { $sec = $1; $c{$sec} = []; next }
|
|
push @{$c{$sec}}, $_ if $sec;
|
|
}
|
|
flush(); die "no mutants in $mf\n" unless $n;
|
|
print "$n\n";' /tmp/mt/mutants.txt "$M" >"$M/count" 2>"$M/err"; then
|
|
echo "YGG-SUMMARY: BAD MUTANT FILE: $(cat "$M/err")"; exit 2
|
|
fi
|
|
count=$(cat "$M/count")
|
|
|
|
# run <command> <logfile>: prints the exit status; 124 means timeout
|
|
run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; }
|
|
|
|
declare -A based
|
|
for n in $(seq 1 "$count"); do
|
|
t=$(cat "$M/$n/test"); t=${t:-$TEST}
|
|
[ -n "$t" ] || { echo "YGG-SUMMARY: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; }
|
|
[ -z "${based[$t]:-}" ] || continue
|
|
based[$t]=1
|
|
rc=$(run "$t" "$M/base.log")
|
|
scan=$(tp_scan "$M/base.log")
|
|
tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}')
|
|
if [ "$rc" != 0 ]; then
|
|
echo "YGG-SUMMARY: BASELINE RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed 's/^/YGG-SUMMARY: /'; exit 2
|
|
fi
|
|
if [ "$tests" -eq 0 ]; then
|
|
echo "YGG-SUMMARY: BASELINE RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2
|
|
fi
|
|
done
|
|
|
|
: >"$M/rows"
|
|
for n in $(seq 1 "$count"); do
|
|
label=$(cat "$M/$n/label"); file=$(cat "$M/$n/file"); t=$(cat "$M/$n/test"); t=${t:-$TEST}
|
|
out=''; detail=''
|
|
if ! detail=$(perl -e '
|
|
local $/; my ($file, $bf, $af) = @ARGV;
|
|
open my $h, "<:raw", $file or do { print "cannot read $file\n"; exit 3 }; my $c = <$h>; close $h;
|
|
open my $b, "<:raw", $bf; my $before = <$b> // ""; open my $a, "<:raw", $af; my $after = <$a> // "";
|
|
if (length($before) == 0) { print "empty before text\n"; exit 3 }
|
|
if ($before eq $after) { print "after equals before\n"; exit 3 }
|
|
my ($cnt, $p) = (0, 0); while (($p = index($c, $before, $p)) >= 0) { $cnt++; $p += length($before) }
|
|
if ($cnt == 0) { print "before text not found\n"; exit 3 }
|
|
if ($cnt > 1) { print "before text occurs $cnt times; make it unique\n"; exit 3 }
|
|
substr($c, index($c, $before), length($before)) = $after;
|
|
open my $o, ">:raw", $file or exit 3; print $o $c; close $o;' "$file" "$M/$n/before" "$M/$n/after" 2>&1); then
|
|
out=NOAPPLY
|
|
elif git diff --quiet -- "$file"; then
|
|
out=NOAPPLY; detail="applied, but git sees no change in $file"
|
|
else
|
|
out=''
|
|
if [ -n "${BUILD:-}" ]; then
|
|
rc=$(run "$BUILD" "$M/run.log")
|
|
if [ "$rc" = 124 ]; then out=TIMEOUT; detail="build exceeded ${tmo}s"
|
|
elif [ "$rc" != 0 ]; then out=UNVIABLE; detail="build exit $rc"; fi
|
|
fi
|
|
if [ -z "$out" ]; then
|
|
rc=$(run "$t" "$M/run.log")
|
|
if [ "$rc" = 0 ]; then out=SURVIVED; detail="tests passed with the mutation applied: $t"
|
|
elif [ "$rc" = 124 ]; then out=TIMEOUT; detail="test exceeded ${tmo}s: $t"
|
|
else
|
|
names=$(tp_scan "$M/run.log" | grep '^FAILED ' | awk '!s[$0]++' | sed 's/^FAILED //')
|
|
if [ -n "$names" ]; then out=CAUGHT; detail=$(printf '%s\n' "$names" | head -n 1); k=$(printf '%s\n' "$names" | wc -l); [ "$k" -le 1 ] || detail="$detail (+$((k - 1)) more)"
|
|
else out=ERROR; detail="exit $rc, no failing test parsed: $(tail -n 1 "$M/run.log" | cut -c1-120)"; fi
|
|
fi
|
|
fi
|
|
fi
|
|
printf '%s | %s | %s\n' "$label" "$out" "$detail" >>"$M/rows"
|
|
git checkout -q -- "$file"
|
|
if ! git diff --quiet -- "$file"; then echo "YGG-SUMMARY: RESTORE FAILED for $file after $label; stopping"; cat "$M/rows" | sed 's/^/YGG-SUMMARY: /'; exit 2; fi
|
|
done
|
|
|
|
tally() { grep -c " | $1 | " "$M/rows" || true; }
|
|
bad=$(grep -vc -E ' \| (CAUGHT|UNVIABLE) \| ' "$M/rows" || true)
|
|
{
|
|
echo "MUTATION $count mutants: caught=$(tally CAUGHT) survived=$(tally SURVIVED) unviable=$(tally UNVIABLE) timeout=$(tally TIMEOUT) noapply=$(tally NOAPPLY) error=$(tally ERROR) (baseline green, tests ran)"
|
|
grep -v ' | CAUGHT | ' "$M/rows"
|
|
grep ' | CAUGHT | ' "$M/rows"
|
|
} | sed 's/^/YGG-SUMMARY: /'
|
|
[ "$bad" -eq 0 ]
|