313 lines
18 KiB
Shell
313 lines
18 KiB
Shell
#!/usr/bin/env bash
|
|
# Hand-mutant kill table: run a list of mutants against a revision on
|
|
# Yggdrasil, in ONE verify job, and get one table back.
|
|
#
|
|
# tools/mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]
|
|
# workdir directory inside the repo where the commands run (e.g.
|
|
# packages/cultcache-rs; CultLib has no Cargo.toml at its root).
|
|
# env TEST default test command for mutants with no `test:` line
|
|
# BUILD optional compile-only command, run on the UNMUTATED
|
|
# tree first (a red or missing build aborts like
|
|
# BASELINE RED); when it fails on a mutant the mutant is
|
|
# UNVIABLE. Without it a mutant that does not compile
|
|
# has no failing test and reads ERROR, never CAUGHT.
|
|
# MUT_TIMEOUT seconds per build or test run (default 600)
|
|
# TIMEOUT seconds for the WHOLE job (ygg-verify's own cap;
|
|
# default 3600 here, stated in the START line). Size it
|
|
# to mutants x (build + test) plus the baselines and one
|
|
# snapshot commit (about 20 s per 400 MB of build output).
|
|
# everything else ygg-verify.sh reads (SUMMARY=<log>, MEM, ...)
|
|
# Typical, from an agent: launch as ONE background shell with
|
|
# SUMMARY=<fresh scratch>/mut1.log TEST='cargo test' tools/mutation-table.sh ...
|
|
# then `tools/stopgap/ygg-verify.sh wait <scratch>/mut1.log` in the foreground,
|
|
# with a Bash tool timeout of at least 280000 ms.
|
|
#
|
|
# Mutants file (plain text; the text before the first `===` is ignored):
|
|
# === label-unique-in-the-file (any text; `|` is allowed)
|
|
# file: src/lib.rs (relative to workdir)
|
|
# test: cargo test add_ (optional; else $TEST)
|
|
# --- before
|
|
# a + b (exact text, may span lines; must occur ONCE;
|
|
# LF text also matches a CRLF file, a file
|
|
# with mixed endings may read NOAPPLY)
|
|
# --- after
|
|
# a - b
|
|
#
|
|
# ISOLATION. Docker owns the per-mutant snapshot; no script restores a tree. The
|
|
# job (tools/stopgap/ygg-verify.sh RUN_ON_HOST=1) copies the revision into ONE
|
|
# baseline container's own filesystem, runs the baselines there (so target/,
|
|
# node_modules and every other build or test cache exist), and `docker commit`s
|
|
# that container to a throwaway image. Each mutant then runs in a fresh
|
|
# `docker run --rm` container from that image, sequentially, under the same
|
|
# CPUS/MEM caps and nice. Everything a mutant's build or test writes, tracked or
|
|
# ignored, lives in that container's copy-on-write layer and is gone when it
|
|
# exits, so no run can decide the next one. The image is removed when the job
|
|
# ends, including by timeout or kill. No container gains privilege. What is NOT
|
|
# isolated: the shared registry, nuget and sccache volumes (compiler caches keyed
|
|
# by input hash) and anything a test sends to the network.
|
|
# The outcome of each mutant container travels as its exit status (10 CAUGHT,
|
|
# 11 SURVIVED, 12 UNVIABLE, 13 TIMEOUT, 14 NOAPPLY, 15 ERROR); any other status
|
|
# (OOM kill, watchdog kill, docker failure) is an ERROR row.
|
|
#
|
|
# One row prints as each mutant finishes, so a job timeout keeps the finished rows.
|
|
# Row (a `YGG-SUMMARY/$YGG_NONCE: ` line, plain `YGG-SUMMARY: ` when no nonce is
|
|
# set): [i/N] label | outcome | killing test or detail. The outcome is a field of
|
|
# its own (the exit status), tallied as such, never read back from the rendered
|
|
# row. Outcomes:
|
|
# CAUGHT the test command failed AND a failing test name was parsed from it
|
|
# SURVIVED the test command passed with the mutation applied
|
|
# UNVIABLE BUILD failed, or only a test FILE failed (vitest `FAIL <file> [ <file> ]`,
|
|
# jest `FAIL <file>` + `Test suite failed to run`, see tools/testparse.sh
|
|
# LOADFAIL) with no test name: nothing ran, not a kill
|
|
# TIMEOUT a run hit MUT_TIMEOUT (not a kill)
|
|
# NOAPPLY before text absent, ambiguous, or equal to after (no mutation made)
|
|
# ERROR the test command failed but no failing test was parsed (a build
|
|
# error without BUILD, a crash): not a kill
|
|
# Before any mutant, every distinct test command and BUILD run UNMUTATED. If one
|
|
# fails or a test command runs zero tests the job aborts with `BASELINE RED` or
|
|
# `BASELINE RAN ZERO TESTS`, because a red or empty baseline would make every
|
|
# mutant look killed or every one survive. Then the same unmutated run repeats in
|
|
# a fresh container from the snapshot (the state every mutant starts from); red
|
|
# there aborts with `SNAPSHOT RED` and no row counted: a test that cannot run
|
|
# twice (it creates a directory, claims a port) passes the baseline and fails
|
|
# every mutant, which would read as every mutant CAUGHT.
|
|
# Exit 0 means every mutant was CAUGHT or UNVIABLE and nothing else happened.
|
|
#
|
|
# DELETION LINE: this runner replaces the per-agent runmut/mutrun/strykrun
|
|
# scripts. It dies with tools/stopgap/ygg-verify.sh when `idunn verify` ships a
|
|
# typed mutation step. It reads tools/testparse.sh for failure names, the same
|
|
# parser the summary uses.
|
|
#
|
|
# Modes (this one file, three places): no flag = launcher on the workstation;
|
|
# --host = the driver on the Yggdrasil host, started by ygg-verify.sh RUN_ON_HOST=1;
|
|
# --baseline / --control / --mutant N = the work inside a container.
|
|
set -euo pipefail
|
|
|
|
SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" # ygg-verify.sh carries only lines tagged with its per-run nonce
|
|
# outcome <-> exit status of a --mutant container
|
|
OUTCOMES=(CAUGHT SURVIVED UNVIABLE TIMEOUT NOAPPLY ERROR); CODE0=10
|
|
|
|
# ---- launcher (workstation) -------------------------------------------------
|
|
if [ "${1:-}" != --host ] && [ "${1:-}" != --baseline ] && [ "${1:-}" != --mutant ] && [ "${1:-}" != --control ]; then
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
repo=${1:?usage: mutation-table.sh <local-repo> <rev> <image> <mutants-file> [workdir]}
|
|
rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.}
|
|
[ -r "$mfile" ] || { echo "mutation-table: cannot read $mfile" >&2; exit 2; }
|
|
stage=$(mktemp -d)
|
|
trap 'rm -rf "$stage"' EXIT
|
|
cp "$here/testparse.sh" "$here/mutation-table.sh" "$stage/"
|
|
tr -d '\r' <"$mfile" >"$stage/mutants.txt"
|
|
bundle=$(tar -C "$stage" -cz testparse.sh mutation-table.sh mutants.txt | base64 | tr -d '\n')
|
|
# The driver runs on the host: it unpacks the bundle into its own directory (mounted read-only into every container) and removes it when done.
|
|
cmd=$(printf 'd=$(mktemp -d) && echo %s | base64 -d | tar -xz -C "$d" && WORKDIR=%q BUILD=%q TEST=%q MUT_TIMEOUT=%q JOB_TIMEOUT=%q MT_BUNDLE="$d" bash "$d/mutation-table.sh" --host' \
|
|
"$bundle" "$workdir" "${BUILD:-}" "${TEST:-}" "${MUT_TIMEOUT:-600}" "${TIMEOUT:-3600}")
|
|
RUN_ON_HOST=1 TIMEOUT=${TIMEOUT:-3600} "$here/stopgap/ygg-verify.sh" "$repo" "$rev" "$image" "$cmd"
|
|
exit $?
|
|
fi
|
|
|
|
# ---- driver (Yggdrasil host, inside the job's slot) --------------------------
|
|
if [ "$1" = --host ]; then
|
|
: "${YGG_WORK:?not started by ygg-verify.sh RUN_ON_HOST=1}" "${YGG_IMAGE:?}" "${YGG_CNAME:?}" "${YGG_LABEL:?}"
|
|
bundle=${MT_BUNDLE:?}
|
|
set +e # every docker step is checked by hand: a mutant container exits non-zero by design
|
|
base="$YGG_CNAME-base"; snap="eureka-mt-$YGG_CNAME:snap"
|
|
# A TERM waits for the foreground container (bounded by MUT_TIMEOUT); the stopgap's watchdog kills every
|
|
# job-labelled container before it signals this driver, so cleanup has no running mutant to stop.
|
|
cleanup() {
|
|
sudo docker rm -f "$base" >/dev/null 2>&1 || true
|
|
sudo docker rmi -f "$snap" >/dev/null 2>&1 || true
|
|
[ -z "$bundle" ] || [ "$bundle" = / ] || rm -rf -- "$bundle"
|
|
}
|
|
trap cleanup EXIT; trap 'exit 143' TERM INT HUP
|
|
# Every container carries the job label (the stopgap's watchdog kills by it) and the same caps.
|
|
common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}"
|
|
-v "$bundle:/mt:ro" -v /etc/machine-id:/etc/machine-id:ro
|
|
-v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache
|
|
-e BASH_ENV=/tmp/.ygg-bash-env -e CARGO_HOME=/cargo-home
|
|
-e "WORKDIR=${WORKDIR:-.}" -e "BUILD=${BUILD:-}" -e "TEST=${TEST:-}" -e "MUT_TIMEOUT=${MUT_TIMEOUT:-600}" -e "JOB_TIMEOUT=${JOB_TIMEOUT:-}"
|
|
-e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*')
|
|
[ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE")
|
|
# DOCKER_ARGS is word-split on purpose: it is a flag string.
|
|
# shellcheck disable=SC2086
|
|
drun() { sudo nice -n 10 docker run "$@"; }
|
|
pre="printf 'set -o pipefail\n' >/tmp/.ygg-bash-env"
|
|
|
|
sudo docker rm -f "$base" >/dev/null 2>&1 || true
|
|
# shellcheck disable=SC2086
|
|
drun --name "$base" "${common[@]}" -v "$YGG_WORK:/seed:ro" -w /src ${DOCKER_ARGS:-} "$YGG_IMAGE" \
|
|
bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --baseline"
|
|
rc=$?
|
|
[ "$rc" = 0 ] || exit "$rc" # the baseline container printed why
|
|
count=$(sudo docker cp "$base:/tmp/mtm/count" - | tar -xO)
|
|
case $count in ''|*[!0-9]*) echo "$SUMTAG: cannot read the mutant count from the baseline container"; exit 2 ;; esac
|
|
t0=$(date +%s)
|
|
sudo docker commit "$base" "$snap" >/dev/null || { echo "$SUMTAG: docker commit failed"; exit 2; }
|
|
sudo docker rm -f "$base" >/dev/null
|
|
echo "$SUMTAG: SNAPSHOT committed in $(( $(date +%s) - t0 ))s; a control run of the unmutated snapshot comes first, then each mutant in a fresh container from it"
|
|
# A test that passed once and cannot pass again (it creates a directory, claims a port, appends to a
|
|
# fixture) is green in the baseline and red in every mutant's container, which would read as a fully
|
|
# killed suite. The control is the baseline's tests run once more from the snapshot, unmutated.
|
|
# shellcheck disable=SC2086
|
|
drun --rm --name "$YGG_CNAME-ctl" "${common[@]}" -w /src ${DOCKER_ARGS:-} "$snap" \
|
|
bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --control"
|
|
rc=$?
|
|
[ "$rc" = 0 ] || { echo "$SUMTAG: the unmutated snapshot is red though the baseline was green, so no mutant result would mean anything; no rows counted"; exit "$rc"; }
|
|
|
|
declare -A tally; for o in "${OUTCOMES[@]}"; do tally[$o]=0; done
|
|
for n in $(seq 1 "$count"); do
|
|
if [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; then
|
|
echo "$SUMTAG: JOB KILLED by the watchdog before mutant $n of $count; the finished rows above stand"; exit 124
|
|
fi
|
|
# shellcheck disable=SC2086
|
|
drun --rm --name "$YGG_CNAME-m$n" "${common[@]}" -w /src ${DOCKER_ARGS:-} "$snap" \
|
|
bash -o pipefail -c "$pre; bash /mt/mutation-table.sh --mutant $n"
|
|
rc=$?
|
|
if [ "$rc" -ge "$CODE0" ] && [ "$rc" -lt $((CODE0 + ${#OUTCOMES[@]})) ]; then
|
|
o=${OUTCOMES[$((rc - CODE0))]}
|
|
else
|
|
o=ERROR; echo "$SUMTAG: [$n/$count] mutant $n | ERROR | the mutant container exited $rc without a row (137 is a kill or OOM)"
|
|
fi
|
|
tally[$o]=$(( ${tally[$o]} + 1 ))
|
|
done
|
|
echo "$SUMTAG: MUTATION $count mutants: caught=${tally[CAUGHT]} survived=${tally[SURVIVED]} unviable=${tally[UNVIABLE]} timeout=${tally[TIMEOUT]} noapply=${tally[NOAPPLY]} error=${tally[ERROR]} (baseline green, tests ran)"
|
|
[ $(( ${tally[CAUGHT]} + ${tally[UNVIABLE]} )) -eq "$count" ]
|
|
exit $?
|
|
fi
|
|
|
|
# ---- inside a container -----------------------------------------------------
|
|
set +e
|
|
# overridable only so tools/test-tools.sh can run this half against a mock
|
|
SRC=${MT_SRC:-/src}; MT_DIR=${MT_DIR:-/mt}; M=${MT_M:-/tmp/mtm}; SEED=${MT_SEED:-/seed}
|
|
# shellcheck source=testparse.sh
|
|
. "$MT_DIR/testparse.sh"
|
|
tmo=${MUT_TIMEOUT:-600}
|
|
# run <command> <logfile>: prints the exit status; 124 means timeout
|
|
run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; }
|
|
|
|
# unmutated <BASELINE|SNAPSHOT> <count>: run every distinct test command, then BUILD, on the tree as it
|
|
# stands. A red or empty run aborts the job (exit 2) under that name: a red baseline would make every
|
|
# mutant look killed, an empty one every mutant survive, and a SNAPSHOT that is red where the BASELINE
|
|
# was green (a test that cannot run twice) would do the first again.
|
|
unmutated() {
|
|
local what=$1 count=$2 n t rc scan tests; declare -A based
|
|
for n in $(seq 1 "$count"); do
|
|
t=$(cat "$M/$n/test"); t=${t:-$TEST}
|
|
[ -n "$t" ] || { echo "$SUMTAG: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; }
|
|
[ -z "${based[$t]:-}" ] || continue
|
|
based[$t]=1
|
|
rc=$(run "$t" "$M/base.log")
|
|
scan=$(tp_scan "$M/base.log")
|
|
tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}')
|
|
if [ "$rc" != 0 ]; then
|
|
echo "$SUMTAG: $what RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2
|
|
fi
|
|
if [ "$tests" -eq 0 ]; then
|
|
echo "$SUMTAG: $what RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2
|
|
fi
|
|
done
|
|
# BUILD runs unmutated too. A mistyped or missing build command fails every
|
|
# mutant, and a table of all-UNVIABLE rows reads as a fully killed suite.
|
|
if [ -n "${BUILD:-}" ]; then
|
|
rc=$(run "$BUILD" "$M/base.log")
|
|
if [ "$rc" != 0 ]; then
|
|
echo "$SUMTAG: $what RED: BUILD failed on the unmutated tree (exit $rc; 124 is a timeout): $BUILD"
|
|
tail -n 15 "$M/base.log" | sed "s|^|$SUMTAG: |"; exit 2
|
|
fi
|
|
fi
|
|
}
|
|
|
|
if [ "$1" = --baseline ]; then
|
|
# The revision lives in this container's own filesystem, so `docker commit` keeps it.
|
|
if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "$SUMTAG: cannot copy $SEED into $SRC"; exit 2; }; fi
|
|
cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
|
|
rm -rf "$M"; mkdir -p "$M"
|
|
if ! perl -e '
|
|
my ($mf, $out) = @ARGV; my ($n, $sec, %c, %seen) = (0, "");
|
|
open my $f, "<", $mf or die "cannot open $mf\n";
|
|
sub flush {
|
|
return unless exists $c{label};
|
|
for my $k (qw(file before after)) { die "mutant $c{label}: no $k\n" unless defined $c{$k} }
|
|
my $d = "$out/$n"; mkdir $d;
|
|
for my $k (qw(label file test before after)) {
|
|
my $v = join("\n", ref $c{$k} ? @{$c{$k}} : ($c{$k} // ""));
|
|
open my $o, ">:raw", "$d/$k" or die; print $o $v; close $o;
|
|
}
|
|
}
|
|
while (<$f>) {
|
|
chomp;
|
|
if (/^=== (.+?)\s*$/) { flush(); die "duplicate label $1\n" if $seen{$1}++; %c = (label => $1); $n++; $sec = ""; next }
|
|
next unless exists $c{label};
|
|
if ($sec eq "" && /^file: (.+?)\s*$/) { $c{file} = $1; next }
|
|
if ($sec eq "" && /^test: (.+?)\s*$/) { $c{test} = $1; next }
|
|
if (/^--- (before|after)\s*$/) { $sec = $1; $c{$sec} = []; next }
|
|
push @{$c{$sec}}, $_ if $sec;
|
|
}
|
|
flush(); die "no mutants in $mf\n" unless $n;
|
|
print "$n\n";' "$MT_DIR/mutants.txt" "$M" >"$M/count" 2>"$M/err"; then
|
|
echo "$SUMTAG: BAD MUTANT FILE: $(cat "$M/err")"; exit 2
|
|
fi
|
|
count=$(cat "$M/count")
|
|
echo "$SUMTAG: START $count mutants; per-run timeout ${tmo}s; whole-job timeout ${JOB_TIMEOUT:-unknown}s; one row per mutant follows as it finishes, so a timeout keeps the finished rows"
|
|
unmutated BASELINE "$count"
|
|
# A baseline that rewrote a tracked file would hand every mutant a changed tree
|
|
# and make `git diff` blind to the mutation; untracked and ignored output stays.
|
|
git -C "$SRC" reset -q --hard 2>/dev/null
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$1" = --control ]; then
|
|
# One unmutated run from the committed snapshot, the state every mutant starts from.
|
|
cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
|
|
unmutated SNAPSHOT "$(cat "$M/count")"
|
|
exit 0
|
|
fi
|
|
|
|
# --mutant <n>
|
|
n=${2:?mutant number}
|
|
cd "$SRC/${WORKDIR:-.}" || { echo "$SUMTAG: BAD WORKDIR ${WORKDIR:-.}"; exit 2; }
|
|
count=$(cat "$M/count")
|
|
label=$(cat "$M/$n/label"); file=$(cat "$M/$n/file"); t=$(cat "$M/$n/test"); t=${t:-$TEST}
|
|
out=''; detail=''
|
|
if ! detail=$(perl -e '
|
|
local $/; my ($file, $bf, $af) = @ARGV;
|
|
open my $h, "<:raw", $file or do { print "cannot read $file\n"; exit 3 }; my $c = <$h>; close $h;
|
|
open my $b, "<:raw", $bf; my $before = <$b> // ""; open my $a, "<:raw", $af; my $after = <$a> // "";
|
|
if (length($before) == 0) { print "empty before text\n"; exit 3 }
|
|
if ($before eq $after) { print "after equals before\n"; exit 3 }
|
|
# a CRLF file matches multi-line before text written with LF
|
|
if ($c =~ /\r\n/ && $before !~ /\r/) { s/\n/\r\n/g for $before, $after }
|
|
my ($cnt, $p) = (0, 0); while (($p = index($c, $before, $p)) >= 0) { $cnt++; $p += length($before) }
|
|
if ($cnt == 0) { print "before text not found\n"; exit 3 }
|
|
if ($cnt > 1) { print "before text occurs $cnt times; make it unique\n"; exit 3 }
|
|
substr($c, index($c, $before), length($before)) = $after;
|
|
open my $o, ">:raw", $file or exit 3; print $o $c; close $o;' "$file" "$M/$n/before" "$M/$n/after" 2>&1); then
|
|
out=NOAPPLY
|
|
elif git diff --quiet -- "$file"; then
|
|
out=NOAPPLY; detail="applied, but git sees no change in $file"
|
|
else
|
|
out=''; detail=''
|
|
if [ -n "${BUILD:-}" ]; then
|
|
rc=$(run "$BUILD" "$M/run.log")
|
|
if [ "$rc" = 124 ]; then out=TIMEOUT; detail="build exceeded ${tmo}s"
|
|
elif [ "$rc" != 0 ]; then out=UNVIABLE; detail="build exit $rc"; fi
|
|
fi
|
|
if [ -z "$out" ]; then
|
|
rc=$(run "$t" "$M/run.log")
|
|
if [ "$rc" = 0 ]; then out=SURVIVED; detail="tests passed with the mutation applied: $t"
|
|
elif [ "$rc" = 124 ]; then out=TIMEOUT; detail="test exceeded ${tmo}s: $t"
|
|
else
|
|
scan=$(tp_scan "$M/run.log")
|
|
named=$(printf '%s\n' "$scan" | grep '^FAILED ' | awk '!s[$0]++' | sed 's/^FAILED //')
|
|
loads=$(printf '%s\n' "$scan" | grep '^LOADFAIL ' | awk '!s[$0]++' | sed 's/^LOADFAIL //')
|
|
if [ -n "$named" ]; then
|
|
out=CAUGHT; detail=$(printf '%s\n' "$named" | head -n 1)
|
|
k=$(printf '%s\n' "$named" | wc -l); [ "$k" -le 1 ] || detail="$detail (+$((k - 1)) more)"
|
|
elif [ -n "$loads" ]; then out=UNVIABLE; detail="a test file failed to load, no test ran: $(printf '%s\n' "$loads" | head -n 1)"
|
|
else out=ERROR; detail="exit $rc, no failing test parsed: $(tail -n 1 "$M/run.log" | cut -c1-120)"; fi
|
|
fi
|
|
fi
|
|
fi
|
|
detail=${detail//$'\n'/ }
|
|
printf '%s: [%s/%s] %s | %s | %s\n' "$SUMTAG" "$n" "$count" "${label//|/\\|}" "$out" "$detail"
|
|
for i in "${!OUTCOMES[@]}"; do [ "${OUTCOMES[$i]}" = "$out" ] && exit $((CODE0 + i)); done
|
|
exit 1
|