389 lines
22 KiB
Shell
389 lines
22 KiB
Shell
#!/usr/bin/env bash
|
|
# cargo-mutants on Yggdrasil for a revision range: ONE verify job, per-file scoped
|
|
# tests, survivors confirmed on the full suite, sharded over free slots, one merged
|
|
# survivor list. The one owner of "run cargo-mutants" (no brief, no job script runs
|
|
# an inline `cargo mutants`).
|
|
#
|
|
# SUMMARY=<log> tools/cargo-mutants.sh <local-repo> <rev> <base> [workdir]
|
|
# workdir directory inside the repo where cargo runs (a package or workspace
|
|
# member, e.g. crates/mind-body; CultLib has no Cargo.toml at its root).
|
|
# The mutated range is `git diff --relative <base> <rev>` taken at the
|
|
# WORKSPACE ROOT: cargo-mutants matches --in-diff paths, -f paths and
|
|
# prints mutant names relative to the workspace root, not to workdir.
|
|
# env FILTERS '<path>=<filter>,...' overrides a file's derived test filter
|
|
# (path as the mutant names print it); '-' = full suite
|
|
# TEST_ARGS harness args, e.g. '--skip flaky_one' (a flaky test would
|
|
# make a surviving mutant read as killed in one run)
|
|
# NAMES local file of mutant names (one per line, copied from a
|
|
# previous MISSED/TIMEOUT list): skips stage 1 and the listing
|
|
# and runs only those names on the full suite. This is the rerun
|
|
# after killing survivors with tests.
|
|
# SHARDS (3) JOBS (2) MIN_TEST_TIMEOUT (20) CONFIRM_TIMEOUT_MULT (5)
|
|
# TIMEOUT seconds for the whole job (default 10800)
|
|
# everything else tools/stopgap/ygg-verify.sh reads (MEM, CPUS, ...)
|
|
# Typical, from an agent: launch as ONE background shell with a fresh log, then
|
|
# tools/stopgap/ygg-verify.sh wait <log> (Bash tool timeout >= 280000 ms)
|
|
# Run it ONCE per pass, at the end, on the final spelling of the code. Chasing
|
|
# survivors is a NAMES rerun, never a second whole-range sweep.
|
|
#
|
|
# STAGES (all inside the one job and the one host driver):
|
|
# plan one container writes the range diff, lists every mutant in it, derives each
|
|
# touched file's test filter and counts the tests that filter selects
|
|
# (`cargo test -- <filter> --list`, one build for all files).
|
|
# stage 1 each touched file whose filter selects tests runs its mutants against that
|
|
# file's tests only (`cargo mutants -f <file> -- <filter>`). The filter is
|
|
# derived from the path: the module path after the last src/ (a/mod.rs ->
|
|
# `a::`, a/b.rs -> `a::b::`); lib.rs, main.rs, bin/ and files outside src/
|
|
# have none. FILTERS overrides. Stage 1 decides only CAUGHT (its own green
|
|
# filtered baseline guards false kills) and UNVIABLE (a build fact). Its
|
|
# MISSED and TIMEOUT are never reported: a mutant that only another module's
|
|
# tests or tests/ kill survives a filtered run. A file whose filter selects 0
|
|
# tests (tests=0) never runs stage 1: all its mutants go to stage 2.
|
|
# stage 2 every name stage 1 did not settle (survivors, files without a filter,
|
|
# zero-test files, or NAMES) runs on the full suite in cargo-mutants' own
|
|
# baseline, sharded over the containers. Stage 2 ALWAYS runs: that baseline is
|
|
# the only full-suite check, so a red suite is a red job even when stage 1
|
|
# settled every mutant (one name then rides along; its outcome is dropped).
|
|
# The test timeout is max(MIN_TEST_TIMEOUT, CONFIRM_TIMEOUT_MULT x baseline);
|
|
# the generous default keeps a slow-but-finishing test from flipping a
|
|
# MISSED mutant to TIMEOUT with the host's load. Its outcome is final.
|
|
# merge each name counted once; MISSED and TIMEOUT are stage-2 outcomes only.
|
|
# Containers per stage: min(SHARDS, SLOTS, units, 1 + free slots now), units being the
|
|
# stage's files (stage 1) or names (stage 2). The job holds one slot (ygg-verify's);
|
|
# the driver leases further slot locks with flock -n, never waits for one, takes none
|
|
# while /proc/pressure/io full avg60 > 20, and releases them when each stage ends. A
|
|
# killed job (YGG_KILLED) starts nothing further.
|
|
#
|
|
# OUTPUT: the YGG-SUMMARY/$YGG_NONCE lines ygg-verify's summary carries; per-container
|
|
# logs stay on the host and go with the job.
|
|
# FILTER <file> <filter> tests=<n> mutants=<m> one per stage-1 file (zero-test first)
|
|
# MUTANTS tested=N caught=C missed=M timeout=T unviable=U | stage1 filtered=F
|
|
# survivors=S confirmed_caught=X | containers=K | plan=..s stage1=..s/<k1> stage2=..s/<k2> | <m>m
|
|
# MISSED <name> / TIMEOUT <name> final outcomes, at most 50
|
|
# ERROR ... a red baseline, a container that failed, mutants with no outcome:
|
|
# the job exits non-zero and prints no MUTANTS line
|
|
# Exit 0 when every stage ran on a green baseline; survivors do not fail the job (the
|
|
# score is not a gate). cargo-mutants exits 2 (missed) and 3 (timeouts) normally; 4 is a
|
|
# red baseline; anything else is an error.
|
|
#
|
|
# DELETION LINE: this file dies with tools/stopgap/ygg-verify.sh when `idunn verify`
|
|
# ships a typed mutation step. Modes in this one file: no flag = launcher on the
|
|
# workstation; --host = the driver on the Yggdrasil host (ygg-verify.sh RUN_ON_HOST=1);
|
|
# --plan / --item = the work inside a container.
|
|
set -euo pipefail
|
|
|
|
SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}"
|
|
sum() { printf '%s: %s\n' "$SUMTAG" "$*"; }
|
|
|
|
# ---- --escape: mutant names on stdin -> one anchored -F pattern per line ----------------------
|
|
# Only regex metacharacters are escaped, so names holding * + ( ) | stay literal; ^...$ pins the whole name.
|
|
if [ "${1:-}" = --escape ]; then
|
|
exec perl -pe 'chomp; s/([\\.+*?()|\[\]{}^\$])/\\$1/g; $_ = "^$_\$\n"'
|
|
fi
|
|
|
|
# ---- launcher (workstation) -------------------------------------------------
|
|
if [ "${1:-}" != --host ] && [ "${1:-}" != --plan ] && [ "${1:-}" != --item ]; then
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
repo=${1:?usage: cargo-mutants.sh <local-repo> <rev> <base> [workdir]}
|
|
rev=${2:?revision}; base=${3:?base revision}; workdir=${4:-.}
|
|
: "${SHARDS:=3}" "${JOBS:=2}" "${MIN_TEST_TIMEOUT:=20}" "${CONFIRM_TIMEOUT_MULT:=5}" "${TIMEOUT:=10800}"
|
|
for v in SHARDS JOBS MIN_TEST_TIMEOUT TIMEOUT; do
|
|
case ${!v} in ''|*[!0-9]*|0) echo "cargo-mutants: $v must be a positive integer" >&2; exit 2 ;; esac
|
|
done
|
|
case $CONFIRM_TIMEOUT_MULT in ''|*[!0-9.]*|.*|*.|*.*.*) echo "cargo-mutants: CONFIRM_TIMEOUT_MULT must be a positive number" >&2; exit 2 ;; esac
|
|
if [ -n "${FILTERS:-}" ]; then
|
|
old=$IFS; IFS=,
|
|
for f in $FILTERS; do
|
|
case $f in ?*=?*) ;; *) IFS=$old; echo "cargo-mutants: FILTERS entries are <path>=<filter>" >&2; exit 2 ;; esac
|
|
done
|
|
IFS=$old
|
|
fi
|
|
sha_rev=$(git -C "$repo" rev-parse --verify "$rev^{commit}") || { echo "cargo-mutants: unknown revision" >&2; exit 2; }
|
|
sha_base=$(git -C "$repo" rev-parse --verify "$base^{commit}") || { echo "cargo-mutants: unknown base revision" >&2; exit 2; }
|
|
git -C "$repo" merge-base --is-ancestor "$sha_base" "$sha_rev" || { echo "cargo-mutants: base is not an ancestor of rev (the verify mirror carries only rev's history)" >&2; exit 2; }
|
|
stage=$(mktemp -d)
|
|
trap 'rm -rf "$stage"' EXIT
|
|
cp "$here/cargo-mutants.sh" "$stage/"
|
|
files=(cargo-mutants.sh)
|
|
if [ -n "${NAMES:-}" ]; then
|
|
[ -r "$NAMES" ] || { echo "cargo-mutants: cannot read the NAMES file" >&2; exit 2; }
|
|
tr -d '\r' <"$NAMES" >"$stage/names.txt"; files+=(names.txt)
|
|
fi
|
|
bundle=$(tar -C "$stage" -cz "${files[@]}" | base64 | tr -d '\n')
|
|
# The driver runs on the host: it unpacks the bundle into its own directory (mounted read-only into every container) and removes it when done.
|
|
cmd=$(printf 'd=$(mktemp -d) && echo %s | base64 -d | tar -xz -C "$d" && WORKDIR=%q BASE=%q FILTERS=%q TEST_ARGS=%q SHARDS=%q JOBS=%q MIN_TEST_TIMEOUT=%q CONFIRM_TIMEOUT_MULT=%q CM_NAMES=%q CM_DIR="$d" bash "$d/cargo-mutants.sh" --host' \
|
|
"$bundle" "$workdir" "$sha_base" "${FILTERS:-}" "${TEST_ARGS:-}" "$SHARDS" "$JOBS" "$MIN_TEST_TIMEOUT" "$CONFIRM_TIMEOUT_MULT" "${NAMES:+1}")
|
|
RUN_ON_HOST=1 TIMEOUT=$TIMEOUT "$here/stopgap/ygg-verify.sh" "$repo" "$sha_rev" rust "$cmd"
|
|
exit $?
|
|
fi
|
|
|
|
# ---- driver (Yggdrasil host, inside the job's slot) --------------------------
|
|
if [ "$1" = --host ]; then
|
|
: "${YGG_WORK:?not started by ygg-verify.sh RUN_ON_HOST=1}" "${YGG_IMAGE:?}" "${YGG_CNAME:?}" "${YGG_LABEL:?}" "${YGG_SLOTS:?}" "${YGG_ROOT:?}"
|
|
D=${CM_DIR:?}
|
|
set +e # every docker step is checked by hand: a container exits non-zero by design
|
|
export LC_ALL=C
|
|
t0=$(date +%s)
|
|
LEASE_FDS=()
|
|
unlease() { local lf; for lf in "${LEASE_FDS[@]}"; do exec {lf}>&-; done; LEASE_FDS=(); }
|
|
cleanup() {
|
|
local p; for p in $(jobs -p); do kill "$p" 2>/dev/null; done
|
|
unlease
|
|
# containers write as root
|
|
[ -z "$D" ] || [ "$D" = / ] || sudo rm -rf -- "$D"
|
|
}
|
|
trap cleanup EXIT; trap 'exit 143' TERM INT HUP
|
|
killed() { [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; }
|
|
now() { date +%s; }
|
|
# The shared volume is CARGO_HOME itself so sibling containers share cargo's package-cache lock; a registry-only mount left
|
|
# the lock in each container's layer and concurrent unpacks removed each other's crate directories (2026-11-03).
|
|
# Every container carries the job label (the stopgap's watchdog kills by it) and the same caps.
|
|
common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}"
|
|
-v "$YGG_WORK:/seed:ro" -v "$D:/cm:ro" -v /etc/machine-id:/etc/machine-id:ro
|
|
-v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache
|
|
-e BASH_ENV=/tmp/.ygg-bash-env -e CARGO_HOME=/cargo-home
|
|
-e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "FILTERS=${FILTERS:-}" -e "CM_NAMES=${CM_NAMES:-}" -e "JOBS=$JOBS"
|
|
-e "MIN_TEST_TIMEOUT=$MIN_TEST_TIMEOUT" -e "CONFIRM_TIMEOUT_MULT=$CONFIRM_TIMEOUT_MULT"
|
|
-e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*')
|
|
[ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE")
|
|
pre="printf 'set -o pipefail\n' >/tmp/.ygg-bash-env"
|
|
# drun <name> <out dir> [-e flags]: one container (the half named by $MODE), its output to <out>/stdout, its
|
|
# status in <out>/hostrc. A killed job starts nothing (125). DOCKER_ARGS is a flag string, split on purpose.
|
|
# shellcheck disable=SC2086
|
|
drun() {
|
|
local name=$1 out=$2 rc; shift 2
|
|
killed && return 125
|
|
mkdir -p "$out"
|
|
sudo nice -n 10 docker run --rm --name "$name" "${common[@]}" -v "$out:/out" "$@" -w /src ${DOCKER_ARGS:-} "$YGG_IMAGE" \
|
|
bash -o pipefail -c "$pre; bash /cm/cargo-mutants.sh $MODE" >"$out/stdout" 2>&1
|
|
rc=$?; echo "$rc" >"$out/hostrc"; return "$rc"
|
|
}
|
|
killed_exit() { sum "JOB KILLED by the watchdog before the run finished; no totals"; exit 124; }
|
|
mkdir -p "$D/items" "$D/plan" "$D/q" "$D/m"
|
|
|
|
# ---- plan
|
|
tp=$(now)
|
|
MODE=--plan drun "$YGG_CNAME-plan" "$D/plan"
|
|
rc=$?
|
|
killed && killed_exit
|
|
if [ "$rc" != 0 ]; then
|
|
sum "ERROR the plan container exited $rc: the range diff or the mutant listing failed"
|
|
tail -n 6 "$D/plan/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR |"
|
|
exit 2
|
|
fi
|
|
tp=$(( $(now) - tp ))
|
|
if [ "${CM_NAMES:-}" = 1 ]; then sort -u "$D/names.txt" >"$D/m/universe"
|
|
else sort -u "$D/plan/names.txt" >"$D/m/universe"; fi
|
|
for f in s1.caught s1.unviable s1.surv s1.names; do : >"$D/m/$f"; done
|
|
|
|
# lease <units>: the containers of a stage = min(SHARDS, SLOTS, units, 1 + free slots now); sets K and holds the leases
|
|
lease() {
|
|
local want=$1 i lf psi
|
|
[ "$want" -le "$SHARDS" ] || want=$SHARDS
|
|
[ "$want" -le "$YGG_SLOTS" ] || want=$YGG_SLOTS
|
|
K=1; [ "$want" -gt 1 ] || return 0
|
|
psi=$(awk '$1 == "full" { for (i = 2; i <= NF; i++) if ($i ~ /^avg60=/) { sub("avg60=", "", $i); print $i } }' "${CM_PSI:-/proc/pressure/io}" 2>/dev/null)
|
|
if awk -v p="${psi:-0}" 'BEGIN { exit !(p > 20) }'; then return 0; fi
|
|
for i in $(seq 1 "$YGG_SLOTS"); do
|
|
[ "$K" -lt "$want" ] || break
|
|
exec {lf}>"$YGG_ROOT/slot-$i.lock"
|
|
if flock -n "$lf"; then LEASE_FDS+=("$lf"); K=$((K + 1)); else exec {lf}>&-; fi
|
|
done
|
|
}
|
|
# run_stage <stage> <count> <fn>: K workers claim items 1..count by mkdir, in order (largest first)
|
|
run_stage() {
|
|
local stage=$1 count=$2 fn=$3 w pids=()
|
|
for w in $(seq 1 "$K"); do
|
|
( for lf in "${LEASE_FDS[@]}"; do exec {lf}>&-; done # the lease belongs to the driver, never to a child (2026-09-23 inherited-fd scar)
|
|
i=0
|
|
while [ "$i" -lt "$count" ]; do
|
|
i=$((i + 1))
|
|
killed && break
|
|
[ ! -e "$D/failed" ] || break
|
|
mkdir "$D/q/$stage-$i" 2>/dev/null || continue
|
|
"$fn" "$i"
|
|
done ) &
|
|
pids+=($!)
|
|
done
|
|
wait "${pids[@]}"
|
|
}
|
|
container_error() { # container_error <label> <out dir>
|
|
local rc; rc=$(cat "$2/hostrc" 2>/dev/null)
|
|
sum "ERROR $1 container exited ${rc:-?} (4 is a red baseline: the unmutated tests failed there)"
|
|
tail -n 25 "$2/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR |"
|
|
}
|
|
|
|
# ---- stage 1: files whose filter selects tests (the plan container derived each file's filter and counted its tests)
|
|
S1_FILE=(); S1_FILTER=(); s1n=0
|
|
if [ "${CM_NAMES:-}" != 1 ]; then
|
|
while IFS=$'\t' read -r cnt file f t; do # files by mutant count, largest first
|
|
[ "$t" != 0 ] || continue # a filter that selects no test settles nothing: straight to stage 2
|
|
s1n=$((s1n + 1)); S1_FILE[$s1n]=$file; S1_FILTER[$s1n]=$f
|
|
awk -F: -v f="$file" '$1 == f' "$D/m/universe" >"$D/items/s1-$s1n.names"
|
|
done <"$D/plan/filters.txt"
|
|
fi
|
|
s1_item() {
|
|
local n=$1 rc
|
|
MODE=--item drun "$YGG_CNAME-s1-$n" "$D/items/s1-$n/out" -e CM_STAGE=1 -e "CM_FILE=${S1_FILE[$n]}" -e "CM_FILTER=${S1_FILTER[$n]}"
|
|
rc=$?
|
|
if [ "$rc" != 0 ] && ! killed; then touch "$D/failed"; fi
|
|
}
|
|
K1=0; ts1=0
|
|
if [ "$s1n" -gt 0 ]; then
|
|
ts1=$(now)
|
|
lease "$s1n"; K1=$K
|
|
run_stage s1 "$s1n" s1_item
|
|
unlease
|
|
killed && killed_exit
|
|
ts1=$(( $(now) - ts1 ))
|
|
bad=0
|
|
for n in $(seq 1 "$s1n"); do
|
|
o="$D/items/s1-$n/out"
|
|
if [ "$(cat "$o/hostrc" 2>/dev/null)" != 0 ]; then
|
|
[ -e "$D/q/s1-$n" ] || continue # never started: the job is already red for another file
|
|
container_error "stage 1 (${S1_FILE[$n]})" "$o"; bad=1; continue
|
|
fi
|
|
done
|
|
if [ "$bad" != 0 ] || [ -e "$D/failed" ]; then exit 1; fi
|
|
fi
|
|
# FILTER lines: the anomalies (0 tests) first, then the rest, at most 12 in all
|
|
if [ "${CM_NAMES:-}" != 1 ]; then
|
|
{ awk -F'\t' '$4 == 0' "$D/plan/filters.txt"; awk -F'\t' '$4 != 0' "$D/plan/filters.txt"; } |
|
|
awk -F'\t' '{ if (NR <= 12) printf "FILTER %s %s tests=%s mutants=%s\n", $2, $3, $4, $1; else h++ } END { if (h) printf "FILTER ... %d more files\n", h }' |
|
|
while IFS= read -r l; do sum "$l"; done
|
|
fi
|
|
for n in $(seq 1 "$s1n"); do
|
|
o="$D/items/s1-$n/out"
|
|
cat "$D/items/s1-$n.names" >>"$D/m/s1.names"
|
|
cat "$o/caught.txt" >>"$D/m/s1.caught"; cat "$o/unviable.txt" >>"$D/m/s1.unviable"
|
|
cat "$o/missed.txt" "$o/timeout.txt" >>"$D/m/s1.surv"
|
|
done
|
|
# only names of this run's range count; stage 1 settles caught and unviable, nothing else
|
|
for f in s1.caught s1.unviable s1.surv s1.names; do
|
|
sort -u "$D/m/$f" | comm -12 - "$D/m/universe" >"$D/m/$f.t" && mv "$D/m/$f.t" "$D/m/$f"
|
|
done
|
|
cat "$D/m/s1.caught" "$D/m/s1.unviable" | sort -u >"$D/m/s1.settled"
|
|
comm -23 "$D/m/universe" "$D/m/s1.settled" >"$D/s2.txt"
|
|
# Stage 2 always runs, so cargo-mutants' full-suite baseline always does (cargo-mutants runs no baseline for an
|
|
# empty selection). With every name settled it carries one name; the merge drops that name's outcome.
|
|
if [ -s "$D/s2.txt" ]; then bash "$D/cargo-mutants.sh" --escape <"$D/s2.txt" >"$D/s2.re"
|
|
else head -n 1 "$D/m/universe" | bash "$D/cargo-mutants.sh" --escape >"$D/s2.re"; fi
|
|
|
|
# ---- stage 2: the full suite on everything stage 1 did not settle
|
|
s2n=$(wc -l <"$D/s2.re" | tr -d " ")
|
|
K2=0; ts2=0
|
|
for l in caught missed timeout unviable; do : >"$D/m/s2.$l"; done
|
|
s2_shard() {
|
|
local k=$1 rc
|
|
MODE=--item drun "$YGG_CNAME-s2-$k" "$D/items/s2-$k/out" -e CM_STAGE=2 -e "CM_SHARD=$((k - 1))/$K2"
|
|
rc=$?
|
|
if [ "$rc" != 0 ] && ! killed; then touch "$D/failed"; fi
|
|
}
|
|
if [ "$s2n" -gt 0 ]; then
|
|
ts2=$(now)
|
|
lease "$s2n"; K2=$K
|
|
run_stage s2 "$K2" s2_shard
|
|
unlease
|
|
killed && killed_exit
|
|
ts2=$(( $(now) - ts2 ))
|
|
bad=0
|
|
for k in $(seq 1 "$K2"); do
|
|
o="$D/items/s2-$k/out"
|
|
if [ "$(cat "$o/hostrc" 2>/dev/null)" != 0 ]; then
|
|
[ -e "$D/q/s2-$k" ] || continue
|
|
container_error "stage 2 (shard $k of $K2)" "$o"; bad=1; continue
|
|
fi
|
|
for l in caught missed timeout unviable; do cat "$o/$l.txt" >>"$D/m/s2.$l"; done
|
|
done
|
|
if [ "$bad" != 0 ] || [ -e "$D/failed" ]; then exit 1; fi
|
|
fi
|
|
|
|
# ---- merge: a name's outcome is stage 2's where it ran, else stage 1's; each name once
|
|
for l in caught missed timeout unviable; do
|
|
sort -u "$D/m/s2.$l" | comm -12 - "$D/s2.txt" >"$D/m/s2.$l.t" && mv "$D/m/s2.$l.t" "$D/m/s2.$l"
|
|
done
|
|
cat "$D/m/s2.caught" "$D/m/s2.missed" "$D/m/s2.timeout" "$D/m/s2.unviable" | sort -u >"$D/m/s2.all"
|
|
nout=$(comm -23 "$D/s2.txt" "$D/m/s2.all" | wc -l | tr -d ' ')
|
|
if [ "$nout" != 0 ]; then
|
|
sum "ERROR $nout mutants selected for the full-suite run have no outcome in any shard (a name unknown to cargo-mutants at this revision, or a shard that died)"
|
|
exit 1
|
|
fi
|
|
C=$(cat "$D/m/s1.caught" "$D/m/s2.caught" | sort -u | wc -l | tr -d ' ')
|
|
U=$(cat "$D/m/s1.unviable" "$D/m/s2.unviable" | sort -u | wc -l | tr -d ' ')
|
|
M=$(wc -l <"$D/m/s2.missed" | tr -d ' '); T=$(wc -l <"$D/m/s2.timeout" | tr -d ' ')
|
|
S=$(wc -l <"$D/m/s1.surv" | tr -d ' '); F=$(wc -l <"$D/m/s1.names" | tr -d ' ')
|
|
X=$(comm -12 "$D/m/s1.surv" "$D/m/s2.caught" | wc -l | tr -d ' ')
|
|
K=$K1; [ "$K2" -le "$K" ] || K=$K2
|
|
sum "MUTANTS tested=$((C + M + T + U)) caught=$C missed=$M timeout=$T unviable=$U | stage1 filtered=$F survivors=$S confirmed_caught=$X | containers=$K | plan=${tp}s stage1=${ts1}s/${K1} stage2=${ts2}s/${K2} | $(awk -v e="$(( $(now) - t0 ))" 'BEGIN { printf "%.1f", e / 60 }')m"
|
|
{ sed 's/^/MISSED /' "$D/m/s2.missed"; sed 's/^/TIMEOUT /' "$D/m/s2.timeout"; } >"$D/m/surv.txt"
|
|
total=$(wc -l <"$D/m/surv.txt" | tr -d ' ')
|
|
head -n 50 "$D/m/surv.txt" | while IFS= read -r l; do sum "$l"; done
|
|
if [ "$total" -gt 50 ]; then
|
|
sum "... $((total - 50)) more MISSED/TIMEOUT names; they follow in the job log as cm-survivor lines"
|
|
tail -n +51 "$D/m/surv.txt" | sed 's/^/cm-survivor: /'
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# ---- inside a container -----------------------------------------------------
|
|
set +e
|
|
# overridable only so tools/test-tools.sh can run this half against a mock
|
|
SRC=${CM_SRC:-/src}; CM=${CM_DIR:-/cm}; OUT=${CM_OUT:-/out}; SEED=${CM_SEED:-/seed}
|
|
if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "cannot copy $SEED into $SRC"; exit 2; }; fi
|
|
cd "$SRC/${WORKDIR:-.}" || { echo "BAD WORKDIR"; exit 2; }
|
|
set -f; read -ra targs <<<"${TEST_ARGS:-}"; set +f
|
|
if [ "$1" = --plan ]; then
|
|
wsroot=$(dirname "$(cargo locate-project --workspace --message-format plain)") || { echo "cargo locate-project failed"; exit 2; }
|
|
git -C "$wsroot" diff --relative "$BASE" HEAD >"$OUT/d" || { echo "git diff of the range failed"; exit 2; }
|
|
cargo mutants --list --in-diff "$OUT/d" >"$OUT/names.txt" 2>"$OUT/list.err" || { echo "cargo mutants --list failed"; tail -n 5 "$OUT/list.err"; exit 2; }
|
|
: >"$OUT/filters.txt"
|
|
[ "$CM_NAMES" != 1 ] || exit 0
|
|
# stage 1's plan: each touched file's test filter and how many tests it selects (0 = nothing for stage 1 to run)
|
|
declare -A fover
|
|
if [ -n "${FILTERS:-}" ]; then
|
|
IFS=, read -ra fl <<<"$FILTERS"
|
|
for e in "${fl[@]}"; do fover[${e%%=*}]=${e#*=}; done
|
|
fi
|
|
derive_filter() { # derive_filter <path>: the test filter of a file, nothing for the full suite
|
|
local p=$1 rest
|
|
if [ -n "${fover[$p]+x}" ]; then [ "${fover[$p]}" = - ] || echo "${fover[$p]}"; return 0; fi
|
|
case $p in */src/*.rs|src/*.rs) ;; *) return 0 ;; esac
|
|
rest=${p##*src/}; rest=${rest%.rs}
|
|
case $rest in lib|main|bin/*|*/bin/*) return 0 ;; esac
|
|
rest=${rest%/mod}
|
|
echo "${rest//\//::}::"
|
|
}
|
|
built=0
|
|
while IFS=$'\t' read -r cnt file; do # files by mutant count, largest first
|
|
f=$(derive_filter "$file")
|
|
[ -n "$f" ] || continue
|
|
if [ "$built" = 0 ]; then
|
|
cargo test --no-run >"$OUT/build.log" 2>&1 || { echo "cargo test --no-run failed"; tail -n 5 "$OUT/build.log"; exit 2; }
|
|
built=1
|
|
fi
|
|
t=$(cargo test -- "$f" "${targs[@]}" --list 2>/dev/null | awk '/^[0-9]+ tests?, / { s += $1 } END { print s + 0 }')
|
|
printf '%s\t%s\t%s\t%s\n' "$cnt" "$file" "$f" "$t" >>"$OUT/filters.txt"
|
|
done < <(awk -F: '{ c[$1]++ } END { for (f in c) print c[f] "\t" f }' "$OUT/names.txt" | sort -t "$(printf '\t')" -k1,1nr -k2)
|
|
exit 0
|
|
fi
|
|
# --item: stage 1 (CM_FILE with its CM_FILTER) or stage 2 (shard CM_SHARD of the names in $CM/s2.re)
|
|
o=${CM_TMP:-/tmp}/cmo
|
|
args=(mutants --in-diff "$CM/plan/d" -o "$o" --jobs "$JOBS" --minimum-test-timeout "$MIN_TEST_TIMEOUT")
|
|
tail_args=()
|
|
if [ "$CM_STAGE" = 1 ]; then
|
|
args+=(-f "$CM_FILE")
|
|
tail_args=(-- "$CM_FILTER"); [ "${#targs[@]}" = 0 ] || tail_args+=(-- "${targs[@]}")
|
|
else
|
|
mapfile -t res <"$CM/s2.re"
|
|
for r in "${res[@]}"; do args+=(-F "$r"); done
|
|
args+=(--timeout-multiplier "$CONFIRM_TIMEOUT_MULT")
|
|
case $CM_SHARD in */1) ;; *) args+=(--shard "$CM_SHARD") ;; esac
|
|
[ "${#targs[@]}" = 0 ] || tail_args=(-- -- "${targs[@]}")
|
|
fi
|
|
cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1
|
|
rc=$?
|
|
for l in caught missed timeout unviable; do cp "$o/mutants.out/$l.txt" "$OUT/$l.txt" 2>/dev/null || : >"$OUT/$l.txt"; done
|
|
echo "$rc" >"$OUT/rc"
|
|
tail -n 30 "$OUT/cargo.log"
|
|
# 0 clean, 2 missed and 3 timeouts are normal; 4 (red baseline) and everything else are errors
|
|
case $rc in 0|2|3) exit 0 ;; *) exit "$rc" ;; esac
|