cargo-mutants.sh: stage 2 always runs the full-suite baseline; zero-test files found at plan time; 5x timeout with a floor; --escape pinned
- full-baseline-skipped: stage 2 runs even when stage 1 settled every name (one name rides along; cargo-mutants runs no baseline for an empty selection), so a red suite is a red job - timeout-mult-splits-missed: CONFIRM_TIMEOUT_MULT defaults to 5 and stage 2 gets --minimum-test-timeout like stage 1 - escaper-pipe-untested: the escaper is the --escape mode, run by the driver and by tests/test-tools.sh with each metacharacter - zero-test-stage1-wasted: the plan container derives each file's filter and counts its tests (cargo test -- <filter> --list); 0 tests goes straight to stage 2; the container-side baseline.log count is deleted Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a2bfb59ab3
commit
0bd4aeca1b
3 changed files with 118 additions and 63 deletions
|
|
@ -18,7 +18,7 @@
|
|||
# previous MISSED/TIMEOUT list): skips stage 1 and the listing
|
||||
# and runs only those names on the full suite. This is the rerun
|
||||
# after killing survivors with tests.
|
||||
# SHARDS (3) JOBS (2) MIN_TEST_TIMEOUT (20) CONFIRM_TIMEOUT_MULT (2)
|
||||
# SHARDS (3) JOBS (2) MIN_TEST_TIMEOUT (20) CONFIRM_TIMEOUT_MULT (5)
|
||||
# TIMEOUT seconds for the whole job (default 10800)
|
||||
# everything else tools/stopgap/ygg-verify.sh reads (MEM, CPUS, ...)
|
||||
# Typical, from an agent: launch as ONE background shell with a fresh log, then
|
||||
|
|
@ -27,20 +27,26 @@
|
|||
# survivors is a NAMES rerun, never a second whole-range sweep.
|
||||
#
|
||||
# STAGES (all inside the one job and the one host driver):
|
||||
# plan one container writes the range diff and lists every mutant in it.
|
||||
# stage 1 each touched file with a test filter runs its mutants against that
|
||||
# plan one container writes the range diff, lists every mutant in it, derives each
|
||||
# touched file's test filter and counts the tests that filter selects
|
||||
# (`cargo test -- <filter> --list`, one build for all files).
|
||||
# stage 1 each touched file whose filter selects tests runs its mutants against that
|
||||
# file's tests only (`cargo mutants -f <file> -- <filter>`). The filter is
|
||||
# derived from the path: the module path after the last src/ (a/mod.rs ->
|
||||
# `a::`, a/b.rs -> `a::b::`); lib.rs, main.rs, bin/ and files outside src/
|
||||
# have none. FILTERS overrides. Stage 1 decides only CAUGHT (its own green
|
||||
# filtered baseline guards false kills) and UNVIABLE (a build fact). Its
|
||||
# MISSED and TIMEOUT are never reported: a mutant that only another module's
|
||||
# tests or tests/ kill survives a filtered run. A filter whose baseline ran
|
||||
# 0 tests settles nothing and all its file's mutants go to stage 2.
|
||||
# tests or tests/ kill survives a filtered run. A file whose filter selects 0
|
||||
# tests (tests=0) never runs stage 1: all its mutants go to stage 2.
|
||||
# stage 2 every name stage 1 did not settle (survivors, files without a filter,
|
||||
# zero-test files, or NAMES) runs on the full suite in cargo-mutants' own
|
||||
# baseline, sharded over the containers, with the test timeout at
|
||||
# CONFIRM_TIMEOUT_MULT x that baseline. Its outcome is final.
|
||||
# baseline, sharded over the containers. Stage 2 ALWAYS runs: that baseline is
|
||||
# the only full-suite check, so a red suite is a red job even when stage 1
|
||||
# settled every mutant (one name then rides along; its outcome is dropped).
|
||||
# The test timeout is max(MIN_TEST_TIMEOUT, CONFIRM_TIMEOUT_MULT x baseline);
|
||||
# the generous default keeps a slow-but-finishing test from flipping a
|
||||
# MISSED mutant to TIMEOUT with the host's load. Its outcome is final.
|
||||
# merge each name counted once; MISSED and TIMEOUT are stage-2 outcomes only.
|
||||
# Containers per stage: min(SHARDS, SLOTS, units, 1 + free slots now), units being the
|
||||
# stage's files (stage 1) or names (stage 2). The job holds one slot (ygg-verify's);
|
||||
|
|
@ -69,12 +75,18 @@ set -euo pipefail
|
|||
SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}"
|
||||
sum() { printf '%s: %s\n' "$SUMTAG" "$*"; }
|
||||
|
||||
# ---- --escape: mutant names on stdin -> one anchored -F pattern per line ----------------------
|
||||
# Only regex metacharacters are escaped, so names holding * + ( ) | stay literal; ^...$ pins the whole name.
|
||||
if [ "${1:-}" = --escape ]; then
|
||||
exec perl -pe 'chomp; s/([\\.+*?()|\[\]{}^\$])/\\$1/g; $_ = "^$_\$\n"'
|
||||
fi
|
||||
|
||||
# ---- launcher (workstation) -------------------------------------------------
|
||||
if [ "${1:-}" != --host ] && [ "${1:-}" != --plan ] && [ "${1:-}" != --item ]; then
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
repo=${1:?usage: cargo-mutants.sh <local-repo> <rev> <base> [workdir]}
|
||||
rev=${2:?revision}; base=${3:?base revision}; workdir=${4:-.}
|
||||
: "${SHARDS:=3}" "${JOBS:=2}" "${MIN_TEST_TIMEOUT:=20}" "${CONFIRM_TIMEOUT_MULT:=2}" "${TIMEOUT:=10800}"
|
||||
: "${SHARDS:=3}" "${JOBS:=2}" "${MIN_TEST_TIMEOUT:=20}" "${CONFIRM_TIMEOUT_MULT:=5}" "${TIMEOUT:=10800}"
|
||||
for v in SHARDS JOBS MIN_TEST_TIMEOUT TIMEOUT; do
|
||||
case ${!v} in ''|*[!0-9]*|0) echo "cargo-mutants: $v must be a positive integer" >&2; exit 2 ;; esac
|
||||
done
|
||||
|
|
@ -128,7 +140,7 @@ if [ "$1" = --host ]; then
|
|||
-v "$YGG_WORK:/seed:ro" -v "$D:/cm:ro" -v /etc/machine-id:/etc/machine-id:ro
|
||||
-v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache
|
||||
-e BASH_ENV=/tmp/.ygg-bash-env
|
||||
-e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "JOBS=$JOBS"
|
||||
-e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "FILTERS=${FILTERS:-}" -e "CM_NAMES=${CM_NAMES:-}" -e "JOBS=$JOBS"
|
||||
-e "MIN_TEST_TIMEOUT=$MIN_TEST_TIMEOUT" -e "CONFIRM_TIMEOUT_MULT=$CONFIRM_TIMEOUT_MULT"
|
||||
-e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*')
|
||||
[ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE")
|
||||
|
|
@ -160,7 +172,7 @@ if [ "$1" = --host ]; then
|
|||
tp=$(( $(now) - tp ))
|
||||
if [ "${CM_NAMES:-}" = 1 ]; then sort -u "$D/names.txt" >"$D/m/universe"
|
||||
else sort -u "$D/plan/names.txt" >"$D/m/universe"; fi
|
||||
for f in s1.caught s1.unviable s1.surv s1.names s1.tests; do : >"$D/m/$f"; done
|
||||
for f in s1.caught s1.unviable s1.surv s1.names; do : >"$D/m/$f"; done
|
||||
|
||||
# lease <units>: the containers of a stage = min(SHARDS, SLOTS, units, 1 + free slots now); sets K and holds the leases
|
||||
lease() {
|
||||
|
|
@ -199,31 +211,14 @@ if [ "$1" = --host ]; then
|
|||
tail -n 5 "$2/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR |"
|
||||
}
|
||||
|
||||
# ---- stage 1: files with a filter
|
||||
declare -A fover
|
||||
if [ -n "${FILTERS:-}" ]; then
|
||||
IFS=, read -ra fl <<<"$FILTERS"
|
||||
for e in "${fl[@]}"; do fover[${e%%=*}]=${e#*=}; done
|
||||
fi
|
||||
derive_filter() { # derive_filter <path>: the test filter of a file, nothing for the full suite
|
||||
local p=$1 rest
|
||||
if [ -n "${fover[$p]+x}" ]; then [ "${fover[$p]}" = - ] || echo "${fover[$p]}"; return 0; fi
|
||||
case $p in */src/*.rs|src/*.rs) ;; *) return 0 ;; esac
|
||||
rest=${p##*src/}; rest=${rest%.rs}
|
||||
case $rest in lib|main|bin/*|*/bin/*) return 0 ;; esac
|
||||
rest=${rest%/mod}
|
||||
echo "${rest//\//::}::"
|
||||
}
|
||||
# ---- stage 1: files whose filter selects tests (the plan container derived each file's filter and counted its tests)
|
||||
S1_FILE=(); S1_FILTER=(); s1n=0
|
||||
if [ "${CM_NAMES:-}" != 1 ]; then
|
||||
# files by mutant count, largest first
|
||||
while read -r cnt file; do
|
||||
f=$(derive_filter "$file")
|
||||
[ -n "$f" ] || continue
|
||||
while IFS=$'\t' read -r cnt file f t; do # files by mutant count, largest first
|
||||
[ "$t" != 0 ] || continue # a filter that selects no test settles nothing: straight to stage 2
|
||||
s1n=$((s1n + 1)); S1_FILE[$s1n]=$file; S1_FILTER[$s1n]=$f
|
||||
awk -F: -v f="$file" '$1 == f' "$D/m/universe" >"$D/items/s1-$s1n.names"
|
||||
echo "$cnt" >"$D/items/s1-$s1n.count"
|
||||
done < <(awk -F: '{ c[$1]++ } END { for (f in c) print c[f], f }' "$D/m/universe" | sort -k1,1nr -k2)
|
||||
done <"$D/plan/filters.txt"
|
||||
fi
|
||||
s1_item() {
|
||||
local n=$1 rc
|
||||
|
|
@ -246,25 +241,16 @@ if [ "$1" = --host ]; then
|
|||
[ -e "$D/q/s1-$n" ] || continue # never started: the job is already red for another file
|
||||
container_error "stage 1 (${S1_FILE[$n]})" "$o"; bad=1; continue
|
||||
fi
|
||||
echo "$n $(cat "$o/tests" 2>/dev/null || echo 0)" >>"$D/m/s1.tests"
|
||||
done
|
||||
if [ "$bad" != 0 ] || [ -e "$D/failed" ]; then exit 1; fi
|
||||
fi
|
||||
# FILTER lines: the anomalies (0 tests) first, then the rest, at most 12 in all
|
||||
shown=0; hidden=0
|
||||
for pass in zero rest; do
|
||||
for n in $(seq 1 "$s1n"); do
|
||||
t=$(awk -v n="$n" '$1 == n { print $2 }' "$D/m/s1.tests"); t=${t:-0}
|
||||
if [ "$pass" = zero ] && [ "$t" != 0 ]; then continue; fi
|
||||
if [ "$pass" = rest ] && [ "$t" = 0 ]; then continue; fi
|
||||
if [ "$shown" -lt 12 ]; then sum "FILTER ${S1_FILE[$n]} ${S1_FILTER[$n]} tests=$t mutants=$(cat "$D/items/s1-$n.count")"; shown=$((shown + 1))
|
||||
else hidden=$((hidden + 1)); fi
|
||||
done
|
||||
done
|
||||
[ "$hidden" = 0 ] || sum "FILTER ... $hidden more files"
|
||||
if [ "${CM_NAMES:-}" != 1 ]; then
|
||||
{ awk -F'\t' '$4 == 0' "$D/plan/filters.txt"; awk -F'\t' '$4 != 0' "$D/plan/filters.txt"; } |
|
||||
awk -F'\t' '{ if (NR <= 12) printf "FILTER %s %s tests=%s mutants=%s\n", $2, $3, $4, $1; else h++ } END { if (h) printf "FILTER ... %d more files\n", h }' |
|
||||
while IFS= read -r l; do sum "$l"; done
|
||||
fi
|
||||
for n in $(seq 1 "$s1n"); do
|
||||
t=$(awk -v n="$n" '$1 == n { print $2 }' "$D/m/s1.tests"); t=${t:-0}
|
||||
[ "$t" != 0 ] || continue # a filter that ran no test settled nothing
|
||||
o="$D/items/s1-$n/out"
|
||||
cat "$D/items/s1-$n.names" >>"$D/m/s1.names"
|
||||
cat "$o/caught.txt" >>"$D/m/s1.caught"; cat "$o/unviable.txt" >>"$D/m/s1.unviable"
|
||||
|
|
@ -276,11 +262,13 @@ if [ "$1" = --host ]; then
|
|||
done
|
||||
cat "$D/m/s1.caught" "$D/m/s1.unviable" | sort -u >"$D/m/s1.settled"
|
||||
comm -23 "$D/m/universe" "$D/m/s1.settled" >"$D/s2.txt"
|
||||
# only regex metacharacters are escaped, so names holding * + ( ) | stay literal; ^...$ pins the whole name
|
||||
perl -pe 'chomp; s/([\\.+*?()|\[\]{}^\$])/\\$1/g; $_ = "^$_\$\n"' "$D/s2.txt" >"$D/s2.re"
|
||||
# Stage 2 always runs, so cargo-mutants' full-suite baseline always does (cargo-mutants runs no baseline for an
|
||||
# empty selection). With every name settled it carries one name; the merge drops that name's outcome.
|
||||
if [ -s "$D/s2.txt" ]; then bash "$D/cargo-mutants.sh" --escape <"$D/s2.txt" >"$D/s2.re"
|
||||
else head -n 1 "$D/m/universe" | bash "$D/cargo-mutants.sh" --escape >"$D/s2.re"; fi
|
||||
|
||||
# ---- stage 2: the full suite on everything stage 1 did not settle
|
||||
s2n=$(wc -l <"$D/s2.txt" | tr -d ' ')
|
||||
s2n=$(wc -l <"$D/s2.re" | tr -d " ")
|
||||
K2=0; ts2=0
|
||||
for l in caught missed timeout unviable; do : >"$D/m/s2.$l"; done
|
||||
s2_shard() {
|
||||
|
|
@ -341,19 +329,47 @@ set +e
|
|||
SRC=${CM_SRC:-/src}; CM=${CM_DIR:-/cm}; OUT=${CM_OUT:-/out}; SEED=${CM_SEED:-/seed}
|
||||
if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "cannot copy $SEED into $SRC"; exit 2; }; fi
|
||||
cd "$SRC/${WORKDIR:-.}" || { echo "BAD WORKDIR"; exit 2; }
|
||||
set -f; read -ra targs <<<"${TEST_ARGS:-}"; set +f
|
||||
if [ "$1" = --plan ]; then
|
||||
wsroot=$(dirname "$(cargo locate-project --workspace --message-format plain)") || { echo "cargo locate-project failed"; exit 2; }
|
||||
git -C "$wsroot" diff --relative "$BASE" HEAD >"$OUT/d" || { echo "git diff of the range failed"; exit 2; }
|
||||
cargo mutants --list --in-diff "$OUT/d" >"$OUT/names.txt" 2>"$OUT/list.err" || { echo "cargo mutants --list failed"; tail -n 5 "$OUT/list.err"; exit 2; }
|
||||
: >"$OUT/filters.txt"
|
||||
[ "$CM_NAMES" != 1 ] || exit 0
|
||||
# stage 1's plan: each touched file's test filter and how many tests it selects (0 = nothing for stage 1 to run)
|
||||
declare -A fover
|
||||
if [ -n "${FILTERS:-}" ]; then
|
||||
IFS=, read -ra fl <<<"$FILTERS"
|
||||
for e in "${fl[@]}"; do fover[${e%%=*}]=${e#*=}; done
|
||||
fi
|
||||
derive_filter() { # derive_filter <path>: the test filter of a file, nothing for the full suite
|
||||
local p=$1 rest
|
||||
if [ -n "${fover[$p]+x}" ]; then [ "${fover[$p]}" = - ] || echo "${fover[$p]}"; return 0; fi
|
||||
case $p in */src/*.rs|src/*.rs) ;; *) return 0 ;; esac
|
||||
rest=${p##*src/}; rest=${rest%.rs}
|
||||
case $rest in lib|main|bin/*|*/bin/*) return 0 ;; esac
|
||||
rest=${rest%/mod}
|
||||
echo "${rest//\//::}::"
|
||||
}
|
||||
built=0
|
||||
while IFS=$'\t' read -r cnt file; do # files by mutant count, largest first
|
||||
f=$(derive_filter "$file")
|
||||
[ -n "$f" ] || continue
|
||||
if [ "$built" = 0 ]; then
|
||||
cargo test --no-run >"$OUT/build.log" 2>&1 || { echo "cargo test --no-run failed"; tail -n 5 "$OUT/build.log"; exit 2; }
|
||||
built=1
|
||||
fi
|
||||
t=$(cargo test -- "$f" "${targs[@]}" --list 2>/dev/null | awk '/^[0-9]+ tests?, / { s += $1 } END { print s + 0 }')
|
||||
printf '%s\t%s\t%s\t%s\n' "$cnt" "$file" "$f" "$t" >>"$OUT/filters.txt"
|
||||
done < <(awk -F: '{ c[$1]++ } END { for (f in c) print c[f] "\t" f }' "$OUT/names.txt" | sort -t "$(printf '\t')" -k1,1nr -k2)
|
||||
exit 0
|
||||
fi
|
||||
# --item: stage 1 (CM_FILE with its CM_FILTER) or stage 2 (shard CM_SHARD of the names in $CM/s2.re)
|
||||
set -f; read -ra targs <<<"${TEST_ARGS:-}"; set +f
|
||||
o=${CM_TMP:-/tmp}/cmo
|
||||
args=(mutants --in-diff "$CM/plan/d" -o "$o" --jobs "$JOBS")
|
||||
args=(mutants --in-diff "$CM/plan/d" -o "$o" --jobs "$JOBS" --minimum-test-timeout "$MIN_TEST_TIMEOUT")
|
||||
tail_args=()
|
||||
if [ "$CM_STAGE" = 1 ]; then
|
||||
args+=(-f "$CM_FILE" --minimum-test-timeout "$MIN_TEST_TIMEOUT")
|
||||
args+=(-f "$CM_FILE")
|
||||
tail_args=(-- "$CM_FILTER"); [ "${#targs[@]}" = 0 ] || tail_args+=(-- "${targs[@]}")
|
||||
else
|
||||
mapfile -t res <"$CM/s2.re"
|
||||
|
|
@ -365,7 +381,6 @@ fi
|
|||
cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1
|
||||
rc=$?
|
||||
for l in caught missed timeout unviable; do cp "$o/mutants.out/$l.txt" "$OUT/$l.txt" 2>/dev/null || : >"$OUT/$l.txt"; done
|
||||
awk '/^running [0-9]+ tests?/ { s += $2 } END { print s + 0 }' "$o/mutants.out/log/baseline.log" >"$OUT/tests" 2>/dev/null || echo 0 >"$OUT/tests"
|
||||
echo "$rc" >"$OUT/rc"
|
||||
tail -n 30 "$OUT/cargo.log"
|
||||
# 0 clean, 2 missed and 3 timeouts are normal; 4 (red baseline) and everything else are errors
|
||||
|
|
|
|||
|
|
@ -1,20 +1,30 @@
|
|||
#!/usr/bin/env bash
|
||||
# Fake cargo for tools/test-tools.sh, only what tools/cargo-mutants.sh calls:
|
||||
# cargo locate-project --workspace --message-format plain
|
||||
# cargo test --no-run
|
||||
# cargo test -- <filter> [<harness args>] --list (prints "<n> tests, 0 benchmarks" per test binary)
|
||||
# cargo mutants [--list] --in-diff <diff> [-f <file>] [-F <re>]... [--shard k/n] [-o <dir>] [...] [-- <filter> [-- <harness args>]]
|
||||
# FAKE_CM_TABLE: tab-separated <mutant name> <outcome under a filtered run> <outcome under the full suite>
|
||||
# (caught|missed|timeout|unviable). A run with -F is the full suite; a run with -f and a filter is filtered.
|
||||
# FAKE_CM_TESTS: tab-separated <filter> <tests it runs> (default 1; the unfiltered suite runs 5)
|
||||
# FAKE_CM_TESTS: tab-separated <filter> <tests the filter selects> (default 1)
|
||||
# FAKE_CM_RED=1|2: the stage-1 or stage-2 run fails its baseline (exit 4, no output).
|
||||
# FAKE_CM_KILLS: a path a run creates (the watchdog's YGG_KILLED flag appearing mid-job).
|
||||
# FAKE_CM_LOG: appended one line per run (stage, filter, harness args, open fds from 10 up).
|
||||
# FAKE_WSROOT: the workspace root relative to the repository root, with a trailing slash ('' = the repository root).
|
||||
case ${1:-} in
|
||||
locate-project) echo "$(git rev-parse --show-toplevel)/${FAKE_WSROOT:-}Cargo.toml"; exit 0 ;;
|
||||
test)
|
||||
shift; for a; do [ "$a" != --no-run ] || exit 0; done
|
||||
[ "${1:-}" = -- ] || { echo "fake cargo test: unsupported arguments" >&2; exit 64; }
|
||||
shift; f=$1; shift; h=()
|
||||
for a; do [ "$a" = --list ] || h+=("$a"); done
|
||||
echo "count filter=$f harness=${h[*]}" >>"${FAKE_CM_LOG:?}"
|
||||
n=$(awk -F'\t' -v f="$f" '$1 == f { print $2 }' "${FAKE_CM_TESTS:-/dev/null}")
|
||||
printf '%s tests, 0 benchmarks\n0 tests, 0 benchmarks\n' "${n:-1}"; exit 0 ;;
|
||||
mutants) shift ;;
|
||||
*) echo "fake cargo: unsupported ${1:-}" >&2; exit 127 ;;
|
||||
esac
|
||||
list=0; diff=''; ffile=''; res=(); shard=''; out=''; tail=()
|
||||
list=0; mult=''; mintimeout=''; diff=''; ffile=''; res=(); shard=''; out=''; tail=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--list) list=1 ;;
|
||||
|
|
@ -23,7 +33,9 @@ while [ $# -gt 0 ]; do
|
|||
-F) res+=("$2"); shift ;;
|
||||
--shard) shard=$2; shift ;;
|
||||
-o) out=$2; shift ;;
|
||||
--jobs|--minimum-test-timeout|--timeout-multiplier) shift ;;
|
||||
--jobs) shift ;;
|
||||
--minimum-test-timeout) mintimeout=$2; shift ;;
|
||||
--timeout-multiplier) mult=$2; shift ;;
|
||||
--) shift; tail=("$@"); break ;;
|
||||
*) echo "fake cargo mutants: unknown argument $1" >&2; exit 64 ;;
|
||||
esac
|
||||
|
|
@ -48,15 +60,12 @@ if [ "${#tail[@]}" -gt 0 ] && [ "${tail[0]}" != -- ]; then filter=${tail[0]}; fi
|
|||
for i in "${!tail[@]}"; do if [ "${tail[$i]}" = -- ]; then harness=("${tail[@]:$((i + 1))}"); break; fi; done
|
||||
open=$(perl -MPOSIX -e '$n = 0; for (10 .. 40) { my $d = POSIX::dup($_); if (defined $d) { $n++; POSIX::close($d) } } print $n')
|
||||
[ -z "${FAKE_CM_KILLS:-}" ] || touch "$FAKE_CM_KILLS" # a watchdog kill arriving while a run is going
|
||||
echo "stage=$stage filter=$filter harness=${harness[*]} openfds=$open" >>"${FAKE_CM_LOG:?}"
|
||||
echo "stage=$stage filter=$filter harness=${harness[*]} openfds=$open mult=$mult min=$mintimeout" >>"${FAKE_CM_LOG:?}"
|
||||
[ "${FAKE_CM_RED:-}" != "$stage" ] || { echo "baseline failed" >&2; exit 4; }
|
||||
tests=5
|
||||
if [ "$stage" = 1 ] && [ -n "$filter" ]; then tests=$(awk -F'\t' -v f="$filter" '$1 == f { print $2 }' "${FAKE_CM_TESTS:-/dev/null}"); tests=${tests:-1}; fi
|
||||
mkdir -p "$out/mutants.out/log"
|
||||
printf 'running 0 tests\nrunning %s tests\n' "$tests" >"$out/mutants.out/log/baseline.log"
|
||||
col=3; [ "$stage" = 2 ] || col=2
|
||||
for o in caught missed timeout unviable; do
|
||||
awk -F'\t' -v c="$col" -v o="$o" -v z="$tests" '{ x = $c; if (z == 0) x = "missed"; if (x == o) print $1 }' <<<"$sel" >"$out/mutants.out/$o.txt"
|
||||
awk -F'\t' -v c="$col" -v o="$o" '{ x = $c; if (x == o) print $1 }' <<<"$sel" >"$out/mutants.out/$o.txt"
|
||||
done
|
||||
# a run that selected no mutant (the awk above printed empty lines): drop them
|
||||
for o in caught missed timeout unviable; do sed -i '/^$/d' "$out/mutants.out/$o.txt"; done
|
||||
|
|
|
|||
|
|
@ -202,7 +202,7 @@ check "$(seg jest-real.log exit)" 1 "no test counts recognised in the log (cargo
|
|||
# stubbed, by a script that does what RUN_ON_HOST=1 does (run the command inside the checked-out
|
||||
# revision). Slot locks are real flock(2) locks on files, taken by the driver and by the test.
|
||||
CMB=$here/fixtures/mutation-mock/fakebin
|
||||
cmrepo() { # cmrepo A|B: sets cm_repo and cm_wd. A: a workspace at the repo root, workdir crates/x. B: a standalone package pkg/ under the repo root.
|
||||
cmrepo() { # cmrepo A|B|C: sets cm_repo and cm_wd. A: a workspace at the repo root, workdir crates/x. B: a standalone package pkg/ under the repo root. C: B touching only the two files that have tests.
|
||||
local r f c; r=$(mktemp -d); cm_repo=$r/repo; mkdir -p "$cm_repo"; git -C "$cm_repo" init -q
|
||||
c="git -C $cm_repo -c user.name=t -c user.email=t@t"
|
||||
if [ "$1" = A ]; then
|
||||
|
|
@ -213,11 +213,12 @@ cmrepo() { # cmrepo A|B: sets cm_repo and cm_wd. A: a workspace at the repo ro
|
|||
else
|
||||
cm_wd=pkg; mkdir -p "$cm_repo/pkg/src/a" "$cm_repo/pkg/src/bin"
|
||||
printf '[package]\nname = "p"\n' >"$cm_repo/pkg/Cargo.toml"; : >"$cm_repo/pkg/src/lib.rs"; $c add -A 2>/dev/null; $c commit -qm c1
|
||||
for f in src/lib.rs src/main.rs src/bin/x.rs build.rs src/a/mod.rs src/a/b.rs; do echo "// $f" >"$cm_repo/pkg/$f"; done
|
||||
for f in src/a/mod.rs src/a/b.rs; do echo "// $f" >"$cm_repo/pkg/$f"; done
|
||||
[ "$1" = C ] || for f in src/lib.rs src/main.rs src/bin/x.rs build.rs; do echo "// $f" >"$cm_repo/pkg/$f"; done
|
||||
fi
|
||||
$c add -A 2>/dev/null; $c commit -qm c2
|
||||
}
|
||||
cmtable() { # cmtable A|B: sets cm_table (name <TAB> outcome under a filtered run <TAB> outcome under the full suite)
|
||||
cmtable() { # cmtable A|B|C: sets cm_table (name <TAB> outcome under a filtered run <TAB> outcome under the full suite)
|
||||
cm_table=$(mktemp)
|
||||
if [ "$1" = A ]; then
|
||||
local p=crates/x/src
|
||||
|
|
@ -238,6 +239,8 @@ cmtable() { # cmtable A|B: sets cm_table (name <TAB> outcome under a filtered
|
|||
"$p/main.rs:2:5: replace m -> i32 with 0 missed caught" \
|
||||
"crates/x/build.rs:2:5: replace bu -> i32 with 0 missed caught" \
|
||||
"$p/bin/tool.rs:2:5: replace t -> i32 with 0 missed caught" >"$cm_table"
|
||||
elif [ "$1" = C ]; then
|
||||
printf '%s\n' "src/a/mod.rs:2:5: replace a -> i32 with 0 caught caught" "src/a/b.rs:2:5: replace a -> i32 with 0 caught caught" >"$cm_table"
|
||||
else
|
||||
printf '%s\n' "src/lib.rs:2:5: replace a -> i32 with 0 caught caught" "src/main.rs:2:5: replace a -> i32 with 0 caught caught" \
|
||||
"src/bin/x.rs:2:5: replace a -> i32 with 0 caught caught" "build.rs:2:5: replace a -> i32 with 0 caught caught" \
|
||||
|
|
@ -308,6 +311,12 @@ if printf '%s\n' "$cm_out" | grep -qE '^(MISSED|TIMEOUT) '; then echo "FAIL carg
|
|||
if grep -qF 'stage=1 filter=alpha:: harness=--skip flaky' "$cm_log" && grep -qF 'stage=1 filter=alpha::inner:: harness=--skip flaky' "$cm_log" && grep -qF 'stage=2 filter= harness=--skip flaky' "$cm_log"
|
||||
then echo "ok cargo-mutants A-harness-args: TEST_ARGS follow the filter in stage 1 and stand alone in stage 2"; else echo "FAIL cargo-mutants A-harness-args"; sort -u "$cm_log"; fail=1; fi
|
||||
if [ "$(grep -c '^stage=2' "$cm_log")" = 3 ]; then echo "ok cargo-mutants A-stage2-sharded: 3 full-suite shards"; else echo "FAIL cargo-mutants A-stage2-sharded"; fail=1; fi
|
||||
# a file whose filter selects 0 tests is found at plan time and never pays a stage-1 pass; its names run in stage 2
|
||||
if grep -qF 'count filter=beta:: harness=--skip flaky' "$cm_log" && ! grep -q '^stage=1 filter=beta::' "$cm_log"
|
||||
then echo "ok cargo-mutants A-zero-test-plan: counted at plan time, no stage-1 run for it"; else echo "FAIL cargo-mutants A-zero-test-plan"; sort -u "$cm_log"; fail=1; fi
|
||||
# the stage-2 timeout bound is the generous default with a floor, not a tight multiple of a quiet-host baseline
|
||||
if [ "$(grep -c '^stage=2 .* mult=5 min=20$' "$cm_log")" = 3 ] && ! grep '^stage=2' "$cm_log" | grep -qv 'mult=5 min=20$'
|
||||
then echo "ok cargo-mutants A-timeout-bound: stage 2 runs at 5x with a 20 s floor"; else echo "FAIL cargo-mutants A-timeout-bound"; grep '^stage=2' "$cm_log"; fail=1; fi
|
||||
cmclean A; cmlocks A
|
||||
|
||||
# derivation on a standalone package under the repo root: the diff is taken at the workspace root (--relative)
|
||||
|
|
@ -319,6 +328,18 @@ cmcheck B-derivation 'FILTER src/a/mod.rs a:: tests=1 mutants=1' 'FILTER src/a/b
|
|||
cm B 'FILTERS=src/a/b.rs=-,src/lib.rs=special::'
|
||||
cmcheck B-override 'FILTER src/lib.rs special:: tests=1 mutants=1' 'FILTER src/a/mod.rs a:: ' '!FILTER src/a/b.rs' 'tested=6 caught=6'
|
||||
|
||||
# stage 1 settles every mutant: the full-suite baseline still runs (stage 2 is never skipped), and a red one is a red job
|
||||
cm C
|
||||
cmrc C-settled 0
|
||||
cmcheck C-settled 'MUTANTS tested=2 caught=2 missed=0 timeout=0 unviable=0 | stage1 filtered=2 survivors=0 confirmed_caught=0'
|
||||
if [ "$(grep -c '^stage=2' "$cm_log")" = 1 ]; then echo "ok cargo-mutants C-baseline-runs: one full-suite run although stage 1 settled every mutant"; else echo "FAIL cargo-mutants C-baseline-runs"; sort -u "$cm_log"; fail=1; fi
|
||||
cm C FAKE_CM_RED=2
|
||||
[ "$cm_rc" != 0 ] || { echo "FAIL cargo-mutants C-red-baseline: exit 0"; fail=1; }
|
||||
cmcheck C-red-baseline 'ERROR stage 2 (shard 1 of 1)' 'container exited 4' '!MUTANTS'
|
||||
cmclean C-red-baseline
|
||||
# CONFIRM_TIMEOUT_MULT and MIN_TEST_TIMEOUT reach the stage-2 run
|
||||
cm A CONFIRM_TIMEOUT_MULT=9 MIN_TEST_TIMEOUT=33
|
||||
if grep '^stage=2' "$cm_log" | grep -q 'mult=9 min=33$' && ! grep '^stage=2' "$cm_log" | grep -qv 'mult=9 min=33$'; then echo "ok cargo-mutants timeout-overrides: 9x with a 33 s floor in stage 2"; else echo "FAIL cargo-mutants timeout-overrides"; grep '^stage=2' "$cm_log"; fail=1; fi
|
||||
# slot lease: the job's own slot is held by ygg-verify (an inherited descriptor), others by other jobs
|
||||
cm A HOLD_OWN=1 HOLD_OTHERS=1; cmcheck lease-free 'containers=3 |'; cmlocks lease-free
|
||||
cm A HOLD_OWN=1 HOLD_OTHERS=3; cmcheck lease-one-free 'containers=2 |'
|
||||
|
|
@ -368,6 +389,16 @@ done
|
|||
lo=$(bash "$here/cargo-mutants.sh" "$cm_repo" HEAD~1 HEAD "$cm_wd" 2>&1); lrc=$?
|
||||
if [ "$lrc" = 2 ] && printf '%s' "$lo" | grep -q 'not an ancestor'; then echo "ok cargo-mutants launcher refuses a base that is not an ancestor"; else echo "FAIL cargo-mutants launcher base: exit $lrc, output: $lo"; fail=1; fi
|
||||
|
||||
# the -F escaper: every regex metacharacter, the pipe included, is escaped; nothing else is touched
|
||||
esc_bad=''
|
||||
for c in '\' . + '*' '?' '(' ')' '|' '[' ']' '{' '}' '^' '$'; do
|
||||
got=$(printf 'a%sb\n' "$c" | bash "$here/cargo-mutants.sh" --escape)
|
||||
[ "$got" = '^a'\\"$c"'b$' ] || esc_bad="$esc_bad [$c]"
|
||||
done
|
||||
got=$(printf 'replace x -> Vec<i32> with vec![1] & 2 - 3 #\n' | bash "$here/cargo-mutants.sh" --escape)
|
||||
[ "$got" = '^replace x -> Vec<i32> with vec!\[1\] & 2 - 3 #$' ] || esc_bad="$esc_bad [plain]"
|
||||
if [ -z "$esc_bad" ]; then echo "ok cargo-mutants escaper: all 14 metacharacters escaped, others literal"; else echo "FAIL cargo-mutants escaper:$esc_bad"; fail=1; fi
|
||||
|
||||
# wait-on.sh: pid, log and file forms, and the vanished-process exit
|
||||
W=$here/wait-on.sh
|
||||
wo() { local rc; WAIT_MAX=${WM:-4} bash "$W" "$@" >"$T/wo.out" 2>&1; rc=$?; echo "$rc"; }
|
||||
|
|
|
|||
Loading…
Reference in a new issue