From 0bd4aeca1ba7cd1ebc125ca6d410f05d3b662687 Mon Sep 17 00:00:00 2001 From: Eureka Date: Sun, 1 Nov 2026 20:57:04 +0100 Subject: [PATCH] cargo-mutants.sh: stage 2 always runs the full-suite baseline; zero-test files found at plan time; 5x timeout with a floor; --escape pinned - full-baseline-skipped: stage 2 runs even when stage 1 settled every name (one name rides along; cargo-mutants runs no baseline for an empty selection), so a red suite is a red job - timeout-mult-splits-missed: CONFIRM_TIMEOUT_MULT defaults to 5 and stage 2 gets --minimum-test-timeout like stage 1 - escaper-pipe-untested: the escaper is the --escape mode, run by the driver and by tests/test-tools.sh with each metacharacter - zero-test-stage1-wasted: the plan container derives each file's filter and counts its tests (cargo test -- --list); 0 tests goes straight to stage 2; the container-side baseline.log count is deleted Co-Authored-By: Claude Opus 5.5 --- tools/cargo-mutants.sh | 119 ++++++++++++--------- tools/fixtures/mutation-mock/fakebin/cargo | 25 +++-- tools/test-tools.sh | 37 ++++++- 3 files changed, 118 insertions(+), 63 deletions(-) diff --git a/tools/cargo-mutants.sh b/tools/cargo-mutants.sh index 3ecbed2..cc1d19a 100644 --- a/tools/cargo-mutants.sh +++ b/tools/cargo-mutants.sh @@ -18,7 +18,7 @@ # previous MISSED/TIMEOUT list): skips stage 1 and the listing # and runs only those names on the full suite. This is the rerun # after killing survivors with tests. -# SHARDS (3) JOBS (2) MIN_TEST_TIMEOUT (20) CONFIRM_TIMEOUT_MULT (2) +# SHARDS (3) JOBS (2) MIN_TEST_TIMEOUT (20) CONFIRM_TIMEOUT_MULT (5) # TIMEOUT seconds for the whole job (default 10800) # everything else tools/stopgap/ygg-verify.sh reads (MEM, CPUS, ...) # Typical, from an agent: launch as ONE background shell with a fresh log, then @@ -27,20 +27,26 @@ # survivors is a NAMES rerun, never a second whole-range sweep. # # STAGES (all inside the one job and the one host driver): -# plan one container writes the range diff and lists every mutant in it. -# stage 1 each touched file with a test filter runs its mutants against that +# plan one container writes the range diff, lists every mutant in it, derives each +# touched file's test filter and counts the tests that filter selects +# (`cargo test -- --list`, one build for all files). +# stage 1 each touched file whose filter selects tests runs its mutants against that # file's tests only (`cargo mutants -f -- `). The filter is # derived from the path: the module path after the last src/ (a/mod.rs -> # `a::`, a/b.rs -> `a::b::`); lib.rs, main.rs, bin/ and files outside src/ # have none. FILTERS overrides. Stage 1 decides only CAUGHT (its own green # filtered baseline guards false kills) and UNVIABLE (a build fact). Its # MISSED and TIMEOUT are never reported: a mutant that only another module's -# tests or tests/ kill survives a filtered run. A filter whose baseline ran -# 0 tests settles nothing and all its file's mutants go to stage 2. +# tests or tests/ kill survives a filtered run. A file whose filter selects 0 +# tests (tests=0) never runs stage 1: all its mutants go to stage 2. # stage 2 every name stage 1 did not settle (survivors, files without a filter, # zero-test files, or NAMES) runs on the full suite in cargo-mutants' own -# baseline, sharded over the containers, with the test timeout at -# CONFIRM_TIMEOUT_MULT x that baseline. Its outcome is final. +# baseline, sharded over the containers. Stage 2 ALWAYS runs: that baseline is +# the only full-suite check, so a red suite is a red job even when stage 1 +# settled every mutant (one name then rides along; its outcome is dropped). +# The test timeout is max(MIN_TEST_TIMEOUT, CONFIRM_TIMEOUT_MULT x baseline); +# the generous default keeps a slow-but-finishing test from flipping a +# MISSED mutant to TIMEOUT with the host's load. Its outcome is final. # merge each name counted once; MISSED and TIMEOUT are stage-2 outcomes only. # Containers per stage: min(SHARDS, SLOTS, units, 1 + free slots now), units being the # stage's files (stage 1) or names (stage 2). The job holds one slot (ygg-verify's); @@ -69,12 +75,18 @@ set -euo pipefail SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" sum() { printf '%s: %s\n' "$SUMTAG" "$*"; } +# ---- --escape: mutant names on stdin -> one anchored -F pattern per line ---------------------- +# Only regex metacharacters are escaped, so names holding * + ( ) | stay literal; ^...$ pins the whole name. +if [ "${1:-}" = --escape ]; then + exec perl -pe 'chomp; s/([\\.+*?()|\[\]{}^\$])/\\$1/g; $_ = "^$_\$\n"' +fi + # ---- launcher (workstation) ------------------------------------------------- if [ "${1:-}" != --host ] && [ "${1:-}" != --plan ] && [ "${1:-}" != --item ]; then here=$(cd "$(dirname "$0")" && pwd) repo=${1:?usage: cargo-mutants.sh [workdir]} rev=${2:?revision}; base=${3:?base revision}; workdir=${4:-.} - : "${SHARDS:=3}" "${JOBS:=2}" "${MIN_TEST_TIMEOUT:=20}" "${CONFIRM_TIMEOUT_MULT:=2}" "${TIMEOUT:=10800}" + : "${SHARDS:=3}" "${JOBS:=2}" "${MIN_TEST_TIMEOUT:=20}" "${CONFIRM_TIMEOUT_MULT:=5}" "${TIMEOUT:=10800}" for v in SHARDS JOBS MIN_TEST_TIMEOUT TIMEOUT; do case ${!v} in ''|*[!0-9]*|0) echo "cargo-mutants: $v must be a positive integer" >&2; exit 2 ;; esac done @@ -128,7 +140,7 @@ if [ "$1" = --host ]; then -v "$YGG_WORK:/seed:ro" -v "$D:/cm:ro" -v /etc/machine-id:/etc/machine-id:ro -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache -e BASH_ENV=/tmp/.ygg-bash-env - -e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "JOBS=$JOBS" + -e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "FILTERS=${FILTERS:-}" -e "CM_NAMES=${CM_NAMES:-}" -e "JOBS=$JOBS" -e "MIN_TEST_TIMEOUT=$MIN_TEST_TIMEOUT" -e "CONFIRM_TIMEOUT_MULT=$CONFIRM_TIMEOUT_MULT" -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*') [ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE") @@ -160,7 +172,7 @@ if [ "$1" = --host ]; then tp=$(( $(now) - tp )) if [ "${CM_NAMES:-}" = 1 ]; then sort -u "$D/names.txt" >"$D/m/universe" else sort -u "$D/plan/names.txt" >"$D/m/universe"; fi - for f in s1.caught s1.unviable s1.surv s1.names s1.tests; do : >"$D/m/$f"; done + for f in s1.caught s1.unviable s1.surv s1.names; do : >"$D/m/$f"; done # lease : the containers of a stage = min(SHARDS, SLOTS, units, 1 + free slots now); sets K and holds the leases lease() { @@ -199,31 +211,14 @@ if [ "$1" = --host ]; then tail -n 5 "$2/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR |" } - # ---- stage 1: files with a filter - declare -A fover - if [ -n "${FILTERS:-}" ]; then - IFS=, read -ra fl <<<"$FILTERS" - for e in "${fl[@]}"; do fover[${e%%=*}]=${e#*=}; done - fi - derive_filter() { # derive_filter : the test filter of a file, nothing for the full suite - local p=$1 rest - if [ -n "${fover[$p]+x}" ]; then [ "${fover[$p]}" = - ] || echo "${fover[$p]}"; return 0; fi - case $p in */src/*.rs|src/*.rs) ;; *) return 0 ;; esac - rest=${p##*src/}; rest=${rest%.rs} - case $rest in lib|main|bin/*|*/bin/*) return 0 ;; esac - rest=${rest%/mod} - echo "${rest//\//::}::" - } + # ---- stage 1: files whose filter selects tests (the plan container derived each file's filter and counted its tests) S1_FILE=(); S1_FILTER=(); s1n=0 if [ "${CM_NAMES:-}" != 1 ]; then - # files by mutant count, largest first - while read -r cnt file; do - f=$(derive_filter "$file") - [ -n "$f" ] || continue + while IFS=$'\t' read -r cnt file f t; do # files by mutant count, largest first + [ "$t" != 0 ] || continue # a filter that selects no test settles nothing: straight to stage 2 s1n=$((s1n + 1)); S1_FILE[$s1n]=$file; S1_FILTER[$s1n]=$f awk -F: -v f="$file" '$1 == f' "$D/m/universe" >"$D/items/s1-$s1n.names" - echo "$cnt" >"$D/items/s1-$s1n.count" - done < <(awk -F: '{ c[$1]++ } END { for (f in c) print c[f], f }' "$D/m/universe" | sort -k1,1nr -k2) + done <"$D/plan/filters.txt" fi s1_item() { local n=$1 rc @@ -246,25 +241,16 @@ if [ "$1" = --host ]; then [ -e "$D/q/s1-$n" ] || continue # never started: the job is already red for another file container_error "stage 1 (${S1_FILE[$n]})" "$o"; bad=1; continue fi - echo "$n $(cat "$o/tests" 2>/dev/null || echo 0)" >>"$D/m/s1.tests" done if [ "$bad" != 0 ] || [ -e "$D/failed" ]; then exit 1; fi fi # FILTER lines: the anomalies (0 tests) first, then the rest, at most 12 in all - shown=0; hidden=0 - for pass in zero rest; do - for n in $(seq 1 "$s1n"); do - t=$(awk -v n="$n" '$1 == n { print $2 }' "$D/m/s1.tests"); t=${t:-0} - if [ "$pass" = zero ] && [ "$t" != 0 ]; then continue; fi - if [ "$pass" = rest ] && [ "$t" = 0 ]; then continue; fi - if [ "$shown" -lt 12 ]; then sum "FILTER ${S1_FILE[$n]} ${S1_FILTER[$n]} tests=$t mutants=$(cat "$D/items/s1-$n.count")"; shown=$((shown + 1)) - else hidden=$((hidden + 1)); fi - done - done - [ "$hidden" = 0 ] || sum "FILTER ... $hidden more files" + if [ "${CM_NAMES:-}" != 1 ]; then + { awk -F'\t' '$4 == 0' "$D/plan/filters.txt"; awk -F'\t' '$4 != 0' "$D/plan/filters.txt"; } | + awk -F'\t' '{ if (NR <= 12) printf "FILTER %s %s tests=%s mutants=%s\n", $2, $3, $4, $1; else h++ } END { if (h) printf "FILTER ... %d more files\n", h }' | + while IFS= read -r l; do sum "$l"; done + fi for n in $(seq 1 "$s1n"); do - t=$(awk -v n="$n" '$1 == n { print $2 }' "$D/m/s1.tests"); t=${t:-0} - [ "$t" != 0 ] || continue # a filter that ran no test settled nothing o="$D/items/s1-$n/out" cat "$D/items/s1-$n.names" >>"$D/m/s1.names" cat "$o/caught.txt" >>"$D/m/s1.caught"; cat "$o/unviable.txt" >>"$D/m/s1.unviable" @@ -276,11 +262,13 @@ if [ "$1" = --host ]; then done cat "$D/m/s1.caught" "$D/m/s1.unviable" | sort -u >"$D/m/s1.settled" comm -23 "$D/m/universe" "$D/m/s1.settled" >"$D/s2.txt" - # only regex metacharacters are escaped, so names holding * + ( ) | stay literal; ^...$ pins the whole name - perl -pe 'chomp; s/([\\.+*?()|\[\]{}^\$])/\\$1/g; $_ = "^$_\$\n"' "$D/s2.txt" >"$D/s2.re" + # Stage 2 always runs, so cargo-mutants' full-suite baseline always does (cargo-mutants runs no baseline for an + # empty selection). With every name settled it carries one name; the merge drops that name's outcome. + if [ -s "$D/s2.txt" ]; then bash "$D/cargo-mutants.sh" --escape <"$D/s2.txt" >"$D/s2.re" + else head -n 1 "$D/m/universe" | bash "$D/cargo-mutants.sh" --escape >"$D/s2.re"; fi # ---- stage 2: the full suite on everything stage 1 did not settle - s2n=$(wc -l <"$D/s2.txt" | tr -d ' ') + s2n=$(wc -l <"$D/s2.re" | tr -d " ") K2=0; ts2=0 for l in caught missed timeout unviable; do : >"$D/m/s2.$l"; done s2_shard() { @@ -341,19 +329,47 @@ set +e SRC=${CM_SRC:-/src}; CM=${CM_DIR:-/cm}; OUT=${CM_OUT:-/out}; SEED=${CM_SEED:-/seed} if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "cannot copy $SEED into $SRC"; exit 2; }; fi cd "$SRC/${WORKDIR:-.}" || { echo "BAD WORKDIR"; exit 2; } +set -f; read -ra targs <<<"${TEST_ARGS:-}"; set +f if [ "$1" = --plan ]; then wsroot=$(dirname "$(cargo locate-project --workspace --message-format plain)") || { echo "cargo locate-project failed"; exit 2; } git -C "$wsroot" diff --relative "$BASE" HEAD >"$OUT/d" || { echo "git diff of the range failed"; exit 2; } cargo mutants --list --in-diff "$OUT/d" >"$OUT/names.txt" 2>"$OUT/list.err" || { echo "cargo mutants --list failed"; tail -n 5 "$OUT/list.err"; exit 2; } + : >"$OUT/filters.txt" + [ "$CM_NAMES" != 1 ] || exit 0 + # stage 1's plan: each touched file's test filter and how many tests it selects (0 = nothing for stage 1 to run) + declare -A fover + if [ -n "${FILTERS:-}" ]; then + IFS=, read -ra fl <<<"$FILTERS" + for e in "${fl[@]}"; do fover[${e%%=*}]=${e#*=}; done + fi + derive_filter() { # derive_filter : the test filter of a file, nothing for the full suite + local p=$1 rest + if [ -n "${fover[$p]+x}" ]; then [ "${fover[$p]}" = - ] || echo "${fover[$p]}"; return 0; fi + case $p in */src/*.rs|src/*.rs) ;; *) return 0 ;; esac + rest=${p##*src/}; rest=${rest%.rs} + case $rest in lib|main|bin/*|*/bin/*) return 0 ;; esac + rest=${rest%/mod} + echo "${rest//\//::}::" + } + built=0 + while IFS=$'\t' read -r cnt file; do # files by mutant count, largest first + f=$(derive_filter "$file") + [ -n "$f" ] || continue + if [ "$built" = 0 ]; then + cargo test --no-run >"$OUT/build.log" 2>&1 || { echo "cargo test --no-run failed"; tail -n 5 "$OUT/build.log"; exit 2; } + built=1 + fi + t=$(cargo test -- "$f" "${targs[@]}" --list 2>/dev/null | awk '/^[0-9]+ tests?, / { s += $1 } END { print s + 0 }') + printf '%s\t%s\t%s\t%s\n' "$cnt" "$file" "$f" "$t" >>"$OUT/filters.txt" + done < <(awk -F: '{ c[$1]++ } END { for (f in c) print c[f] "\t" f }' "$OUT/names.txt" | sort -t "$(printf '\t')" -k1,1nr -k2) exit 0 fi # --item: stage 1 (CM_FILE with its CM_FILTER) or stage 2 (shard CM_SHARD of the names in $CM/s2.re) -set -f; read -ra targs <<<"${TEST_ARGS:-}"; set +f o=${CM_TMP:-/tmp}/cmo -args=(mutants --in-diff "$CM/plan/d" -o "$o" --jobs "$JOBS") +args=(mutants --in-diff "$CM/plan/d" -o "$o" --jobs "$JOBS" --minimum-test-timeout "$MIN_TEST_TIMEOUT") tail_args=() if [ "$CM_STAGE" = 1 ]; then - args+=(-f "$CM_FILE" --minimum-test-timeout "$MIN_TEST_TIMEOUT") + args+=(-f "$CM_FILE") tail_args=(-- "$CM_FILTER"); [ "${#targs[@]}" = 0 ] || tail_args+=(-- "${targs[@]}") else mapfile -t res <"$CM/s2.re" @@ -365,7 +381,6 @@ fi cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1 rc=$? for l in caught missed timeout unviable; do cp "$o/mutants.out/$l.txt" "$OUT/$l.txt" 2>/dev/null || : >"$OUT/$l.txt"; done -awk '/^running [0-9]+ tests?/ { s += $2 } END { print s + 0 }' "$o/mutants.out/log/baseline.log" >"$OUT/tests" 2>/dev/null || echo 0 >"$OUT/tests" echo "$rc" >"$OUT/rc" tail -n 30 "$OUT/cargo.log" # 0 clean, 2 missed and 3 timeouts are normal; 4 (red baseline) and everything else are errors diff --git a/tools/fixtures/mutation-mock/fakebin/cargo b/tools/fixtures/mutation-mock/fakebin/cargo index 2290061..a722cbf 100755 --- a/tools/fixtures/mutation-mock/fakebin/cargo +++ b/tools/fixtures/mutation-mock/fakebin/cargo @@ -1,20 +1,30 @@ #!/usr/bin/env bash # Fake cargo for tools/test-tools.sh, only what tools/cargo-mutants.sh calls: # cargo locate-project --workspace --message-format plain +# cargo test --no-run +# cargo test -- [] --list (prints " tests, 0 benchmarks" per test binary) # cargo mutants [--list] --in-diff [-f ] [-F ]... [--shard k/n] [-o ] [...] [-- [-- ]] # FAKE_CM_TABLE: tab-separated # (caught|missed|timeout|unviable). A run with -F is the full suite; a run with -f and a filter is filtered. -# FAKE_CM_TESTS: tab-separated (default 1; the unfiltered suite runs 5) +# FAKE_CM_TESTS: tab-separated (default 1) # FAKE_CM_RED=1|2: the stage-1 or stage-2 run fails its baseline (exit 4, no output). # FAKE_CM_KILLS: a path a run creates (the watchdog's YGG_KILLED flag appearing mid-job). # FAKE_CM_LOG: appended one line per run (stage, filter, harness args, open fds from 10 up). # FAKE_WSROOT: the workspace root relative to the repository root, with a trailing slash ('' = the repository root). case ${1:-} in locate-project) echo "$(git rev-parse --show-toplevel)/${FAKE_WSROOT:-}Cargo.toml"; exit 0 ;; + test) + shift; for a; do [ "$a" != --no-run ] || exit 0; done + [ "${1:-}" = -- ] || { echo "fake cargo test: unsupported arguments" >&2; exit 64; } + shift; f=$1; shift; h=() + for a; do [ "$a" = --list ] || h+=("$a"); done + echo "count filter=$f harness=${h[*]}" >>"${FAKE_CM_LOG:?}" + n=$(awk -F'\t' -v f="$f" '$1 == f { print $2 }' "${FAKE_CM_TESTS:-/dev/null}") + printf '%s tests, 0 benchmarks\n0 tests, 0 benchmarks\n' "${n:-1}"; exit 0 ;; mutants) shift ;; *) echo "fake cargo: unsupported ${1:-}" >&2; exit 127 ;; esac -list=0; diff=''; ffile=''; res=(); shard=''; out=''; tail=() +list=0; mult=''; mintimeout=''; diff=''; ffile=''; res=(); shard=''; out=''; tail=() while [ $# -gt 0 ]; do case $1 in --list) list=1 ;; @@ -23,7 +33,9 @@ while [ $# -gt 0 ]; do -F) res+=("$2"); shift ;; --shard) shard=$2; shift ;; -o) out=$2; shift ;; - --jobs|--minimum-test-timeout|--timeout-multiplier) shift ;; + --jobs) shift ;; + --minimum-test-timeout) mintimeout=$2; shift ;; + --timeout-multiplier) mult=$2; shift ;; --) shift; tail=("$@"); break ;; *) echo "fake cargo mutants: unknown argument $1" >&2; exit 64 ;; esac @@ -48,15 +60,12 @@ if [ "${#tail[@]}" -gt 0 ] && [ "${tail[0]}" != -- ]; then filter=${tail[0]}; fi for i in "${!tail[@]}"; do if [ "${tail[$i]}" = -- ]; then harness=("${tail[@]:$((i + 1))}"); break; fi; done open=$(perl -MPOSIX -e '$n = 0; for (10 .. 40) { my $d = POSIX::dup($_); if (defined $d) { $n++; POSIX::close($d) } } print $n') [ -z "${FAKE_CM_KILLS:-}" ] || touch "$FAKE_CM_KILLS" # a watchdog kill arriving while a run is going -echo "stage=$stage filter=$filter harness=${harness[*]} openfds=$open" >>"${FAKE_CM_LOG:?}" +echo "stage=$stage filter=$filter harness=${harness[*]} openfds=$open mult=$mult min=$mintimeout" >>"${FAKE_CM_LOG:?}" [ "${FAKE_CM_RED:-}" != "$stage" ] || { echo "baseline failed" >&2; exit 4; } -tests=5 -if [ "$stage" = 1 ] && [ -n "$filter" ]; then tests=$(awk -F'\t' -v f="$filter" '$1 == f { print $2 }' "${FAKE_CM_TESTS:-/dev/null}"); tests=${tests:-1}; fi mkdir -p "$out/mutants.out/log" -printf 'running 0 tests\nrunning %s tests\n' "$tests" >"$out/mutants.out/log/baseline.log" col=3; [ "$stage" = 2 ] || col=2 for o in caught missed timeout unviable; do - awk -F'\t' -v c="$col" -v o="$o" -v z="$tests" '{ x = $c; if (z == 0) x = "missed"; if (x == o) print $1 }' <<<"$sel" >"$out/mutants.out/$o.txt" + awk -F'\t' -v c="$col" -v o="$o" '{ x = $c; if (x == o) print $1 }' <<<"$sel" >"$out/mutants.out/$o.txt" done # a run that selected no mutant (the awk above printed empty lines): drop them for o in caught missed timeout unviable; do sed -i '/^$/d' "$out/mutants.out/$o.txt"; done diff --git a/tools/test-tools.sh b/tools/test-tools.sh index a52a44b..bedbf38 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -202,7 +202,7 @@ check "$(seg jest-real.log exit)" 1 "no test counts recognised in the log (cargo # stubbed, by a script that does what RUN_ON_HOST=1 does (run the command inside the checked-out # revision). Slot locks are real flock(2) locks on files, taken by the driver and by the test. CMB=$here/fixtures/mutation-mock/fakebin -cmrepo() { # cmrepo A|B: sets cm_repo and cm_wd. A: a workspace at the repo root, workdir crates/x. B: a standalone package pkg/ under the repo root. +cmrepo() { # cmrepo A|B|C: sets cm_repo and cm_wd. A: a workspace at the repo root, workdir crates/x. B: a standalone package pkg/ under the repo root. C: B touching only the two files that have tests. local r f c; r=$(mktemp -d); cm_repo=$r/repo; mkdir -p "$cm_repo"; git -C "$cm_repo" init -q c="git -C $cm_repo -c user.name=t -c user.email=t@t" if [ "$1" = A ]; then @@ -213,11 +213,12 @@ cmrepo() { # cmrepo A|B: sets cm_repo and cm_wd. A: a workspace at the repo ro else cm_wd=pkg; mkdir -p "$cm_repo/pkg/src/a" "$cm_repo/pkg/src/bin" printf '[package]\nname = "p"\n' >"$cm_repo/pkg/Cargo.toml"; : >"$cm_repo/pkg/src/lib.rs"; $c add -A 2>/dev/null; $c commit -qm c1 - for f in src/lib.rs src/main.rs src/bin/x.rs build.rs src/a/mod.rs src/a/b.rs; do echo "// $f" >"$cm_repo/pkg/$f"; done + for f in src/a/mod.rs src/a/b.rs; do echo "// $f" >"$cm_repo/pkg/$f"; done + [ "$1" = C ] || for f in src/lib.rs src/main.rs src/bin/x.rs build.rs; do echo "// $f" >"$cm_repo/pkg/$f"; done fi $c add -A 2>/dev/null; $c commit -qm c2 } -cmtable() { # cmtable A|B: sets cm_table (name outcome under a filtered run outcome under the full suite) +cmtable() { # cmtable A|B|C: sets cm_table (name outcome under a filtered run outcome under the full suite) cm_table=$(mktemp) if [ "$1" = A ]; then local p=crates/x/src @@ -238,6 +239,8 @@ cmtable() { # cmtable A|B: sets cm_table (name outcome under a filtered "$p/main.rs:2:5: replace m -> i32 with 0 missed caught" \ "crates/x/build.rs:2:5: replace bu -> i32 with 0 missed caught" \ "$p/bin/tool.rs:2:5: replace t -> i32 with 0 missed caught" >"$cm_table" + elif [ "$1" = C ]; then + printf '%s\n' "src/a/mod.rs:2:5: replace a -> i32 with 0 caught caught" "src/a/b.rs:2:5: replace a -> i32 with 0 caught caught" >"$cm_table" else printf '%s\n' "src/lib.rs:2:5: replace a -> i32 with 0 caught caught" "src/main.rs:2:5: replace a -> i32 with 0 caught caught" \ "src/bin/x.rs:2:5: replace a -> i32 with 0 caught caught" "build.rs:2:5: replace a -> i32 with 0 caught caught" \ @@ -308,6 +311,12 @@ if printf '%s\n' "$cm_out" | grep -qE '^(MISSED|TIMEOUT) '; then echo "FAIL carg if grep -qF 'stage=1 filter=alpha:: harness=--skip flaky' "$cm_log" && grep -qF 'stage=1 filter=alpha::inner:: harness=--skip flaky' "$cm_log" && grep -qF 'stage=2 filter= harness=--skip flaky' "$cm_log" then echo "ok cargo-mutants A-harness-args: TEST_ARGS follow the filter in stage 1 and stand alone in stage 2"; else echo "FAIL cargo-mutants A-harness-args"; sort -u "$cm_log"; fail=1; fi if [ "$(grep -c '^stage=2' "$cm_log")" = 3 ]; then echo "ok cargo-mutants A-stage2-sharded: 3 full-suite shards"; else echo "FAIL cargo-mutants A-stage2-sharded"; fail=1; fi +# a file whose filter selects 0 tests is found at plan time and never pays a stage-1 pass; its names run in stage 2 +if grep -qF 'count filter=beta:: harness=--skip flaky' "$cm_log" && ! grep -q '^stage=1 filter=beta::' "$cm_log" +then echo "ok cargo-mutants A-zero-test-plan: counted at plan time, no stage-1 run for it"; else echo "FAIL cargo-mutants A-zero-test-plan"; sort -u "$cm_log"; fail=1; fi +# the stage-2 timeout bound is the generous default with a floor, not a tight multiple of a quiet-host baseline +if [ "$(grep -c '^stage=2 .* mult=5 min=20$' "$cm_log")" = 3 ] && ! grep '^stage=2' "$cm_log" | grep -qv 'mult=5 min=20$' +then echo "ok cargo-mutants A-timeout-bound: stage 2 runs at 5x with a 20 s floor"; else echo "FAIL cargo-mutants A-timeout-bound"; grep '^stage=2' "$cm_log"; fail=1; fi cmclean A; cmlocks A # derivation on a standalone package under the repo root: the diff is taken at the workspace root (--relative) @@ -319,6 +328,18 @@ cmcheck B-derivation 'FILTER src/a/mod.rs a:: tests=1 mutants=1' 'FILTER src/a/b cm B 'FILTERS=src/a/b.rs=-,src/lib.rs=special::' cmcheck B-override 'FILTER src/lib.rs special:: tests=1 mutants=1' 'FILTER src/a/mod.rs a:: ' '!FILTER src/a/b.rs' 'tested=6 caught=6' +# stage 1 settles every mutant: the full-suite baseline still runs (stage 2 is never skipped), and a red one is a red job +cm C +cmrc C-settled 0 +cmcheck C-settled 'MUTANTS tested=2 caught=2 missed=0 timeout=0 unviable=0 | stage1 filtered=2 survivors=0 confirmed_caught=0' +if [ "$(grep -c '^stage=2' "$cm_log")" = 1 ]; then echo "ok cargo-mutants C-baseline-runs: one full-suite run although stage 1 settled every mutant"; else echo "FAIL cargo-mutants C-baseline-runs"; sort -u "$cm_log"; fail=1; fi +cm C FAKE_CM_RED=2 +[ "$cm_rc" != 0 ] || { echo "FAIL cargo-mutants C-red-baseline: exit 0"; fail=1; } +cmcheck C-red-baseline 'ERROR stage 2 (shard 1 of 1)' 'container exited 4' '!MUTANTS' +cmclean C-red-baseline +# CONFIRM_TIMEOUT_MULT and MIN_TEST_TIMEOUT reach the stage-2 run +cm A CONFIRM_TIMEOUT_MULT=9 MIN_TEST_TIMEOUT=33 +if grep '^stage=2' "$cm_log" | grep -q 'mult=9 min=33$' && ! grep '^stage=2' "$cm_log" | grep -qv 'mult=9 min=33$'; then echo "ok cargo-mutants timeout-overrides: 9x with a 33 s floor in stage 2"; else echo "FAIL cargo-mutants timeout-overrides"; grep '^stage=2' "$cm_log"; fail=1; fi # slot lease: the job's own slot is held by ygg-verify (an inherited descriptor), others by other jobs cm A HOLD_OWN=1 HOLD_OTHERS=1; cmcheck lease-free 'containers=3 |'; cmlocks lease-free cm A HOLD_OWN=1 HOLD_OTHERS=3; cmcheck lease-one-free 'containers=2 |' @@ -368,6 +389,16 @@ done lo=$(bash "$here/cargo-mutants.sh" "$cm_repo" HEAD~1 HEAD "$cm_wd" 2>&1); lrc=$? if [ "$lrc" = 2 ] && printf '%s' "$lo" | grep -q 'not an ancestor'; then echo "ok cargo-mutants launcher refuses a base that is not an ancestor"; else echo "FAIL cargo-mutants launcher base: exit $lrc, output: $lo"; fail=1; fi +# the -F escaper: every regex metacharacter, the pipe included, is escaped; nothing else is touched +esc_bad='' +for c in '\' . + '*' '?' '(' ')' '|' '[' ']' '{' '}' '^' '$'; do + got=$(printf 'a%sb\n' "$c" | bash "$here/cargo-mutants.sh" --escape) + [ "$got" = '^a'\\"$c"'b$' ] || esc_bad="$esc_bad [$c]" +done +got=$(printf 'replace x -> Vec with vec![1] & 2 - 3 #\n' | bash "$here/cargo-mutants.sh" --escape) +[ "$got" = '^replace x -> Vec with vec!\[1\] & 2 - 3 #$' ] || esc_bad="$esc_bad [plain]" +if [ -z "$esc_bad" ]; then echo "ok cargo-mutants escaper: all 14 metacharacters escaped, others literal"; else echo "FAIL cargo-mutants escaper:$esc_bad"; fail=1; fi + # wait-on.sh: pid, log and file forms, and the vanished-process exit W=$here/wait-on.sh wo() { local rc; WAIT_MAX=${WM:-4} bash "$W" "$@" >"$T/wo.out" 2>&1; rc=$?; echo "$rc"; }