#!/usr/bin/env bash # cargo-mutants on Yggdrasil for a revision range: ONE verify job, per-file scoped # tests, survivors confirmed on the full suite, sharded over free slots, one merged # survivor list. The one owner of "run cargo-mutants" (no brief, no job script runs # an inline `cargo mutants`). # # SUMMARY= tools/cargo-mutants.sh [workdir] # workdir directory inside the repo where cargo runs (a package or workspace # member, e.g. crates/mind-body; CultLib has no Cargo.toml at its root). # The mutated range is `git diff --relative ` taken at the # WORKSPACE ROOT: cargo-mutants matches --in-diff paths, -f paths and # prints mutant names relative to the workspace root, not to workdir. # env FILTERS '=,...' overrides a file's derived test filter # (path as the mutant names print it); '-' = full suite # TEST_ARGS harness args, e.g. '--skip flaky_one' (a flaky test would # make a surviving mutant read as killed in one run) # NAMES local file of mutant names (one per line, copied from a # previous MISSED/TIMEOUT list): skips stage 1 and the listing # and runs only those names on the full suite. This is the rerun # after killing survivors with tests. # SHARDS (3) JOBS (2) MIN_TEST_TIMEOUT (20) CONFIRM_TIMEOUT_MULT (5) # TIMEOUT seconds for the whole job (default 10800) # everything else tools/stopgap/ygg-verify.sh reads (MEM, CPUS, ...) # Typical, from an agent: launch as ONE background shell with a fresh log, then # tools/stopgap/ygg-verify.sh wait (Bash tool timeout >= 280000 ms) # Run it ONCE per pass, at the end, on the final spelling of the code. Chasing # survivors is a NAMES rerun, never a second whole-range sweep. # # STAGES (all inside the one job and the one host driver): # plan one container writes the range diff, lists every mutant in it, derives each # touched file's test filter and counts the tests that filter selects # (`cargo test -- --list`, one build for all files). # stage 1 each touched file whose filter selects tests runs its mutants against that # file's tests only (`cargo mutants -f -- `). The filter is # derived from the path: the module path after the last src/ (a/mod.rs -> # `a::`, a/b.rs -> `a::b::`); lib.rs, main.rs, bin/ and files outside src/ # have none. FILTERS overrides. Stage 1 decides only CAUGHT (its own green # filtered baseline guards false kills) and UNVIABLE (a build fact). Its # MISSED and TIMEOUT are never reported: a mutant that only another module's # tests or tests/ kill survives a filtered run. A file whose filter selects 0 # tests (tests=0) never runs stage 1: all its mutants go to stage 2. # stage 2 every name stage 1 did not settle (survivors, files without a filter, # zero-test files, or NAMES) runs on the full suite in cargo-mutants' own # baseline, sharded over the containers. Stage 2 ALWAYS runs: that baseline is # the only full-suite check, so a red suite is a red job even when stage 1 # settled every mutant (one name then rides along; its outcome is dropped). # The test timeout is max(MIN_TEST_TIMEOUT, CONFIRM_TIMEOUT_MULT x baseline); # the generous default keeps a slow-but-finishing test from flipping a # MISSED mutant to TIMEOUT with the host's load. Its outcome is final. # merge each name counted once; MISSED and TIMEOUT are stage-2 outcomes only. # Containers per stage: min(SHARDS, SLOTS, units, 1 + free slots now), units being the # stage's files (stage 1) or names (stage 2). The job holds one slot (ygg-verify's); # the driver leases further slot locks with flock -n, never waits for one, takes none # while /proc/pressure/io full avg60 > 20, and releases them when each stage ends. A # killed job (YGG_KILLED) starts nothing further. # # OUTPUT: the YGG-SUMMARY/$YGG_NONCE lines ygg-verify's summary carries; per-container # logs stay on the host and go with the job. # FILTER tests= mutants= one per stage-1 file (zero-test first) # MUTANTS tested=N caught=C missed=M timeout=T unviable=U | stage1 filtered=F # survivors=S confirmed_caught=X | containers=K | plan=..s stage1=..s/ stage2=..s/ | m # MISSED / TIMEOUT final outcomes, at most 50 # ERROR ... a red baseline, a container that failed, mutants with no outcome: # the job exits non-zero and prints no MUTANTS line # Exit 0 when every stage ran on a green baseline; survivors do not fail the job (the # score is not a gate). cargo-mutants exits 2 (missed) and 3 (timeouts) normally; 4 is a # red baseline; anything else is an error. # # DELETION LINE: this file dies with tools/stopgap/ygg-verify.sh when `idunn verify` # ships a typed mutation step. Modes in this one file: no flag = launcher on the # workstation; --host = the driver on the Yggdrasil host (ygg-verify.sh RUN_ON_HOST=1); # --plan / --item = the work inside a container. set -euo pipefail SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}" sum() { printf '%s: %s\n' "$SUMTAG" "$*"; } # ---- --escape: mutant names on stdin -> one anchored -F pattern per line ---------------------- # Only regex metacharacters are escaped, so names holding * + ( ) | stay literal; ^...$ pins the whole name. if [ "${1:-}" = --escape ]; then exec perl -pe 'chomp; s/([\\.+*?()|\[\]{}^\$])/\\$1/g; $_ = "^$_\$\n"' fi # ---- launcher (workstation) ------------------------------------------------- if [ "${1:-}" != --host ] && [ "${1:-}" != --plan ] && [ "${1:-}" != --item ]; then here=$(cd "$(dirname "$0")" && pwd) repo=${1:?usage: cargo-mutants.sh [workdir]} rev=${2:?revision}; base=${3:?base revision}; workdir=${4:-.} : "${SHARDS:=3}" "${JOBS:=2}" "${MIN_TEST_TIMEOUT:=20}" "${CONFIRM_TIMEOUT_MULT:=5}" "${TIMEOUT:=10800}" for v in SHARDS JOBS MIN_TEST_TIMEOUT TIMEOUT; do case ${!v} in ''|*[!0-9]*|0) echo "cargo-mutants: $v must be a positive integer" >&2; exit 2 ;; esac done case $CONFIRM_TIMEOUT_MULT in ''|*[!0-9.]*|.*|*.|*.*.*) echo "cargo-mutants: CONFIRM_TIMEOUT_MULT must be a positive number" >&2; exit 2 ;; esac if [ -n "${FILTERS:-}" ]; then old=$IFS; IFS=, for f in $FILTERS; do case $f in ?*=?*) ;; *) IFS=$old; echo "cargo-mutants: FILTERS entries are =" >&2; exit 2 ;; esac done IFS=$old fi sha_rev=$(git -C "$repo" rev-parse --verify "$rev^{commit}") || { echo "cargo-mutants: unknown revision" >&2; exit 2; } sha_base=$(git -C "$repo" rev-parse --verify "$base^{commit}") || { echo "cargo-mutants: unknown base revision" >&2; exit 2; } git -C "$repo" merge-base --is-ancestor "$sha_base" "$sha_rev" || { echo "cargo-mutants: base is not an ancestor of rev (the verify mirror carries only rev's history)" >&2; exit 2; } stage=$(mktemp -d) trap 'rm -rf "$stage"' EXIT cp "$here/cargo-mutants.sh" "$stage/" files=(cargo-mutants.sh) if [ -n "${NAMES:-}" ]; then [ -r "$NAMES" ] || { echo "cargo-mutants: cannot read the NAMES file" >&2; exit 2; } tr -d '\r' <"$NAMES" >"$stage/names.txt"; files+=(names.txt) fi bundle=$(tar -C "$stage" -cz "${files[@]}" | base64 | tr -d '\n') # The driver runs on the host: it unpacks the bundle into its own directory (mounted read-only into every container) and removes it when done. cmd=$(printf 'd=$(mktemp -d) && echo %s | base64 -d | tar -xz -C "$d" && WORKDIR=%q BASE=%q FILTERS=%q TEST_ARGS=%q SHARDS=%q JOBS=%q MIN_TEST_TIMEOUT=%q CONFIRM_TIMEOUT_MULT=%q CM_NAMES=%q CM_DIR="$d" bash "$d/cargo-mutants.sh" --host' \ "$bundle" "$workdir" "$sha_base" "${FILTERS:-}" "${TEST_ARGS:-}" "$SHARDS" "$JOBS" "$MIN_TEST_TIMEOUT" "$CONFIRM_TIMEOUT_MULT" "${NAMES:+1}") RUN_ON_HOST=1 TIMEOUT=$TIMEOUT "$here/stopgap/ygg-verify.sh" "$repo" "$sha_rev" rust "$cmd" exit $? fi # ---- driver (Yggdrasil host, inside the job's slot) -------------------------- if [ "$1" = --host ]; then : "${YGG_WORK:?not started by ygg-verify.sh RUN_ON_HOST=1}" "${YGG_IMAGE:?}" "${YGG_CNAME:?}" "${YGG_LABEL:?}" "${YGG_SLOTS:?}" "${YGG_ROOT:?}" D=${CM_DIR:?} set +e # every docker step is checked by hand: a container exits non-zero by design export LC_ALL=C t0=$(date +%s) LEASE_FDS=() unlease() { local lf; for lf in "${LEASE_FDS[@]}"; do exec {lf}>&-; done; LEASE_FDS=(); } cleanup() { local p; for p in $(jobs -p); do kill "$p" 2>/dev/null; done unlease # containers write as root [ -z "$D" ] || [ "$D" = / ] || sudo rm -rf -- "$D" } trap cleanup EXIT; trap 'exit 143' TERM INT HUP killed() { [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; } now() { date +%s; } # The shared volume is CARGO_HOME itself so sibling containers share cargo's package-cache lock; a registry-only mount left # the lock in each container's layer and concurrent unpacks removed each other's crate directories (2026-11-03). # Every container carries the job label (the stopgap's watchdog kills by it) and the same caps. common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}" -v "$YGG_WORK:/seed:ro" -v "$D:/cm:ro" -v /etc/machine-id:/etc/machine-id:ro -v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache -e BASH_ENV=/tmp/.ygg-bash-env -e CARGO_HOME=/cargo-home -e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "FILTERS=${FILTERS:-}" -e "CM_NAMES=${CM_NAMES:-}" -e "JOBS=$JOBS" -e "MIN_TEST_TIMEOUT=$MIN_TEST_TIMEOUT" -e "CONFIRM_TIMEOUT_MULT=$CONFIRM_TIMEOUT_MULT" -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*') [ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE") pre="printf 'set -o pipefail\n' >/tmp/.ygg-bash-env" # drun [-e flags]: one container (the half named by $MODE), its output to /stdout, its # status in /hostrc. A killed job starts nothing (125). DOCKER_ARGS is a flag string, split on purpose. # shellcheck disable=SC2086 drun() { local name=$1 out=$2 rc; shift 2 killed && return 125 mkdir -p "$out" sudo nice -n 10 docker run --rm --name "$name" "${common[@]}" -v "$out:/out" "$@" -w /src ${DOCKER_ARGS:-} "$YGG_IMAGE" \ bash -o pipefail -c "$pre; bash /cm/cargo-mutants.sh $MODE" >"$out/stdout" 2>&1 rc=$?; echo "$rc" >"$out/hostrc"; return "$rc" } killed_exit() { sum "JOB KILLED by the watchdog before the run finished; no totals"; exit 124; } mkdir -p "$D/items" "$D/plan" "$D/q" "$D/m" # ---- plan tp=$(now) MODE=--plan drun "$YGG_CNAME-plan" "$D/plan" rc=$? killed && killed_exit if [ "$rc" != 0 ]; then sum "ERROR the plan container exited $rc: the range diff or the mutant listing failed" tail -n 6 "$D/plan/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR |" exit 2 fi tp=$(( $(now) - tp )) if [ "${CM_NAMES:-}" = 1 ]; then sort -u "$D/names.txt" >"$D/m/universe" else sort -u "$D/plan/names.txt" >"$D/m/universe"; fi for f in s1.caught s1.unviable s1.surv s1.names; do : >"$D/m/$f"; done # lease : the containers of a stage = min(SHARDS, SLOTS, units, 1 + free slots now); sets K and holds the leases lease() { local want=$1 i lf psi [ "$want" -le "$SHARDS" ] || want=$SHARDS [ "$want" -le "$YGG_SLOTS" ] || want=$YGG_SLOTS K=1; [ "$want" -gt 1 ] || return 0 psi=$(awk '$1 == "full" { for (i = 2; i <= NF; i++) if ($i ~ /^avg60=/) { sub("avg60=", "", $i); print $i } }' "${CM_PSI:-/proc/pressure/io}" 2>/dev/null) if awk -v p="${psi:-0}" 'BEGIN { exit !(p > 20) }'; then return 0; fi for i in $(seq 1 "$YGG_SLOTS"); do [ "$K" -lt "$want" ] || break exec {lf}>"$YGG_ROOT/slot-$i.lock" if flock -n "$lf"; then LEASE_FDS+=("$lf"); K=$((K + 1)); else exec {lf}>&-; fi done } # run_stage : K workers claim items 1..count by mkdir, in order (largest first) run_stage() { local stage=$1 count=$2 fn=$3 w pids=() for w in $(seq 1 "$K"); do ( for lf in "${LEASE_FDS[@]}"; do exec {lf}>&-; done # the lease belongs to the driver, never to a child (2026-09-23 inherited-fd scar) i=0 while [ "$i" -lt "$count" ]; do i=$((i + 1)) killed && break [ ! -e "$D/failed" ] || break mkdir "$D/q/$stage-$i" 2>/dev/null || continue "$fn" "$i" done ) & pids+=($!) done wait "${pids[@]}" } container_error() { # container_error