Eureka/tools/cargo-mutants.sh

#!/usr/bin/env bash
# cargo-mutants on Yggdrasil for a revision range: ONE verify job, per-file scoped
# tests, survivors confirmed on the full suite, sharded over free slots, one merged
# survivor list. The one owner of "run cargo-mutants" (no brief, no job script runs
# an inline `cargo mutants`).
#
#   SUMMARY=<log> tools/cargo-mutants.sh <local-repo> <rev> <base> [workdir]
#     workdir   directory inside the repo where cargo runs (a package or workspace
#               member, e.g. crates/mind-body; CultLib has no Cargo.toml at its root).
#               The mutated range is `git diff --relative <base> <rev>` taken at the
#               WORKSPACE ROOT: cargo-mutants matches --in-diff paths, -f paths and
#               prints mutant names relative to the workspace root, not to workdir.
#     env       FILTERS  '<path>=<filter>,...' overrides a file's derived test filter
#                        (path as the mutant names print it); '-' = full suite
#               TEST_ARGS  harness args, e.g. '--skip flaky_one' (a flaky test would
#                        make a surviving mutant read as killed in one run)
#               NAMES    local file of mutant names (one per line, copied from a
#                        previous MISSED/TIMEOUT list): skips stage 1 and the listing
#                        and runs only those names on the full suite. This is the rerun
#                        after killing survivors with tests.
#               SHARDS (3) JOBS (2) MIN_TEST_TIMEOUT (20) CONFIRM_TIMEOUT_MULT (5)
#               TIMEOUT  seconds for the whole job (default 10800)
#               everything else tools/stopgap/ygg-verify.sh reads (MEM, CPUS, ...)
#     Typical, from an agent: launch as ONE background shell with a fresh log, then
#       tools/stopgap/ygg-verify.sh wait <log>   (Bash tool timeout >= 280000 ms)
#     Run it ONCE per pass, at the end, on the final spelling of the code. Chasing
#     survivors is a NAMES rerun, never a second whole-range sweep.
#
# STAGES (all inside the one job and the one host driver):
#   plan     one container writes the range diff, lists every mutant in it, derives each
#            touched file's test filter and counts the tests that filter selects
#            (`cargo test -- <filter> --list`, one build for all files).
#   stage 1  each touched file whose filter selects tests runs its mutants against that
#            file's tests only (`cargo mutants -f <file> -- <filter>`). The filter is
#            derived from the path: the module path after the last src/ (a/mod.rs ->
#            `a::`, a/b.rs -> `a::b::`); lib.rs, main.rs, bin/ and files outside src/
#            have none. FILTERS overrides. Stage 1 decides only CAUGHT (its own green
#            filtered baseline guards false kills) and UNVIABLE (a build fact). Its
#            MISSED and TIMEOUT are never reported: a mutant that only another module's
#            tests or tests/ kill survives a filtered run. A file whose filter selects 0
#            tests (tests=0) never runs stage 1: all its mutants go to stage 2.
#   stage 2  every name stage 1 did not settle (survivors, files without a filter,
#            zero-test files, or NAMES) runs on the full suite in cargo-mutants' own
#            baseline, sharded over the containers. Stage 2 ALWAYS runs: that baseline is
#            the only full-suite check, so a red suite is a red job even when stage 1
#            settled every mutant (one name then rides along; its outcome is dropped).
#            The test timeout is max(MIN_TEST_TIMEOUT, CONFIRM_TIMEOUT_MULT x baseline);
#            the generous default keeps a slow-but-finishing test from flipping a
#            MISSED mutant to TIMEOUT with the host's load. Its outcome is final.
#   merge    each name counted once; MISSED and TIMEOUT are stage-2 outcomes only.
# Containers per stage: min(SHARDS, SLOTS, units, 1 + free slots now), units being the
# stage's files (stage 1) or names (stage 2). The job holds one slot (ygg-verify's);
# the driver leases further slot locks with flock -n, never waits for one, takes none
# while /proc/pressure/io full avg60 > 20, and releases them when each stage ends. A
# killed job (YGG_KILLED) starts nothing further.
#
# OUTPUT: the YGG-SUMMARY/$YGG_NONCE lines ygg-verify's summary carries; per-container
# logs stay on the host and go with the job.
#   FILTER <file> <filter> tests=<n> mutants=<m>      one per stage-1 file (zero-test first)
#   MUTANTS tested=N caught=C missed=M timeout=T unviable=U | stage1 filtered=F
#     survivors=S confirmed_caught=X | containers=K | plan=..s stage1=..s/<k1> stage2=..s/<k2> | <m>m
#   MISSED <name> / TIMEOUT <name>                    final outcomes, at most 50
#   ERROR ...      a red baseline, a container that failed, mutants with no outcome:
#                  the job exits non-zero and prints no MUTANTS line
# Exit 0 when every stage ran on a green baseline; survivors do not fail the job (the
# score is not a gate). cargo-mutants exits 2 (missed) and 3 (timeouts) normally; 4 is a
# red baseline; anything else is an error.
#
# DELETION LINE: this file dies with tools/stopgap/ygg-verify.sh when `idunn verify`
# ships a typed mutation step. Modes in this one file: no flag = launcher on the
# workstation; --host = the driver on the Yggdrasil host (ygg-verify.sh RUN_ON_HOST=1);
# --plan / --item = the work inside a container.
set -euo pipefail

SUMTAG="YGG-SUMMARY${YGG_NONCE:+/$YGG_NONCE}"
sum() { printf '%s: %s\n' "$SUMTAG" "$*"; }

# ---- --escape: mutant names on stdin -> one anchored -F pattern per line ----------------------
# Only regex metacharacters are escaped, so names holding * + ( ) | stay literal; ^...$ pins the whole name.
if [ "${1:-}" = --escape ]; then
  exec perl -pe 'chomp; s/([\\.+*?()|\[\]{}^\$])/\\$1/g; $_ = "^$_\$\n"'
fi

# ---- launcher (workstation) -------------------------------------------------
if [ "${1:-}" != --host ] && [ "${1:-}" != --plan ] && [ "${1:-}" != --item ]; then
  here=$(cd "$(dirname "$0")" && pwd)
  repo=${1:?usage: cargo-mutants.sh <local-repo> <rev> <base> [workdir]}
  rev=${2:?revision}; base=${3:?base revision}; workdir=${4:-.}
  : "${SHARDS:=3}" "${JOBS:=2}" "${MIN_TEST_TIMEOUT:=20}" "${CONFIRM_TIMEOUT_MULT:=5}" "${TIMEOUT:=10800}"
  for v in SHARDS JOBS MIN_TEST_TIMEOUT TIMEOUT; do
    case ${!v} in ''|*[!0-9]*|0) echo "cargo-mutants: $v must be a positive integer" >&2; exit 2 ;; esac
  done
  case $CONFIRM_TIMEOUT_MULT in ''|*[!0-9.]*|.*|*.|*.*.*) echo "cargo-mutants: CONFIRM_TIMEOUT_MULT must be a positive number" >&2; exit 2 ;; esac
  if [ -n "${FILTERS:-}" ]; then
    old=$IFS; IFS=,
    for f in $FILTERS; do
      case $f in ?*=?*) ;; *) IFS=$old; echo "cargo-mutants: FILTERS entries are <path>=<filter>" >&2; exit 2 ;; esac
    done
    IFS=$old
  fi
  sha_rev=$(git -C "$repo" rev-parse --verify "$rev^{commit}") || { echo "cargo-mutants: unknown revision" >&2; exit 2; }
  sha_base=$(git -C "$repo" rev-parse --verify "$base^{commit}") || { echo "cargo-mutants: unknown base revision" >&2; exit 2; }
  git -C "$repo" merge-base --is-ancestor "$sha_base" "$sha_rev" || { echo "cargo-mutants: base is not an ancestor of rev (the verify mirror carries only rev's history)" >&2; exit 2; }
  stage=$(mktemp -d)
  trap 'rm -rf "$stage"' EXIT
  cp "$here/cargo-mutants.sh" "$stage/"
  files=(cargo-mutants.sh)
  if [ -n "${NAMES:-}" ]; then
    [ -r "$NAMES" ] || { echo "cargo-mutants: cannot read the NAMES file" >&2; exit 2; }
    tr -d '\r' <"$NAMES" >"$stage/names.txt"; files+=(names.txt)
  fi
  bundle=$(tar -C "$stage" -cz "${files[@]}" | base64 | tr -d '\n')
  # The driver runs on the host: it unpacks the bundle into its own directory (mounted read-only into every container) and removes it when done.
  cmd=$(printf 'd=$(mktemp -d) && echo %s | base64 -d | tar -xz -C "$d" && WORKDIR=%q BASE=%q FILTERS=%q TEST_ARGS=%q SHARDS=%q JOBS=%q MIN_TEST_TIMEOUT=%q CONFIRM_TIMEOUT_MULT=%q CM_NAMES=%q CM_DIR="$d" bash "$d/cargo-mutants.sh" --host' \
    "$bundle" "$workdir" "$sha_base" "${FILTERS:-}" "${TEST_ARGS:-}" "$SHARDS" "$JOBS" "$MIN_TEST_TIMEOUT" "$CONFIRM_TIMEOUT_MULT" "${NAMES:+1}")
  RUN_ON_HOST=1 TIMEOUT=$TIMEOUT "$here/stopgap/ygg-verify.sh" "$repo" "$sha_rev" rust "$cmd"
  exit $?
fi

# ---- driver (Yggdrasil host, inside the job's slot) --------------------------
if [ "$1" = --host ]; then
  : "${YGG_WORK:?not started by ygg-verify.sh RUN_ON_HOST=1}" "${YGG_IMAGE:?}" "${YGG_CNAME:?}" "${YGG_LABEL:?}" "${YGG_SLOTS:?}" "${YGG_ROOT:?}"
  D=${CM_DIR:?}
  set +e   # every docker step is checked by hand: a container exits non-zero by design
  export LC_ALL=C
  t0=$(date +%s)
  LEASE_FDS=()
  unlease() { local lf; for lf in "${LEASE_FDS[@]}"; do exec {lf}>&-; done; LEASE_FDS=(); }
  cleanup() {
    local p; for p in $(jobs -p); do kill "$p" 2>/dev/null; done
    unlease
    # containers write as root
    [ -z "$D" ] || [ "$D" = / ] || sudo rm -rf -- "$D"
  }
  trap cleanup EXIT; trap 'exit 143' TERM INT HUP
  killed() { [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; }
  now() { date +%s; }
  # The shared volume is CARGO_HOME itself so sibling containers share cargo's package-cache lock; a registry-only mount left
  # the lock in each container's layer and concurrent unpacks removed each other's crate directories (2026-11-03).
  # Every container carries the job label (the stopgap's watchdog kills by it) and the same caps.
  common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}"
    -v "$YGG_WORK:/seed:ro" -v "$D:/cm:ro" -v /etc/machine-id:/etc/machine-id:ro
    -v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache
    -e BASH_ENV=/tmp/.ygg-bash-env -e CARGO_HOME=/cargo-home
    -e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "FILTERS=${FILTERS:-}" -e "CM_NAMES=${CM_NAMES:-}" -e "JOBS=$JOBS"
    -e "MIN_TEST_TIMEOUT=$MIN_TEST_TIMEOUT" -e "CONFIRM_TIMEOUT_MULT=$CONFIRM_TIMEOUT_MULT"
    -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*')
  [ -z "${YGG_NONCE:-}" ] || common+=(-e "YGG_NONCE=$YGG_NONCE")
  pre="printf 'set -o pipefail\n' >/tmp/.ygg-bash-env"
  # drun <name> <out dir> [-e flags]: one container (the half named by $MODE), its output to <out>/stdout, its
  # status in <out>/hostrc. A killed job starts nothing (125). DOCKER_ARGS is a flag string, split on purpose.
  # shellcheck disable=SC2086
  drun() {
    local name=$1 out=$2 rc; shift 2
    killed && return 125
    mkdir -p "$out"
    sudo nice -n 10 docker run --rm --name "$name" "${common[@]}" -v "$out:/out" "$@" -w /src ${DOCKER_ARGS:-} "$YGG_IMAGE" \
      bash -o pipefail -c "$pre; bash /cm/cargo-mutants.sh $MODE" >"$out/stdout" 2>&1
    rc=$?; echo "$rc" >"$out/hostrc"; return "$rc"
  }
  killed_exit() { sum "JOB KILLED by the watchdog before the run finished; no totals"; exit 124; }
  mkdir -p "$D/items" "$D/plan" "$D/q" "$D/m"

  # ---- plan
  tp=$(now)
  MODE=--plan drun "$YGG_CNAME-plan" "$D/plan"
  rc=$?
  killed && killed_exit
  if [ "$rc" != 0 ]; then
    sum "ERROR the plan container exited $rc: the range diff or the mutant listing failed"
    tail -n 6 "$D/plan/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR   |"
    exit 2
  fi
  tp=$(( $(now) - tp ))
  if [ "${CM_NAMES:-}" = 1 ]; then sort -u "$D/names.txt" >"$D/m/universe"
  else sort -u "$D/plan/names.txt" >"$D/m/universe"; fi
  for f in s1.caught s1.unviable s1.surv s1.names; do : >"$D/m/$f"; done

  # lease <units>: the containers of a stage = min(SHARDS, SLOTS, units, 1 + free slots now); sets K and holds the leases
  lease() {
    local want=$1 i lf psi
    [ "$want" -le "$SHARDS" ] || want=$SHARDS
    [ "$want" -le "$YGG_SLOTS" ] || want=$YGG_SLOTS
    K=1; [ "$want" -gt 1 ] || return 0
    psi=$(awk '$1 == "full" { for (i = 2; i <= NF; i++) if ($i ~ /^avg60=/) { sub("avg60=", "", $i); print $i } }' "${CM_PSI:-/proc/pressure/io}" 2>/dev/null)
    if awk -v p="${psi:-0}" 'BEGIN { exit !(p > 20) }'; then return 0; fi
    for i in $(seq 1 "$YGG_SLOTS"); do
      [ "$K" -lt "$want" ] || break
      exec {lf}>"$YGG_ROOT/slot-$i.lock"
      if flock -n "$lf"; then LEASE_FDS+=("$lf"); K=$((K + 1)); else exec {lf}>&-; fi
    done
  }
  # run_stage <stage> <count> <fn>: K workers claim items 1..count by mkdir, in order (largest first)
  run_stage() {
    local stage=$1 count=$2 fn=$3 w pids=()
    for w in $(seq 1 "$K"); do
      ( for lf in "${LEASE_FDS[@]}"; do exec {lf}>&-; done   # the lease belongs to the driver, never to a child (2026-09-23 inherited-fd scar)
        i=0
        while [ "$i" -lt "$count" ]; do
          i=$((i + 1))
          killed && break
          [ ! -e "$D/failed" ] || break
          mkdir "$D/q/$stage-$i" 2>/dev/null || continue
          "$fn" "$i"
        done ) &
      pids+=($!)
    done
    wait "${pids[@]}"
  }
  container_error() {   # container_error <label> <out dir>
    local rc; rc=$(cat "$2/hostrc" 2>/dev/null)
    sum "ERROR $1 container exited ${rc:-?} (4 is a red baseline: the unmutated tests failed there)"
    tail -n 25 "$2/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR   |"
  }

  # ---- stage 1: files whose filter selects tests (the plan container derived each file's filter and counted its tests)
  S1_FILE=(); S1_FILTER=(); s1n=0
  if [ "${CM_NAMES:-}" != 1 ]; then
    while IFS=$'\t' read -r cnt file f t; do      # files by mutant count, largest first
      [ "$t" != 0 ] || continue                   # a filter that selects no test settles nothing: straight to stage 2
      s1n=$((s1n + 1)); S1_FILE[$s1n]=$file; S1_FILTER[$s1n]=$f
      awk -F: -v f="$file" '$1 == f' "$D/m/universe" >"$D/items/s1-$s1n.names"
    done <"$D/plan/filters.txt"
  fi
  s1_item() {
    local n=$1 rc
    MODE=--item drun "$YGG_CNAME-s1-$n" "$D/items/s1-$n/out" -e CM_STAGE=1 -e "CM_FILE=${S1_FILE[$n]}" -e "CM_FILTER=${S1_FILTER[$n]}"
    rc=$?
    if [ "$rc" != 0 ] && ! killed; then touch "$D/failed"; fi
  }
  K1=0; ts1=0
  if [ "$s1n" -gt 0 ]; then
    ts1=$(now)
    lease "$s1n"; K1=$K
    run_stage s1 "$s1n" s1_item
    unlease
    killed && killed_exit
    ts1=$(( $(now) - ts1 ))
    bad=0
    for n in $(seq 1 "$s1n"); do
      o="$D/items/s1-$n/out"
      if [ "$(cat "$o/hostrc" 2>/dev/null)" != 0 ]; then
        [ -e "$D/q/s1-$n" ] || continue   # never started: the job is already red for another file
        container_error "stage 1 (${S1_FILE[$n]})" "$o"; bad=1; continue
      fi
    done
    if [ "$bad" != 0 ] || [ -e "$D/failed" ]; then exit 1; fi
  fi
  # FILTER lines: the anomalies (0 tests) first, then the rest, at most 12 in all
  if [ "${CM_NAMES:-}" != 1 ]; then
    { awk -F'\t' '$4 == 0' "$D/plan/filters.txt"; awk -F'\t' '$4 != 0' "$D/plan/filters.txt"; } |
      awk -F'\t' '{ if (NR <= 12) printf "FILTER %s %s tests=%s mutants=%s\n", $2, $3, $4, $1; else h++ } END { if (h) printf "FILTER ... %d more files\n", h }' |
      while IFS= read -r l; do sum "$l"; done
  fi
  for n in $(seq 1 "$s1n"); do
    o="$D/items/s1-$n/out"
    cat "$D/items/s1-$n.names" >>"$D/m/s1.names"
    cat "$o/caught.txt" >>"$D/m/s1.caught"; cat "$o/unviable.txt" >>"$D/m/s1.unviable"
    cat "$o/missed.txt" "$o/timeout.txt" >>"$D/m/s1.surv"
  done
  # only names of this run's range count; stage 1 settles caught and unviable, nothing else
  for f in s1.caught s1.unviable s1.surv s1.names; do
    sort -u "$D/m/$f" | comm -12 - "$D/m/universe" >"$D/m/$f.t" && mv "$D/m/$f.t" "$D/m/$f"
  done
  cat "$D/m/s1.caught" "$D/m/s1.unviable" | sort -u >"$D/m/s1.settled"
  comm -23 "$D/m/universe" "$D/m/s1.settled" >"$D/s2.txt"
  # Stage 2 always runs, so cargo-mutants' full-suite baseline always does (cargo-mutants runs no baseline for an
  # empty selection). With every name settled it carries one name; the merge drops that name's outcome.
  if [ -s "$D/s2.txt" ]; then bash "$D/cargo-mutants.sh" --escape <"$D/s2.txt" >"$D/s2.re"
  else head -n 1 "$D/m/universe" | bash "$D/cargo-mutants.sh" --escape >"$D/s2.re"; fi

  # ---- stage 2: the full suite on everything stage 1 did not settle
  s2n=$(wc -l <"$D/s2.re" | tr -d " ")
  K2=0; ts2=0
  for l in caught missed timeout unviable; do : >"$D/m/s2.$l"; done
  s2_shard() {
    local k=$1 rc
    MODE=--item drun "$YGG_CNAME-s2-$k" "$D/items/s2-$k/out" -e CM_STAGE=2 -e "CM_SHARD=$((k - 1))/$K2"
    rc=$?
    if [ "$rc" != 0 ] && ! killed; then touch "$D/failed"; fi
  }
  if [ "$s2n" -gt 0 ]; then
    ts2=$(now)
    lease "$s2n"; K2=$K
    run_stage s2 "$K2" s2_shard
    unlease
    killed && killed_exit
    ts2=$(( $(now) - ts2 ))
    bad=0
    for k in $(seq 1 "$K2"); do
      o="$D/items/s2-$k/out"
      if [ "$(cat "$o/hostrc" 2>/dev/null)" != 0 ]; then
        [ -e "$D/q/s2-$k" ] || continue
        container_error "stage 2 (shard $k of $K2)" "$o"; bad=1; continue
      fi
      for l in caught missed timeout unviable; do cat "$o/$l.txt" >>"$D/m/s2.$l"; done
    done
    if [ "$bad" != 0 ] || [ -e "$D/failed" ]; then exit 1; fi
  fi

  # ---- merge: a name's outcome is stage 2's where it ran, else stage 1's; each name once
  for l in caught missed timeout unviable; do
    sort -u "$D/m/s2.$l" | comm -12 - "$D/s2.txt" >"$D/m/s2.$l.t" && mv "$D/m/s2.$l.t" "$D/m/s2.$l"
  done
  cat "$D/m/s2.caught" "$D/m/s2.missed" "$D/m/s2.timeout" "$D/m/s2.unviable" | sort -u >"$D/m/s2.all"
  nout=$(comm -23 "$D/s2.txt" "$D/m/s2.all" | wc -l | tr -d ' ')
  if [ "$nout" != 0 ]; then
    sum "ERROR $nout mutants selected for the full-suite run have no outcome in any shard (a name unknown to cargo-mutants at this revision, or a shard that died)"
    exit 1
  fi
  C=$(cat "$D/m/s1.caught" "$D/m/s2.caught" | sort -u | wc -l | tr -d ' ')
  U=$(cat "$D/m/s1.unviable" "$D/m/s2.unviable" | sort -u | wc -l | tr -d ' ')
  M=$(wc -l <"$D/m/s2.missed" | tr -d ' '); T=$(wc -l <"$D/m/s2.timeout" | tr -d ' ')
  S=$(wc -l <"$D/m/s1.surv" | tr -d ' '); F=$(wc -l <"$D/m/s1.names" | tr -d ' ')
  X=$(comm -12 "$D/m/s1.surv" "$D/m/s2.caught" | wc -l | tr -d ' ')
  K=$K1; [ "$K2" -le "$K" ] || K=$K2
  sum "MUTANTS tested=$((C + M + T + U)) caught=$C missed=$M timeout=$T unviable=$U | stage1 filtered=$F survivors=$S confirmed_caught=$X | containers=$K | plan=${tp}s stage1=${ts1}s/${K1} stage2=${ts2}s/${K2} | $(awk -v e="$(( $(now) - t0 ))" 'BEGIN { printf "%.1f", e / 60 }')m"
  { sed 's/^/MISSED /' "$D/m/s2.missed"; sed 's/^/TIMEOUT /' "$D/m/s2.timeout"; } >"$D/m/surv.txt"
  total=$(wc -l <"$D/m/surv.txt" | tr -d ' ')
  head -n 50 "$D/m/surv.txt" | while IFS= read -r l; do sum "$l"; done
  if [ "$total" -gt 50 ]; then
    sum "... $((total - 50)) more MISSED/TIMEOUT names; they follow in the job log as cm-survivor lines"
    tail -n +51 "$D/m/surv.txt" | sed 's/^/cm-survivor: /'
  fi
  exit 0
fi

# ---- inside a container -----------------------------------------------------
set +e
# overridable only so tools/test-tools.sh can run this half against a mock
SRC=${CM_SRC:-/src}; CM=${CM_DIR:-/cm}; OUT=${CM_OUT:-/out}; SEED=${CM_SEED:-/seed}
if [ -d "$SEED" ]; then mkdir -p "$SRC" && cp -a "$SEED/." "$SRC/" || { echo "cannot copy $SEED into $SRC"; exit 2; }; fi
cd "$SRC/${WORKDIR:-.}" || { echo "BAD WORKDIR"; exit 2; }
set -f; read -ra targs <<<"${TEST_ARGS:-}"; set +f
if [ "$1" = --plan ]; then
  wsroot=$(dirname "$(cargo locate-project --workspace --message-format plain)") || { echo "cargo locate-project failed"; exit 2; }
  git -C "$wsroot" diff --relative "$BASE" HEAD >"$OUT/d" || { echo "git diff of the range failed"; exit 2; }
  cargo mutants --list --in-diff "$OUT/d" >"$OUT/names.txt" 2>"$OUT/list.err" || { echo "cargo mutants --list failed"; tail -n 5 "$OUT/list.err"; exit 2; }
  : >"$OUT/filters.txt"
  [ "$CM_NAMES" != 1 ] || exit 0
  # stage 1's plan: each touched file's test filter and how many tests it selects (0 = nothing for stage 1 to run)
  declare -A fover
  if [ -n "${FILTERS:-}" ]; then
    IFS=, read -ra fl <<<"$FILTERS"
    for e in "${fl[@]}"; do fover[${e%%=*}]=${e#*=}; done
  fi
  derive_filter() {   # derive_filter <path>: the test filter of a file, nothing for the full suite
    local p=$1 rest
    if [ -n "${fover[$p]+x}" ]; then [ "${fover[$p]}" = - ] || echo "${fover[$p]}"; return 0; fi
    case $p in */src/*.rs|src/*.rs) ;; *) return 0 ;; esac
    rest=${p##*src/}; rest=${rest%.rs}
    case $rest in lib|main|bin/*|*/bin/*) return 0 ;; esac
    rest=${rest%/mod}
    echo "${rest//\//::}::"
  }
  built=0
  while IFS=$'\t' read -r cnt file; do      # files by mutant count, largest first
    f=$(derive_filter "$file")
    [ -n "$f" ] || continue
    if [ "$built" = 0 ]; then
      cargo test --no-run >"$OUT/build.log" 2>&1 || { echo "cargo test --no-run failed"; tail -n 5 "$OUT/build.log"; exit 2; }
      built=1
    fi
    t=$(cargo test -- "$f" "${targs[@]}" --list 2>/dev/null | awk '/^[0-9]+ tests?, / { s += $1 } END { print s + 0 }')
    printf '%s\t%s\t%s\t%s\n' "$cnt" "$file" "$f" "$t" >>"$OUT/filters.txt"
  done < <(awk -F: '{ c[$1]++ } END { for (f in c) print c[f] "\t" f }' "$OUT/names.txt" | sort -t "$(printf '\t')" -k1,1nr -k2)
  exit 0
fi
# --item: stage 1 (CM_FILE with its CM_FILTER) or stage 2 (shard CM_SHARD of the names in $CM/s2.re)
o=${CM_TMP:-/tmp}/cmo
args=(mutants --in-diff "$CM/plan/d" -o "$o" --jobs "$JOBS" --minimum-test-timeout "$MIN_TEST_TIMEOUT")
tail_args=()
if [ "$CM_STAGE" = 1 ]; then
  args+=(-f "$CM_FILE")
  tail_args=(-- "$CM_FILTER"); [ "${#targs[@]}" = 0 ] || tail_args+=(-- "${targs[@]}")
else
  mapfile -t res <"$CM/s2.re"
  for r in "${res[@]}"; do args+=(-F "$r"); done
  args+=(--timeout-multiplier "$CONFIRM_TIMEOUT_MULT")
  case $CM_SHARD in */1) ;; *) args+=(--shard "$CM_SHARD") ;; esac
  [ "${#targs[@]}" = 0 ] || tail_args=(-- -- "${targs[@]}")
fi
cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1
rc=$?
for l in caught missed timeout unviable; do cp "$o/mutants.out/$l.txt" "$OUT/$l.txt" 2>/dev/null || : >"$OUT/$l.txt"; done
echo "$rc" >"$OUT/rc"
tail -n 30 "$OUT/cargo.log"
# 0 clean, 2 missed and 3 timeouts are normal; 4 (red baseline) and everything else are errors
case $rc in 0|2|3) exit 0 ;; *) exit "$rc" ;; esac