cargo-mutants.sh: a failed build under the image's rustc wrapper retries once without it

Stage containers sharing the eureka-sccache volume died mid-build with cargo's
"could not execute process ... sccache ... (never executed)" or a missing dep-info
file in their own target dir, though the image's sccache is present, executes, and
a lone container builds clean. The wrapper only caches, so a failed build with
RUSTC_WRAPPER set is retried once without it; the job summary carries a NOTE line
saying which way it went, the first attempt's log is kept, and a build dead either
way stays a red baseline (exit 4) with its cause. Container errors now tail 25
lines, not 5. Tests: dead wrapper recovers with NOTE; red with and without the
wrapper stays red.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tools/cargo-mutants.sh

@@ -208,7 +208,7 @@ if [ "$1" = --host ]; then
   container_error() {   # container_error <label> <out dir>
     local rc; rc=$(cat "$2/hostrc" 2>/dev/null)
     sum "ERROR $1 container exited ${rc:-?} (4 is a red baseline: the unmutated tests failed there)"
-    tail -n 5 "$2/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR   |"
+    tail -n 25 "$2/stdout" 2>/dev/null | sed "s|^|$SUMTAG: ERROR   |"
   }
 
   # ---- stage 1: files whose filter selects tests (the plan container derived each file's filter and counted its tests)
@@ -251,7 +251,7 @@ if [ "$1" = --host ]; then
       while IFS= read -r l; do sum "$l"; done
   fi
   for n in $(seq 1 "$s1n"); do
-    o="$D/items/s1-$n/out"
+    o="$D/items/s1-$n/out"; [ ! -s "$o/note" ] || sum "NOTE stage 1 (${S1_FILE[$n]}): $(cat "$o/note")"
     cat "$D/items/s1-$n.names" >>"$D/m/s1.names"
     cat "$o/caught.txt" >>"$D/m/s1.caught"; cat "$o/unviable.txt" >>"$D/m/s1.unviable"
     cat "$o/missed.txt" "$o/timeout.txt" >>"$D/m/s1.surv"
@@ -286,7 +286,7 @@ if [ "$1" = --host ]; then
     ts2=$(( $(now) - ts2 ))
     bad=0
     for k in $(seq 1 "$K2"); do
-      o="$D/items/s2-$k/out"
+      o="$D/items/s2-$k/out"; [ ! -s "$o/note" ] || sum "NOTE stage 2 (shard $k of $K2): $(cat "$o/note")"
       if [ "$(cat "$o/hostrc" 2>/dev/null)" != 0 ]; then
         [ -e "$D/q/s2-$k" ] || continue
         container_error "stage 2 (shard $k of $K2)" "$o"; bad=1; continue
@@ -378,10 +378,25 @@ else
   case $CM_SHARD in */1) ;; *) args+=(--shard "$CM_SHARD") ;; esac
   [ "${#targs[@]}" = 0 ] || tail_args=(-- -- "${targs[@]}")
 fi
-cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1
-rc=$?
+run_cargo() { cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1; rc=$?; }
+run_cargo
+# The image's rustc wrapper (sccache, one cache volume shared by sibling containers) is a cache, never an input. 2026-11-02:
+# builds died mid-way in sibling containers, cargo reporting the wrapper "(never executed)" or a dep-info file of its own
+# target dir missing (ENOENT), while the image's sccache runs fine alone and RUSTC_WRAPPER= builds clean. A failed build
+# with a wrapper set is therefore retried once without it, and the job summary says which way it went. The failure itself
+# is not diagnosed here: the first attempt's log is kept as cargo-wrapper.log in the job's output.
+if [ "$rc" != 0 ] && [ "$rc" != 2 ] && [ "$rc" != 3 ] && [ -n "${RUSTC_WRAPPER:-}" ]; then
+  w=$RUSTC_WRAPPER; rc1=$rc; cp "$OUT/cargo.log" "$OUT/cargo-wrapper.log"
+  { echo "DIAG wrapper=$w"; ls -l "$w"; "$w" --version; df -h /tmp /sccache; free -m; grep -m3 -E "never executed|No such file" "$OUT/cargo.log"; } >"$OUT/diag.txt" 2>&1
+  unset RUSTC_WRAPPER
+  run_cargo
+  case $rc in 0|2|3) echo "RUSTC_WRAPPER=$w: the build failed with it (exit $rc1) and passed without it; the run continued without it" >"$OUT/note" ;;
+    *) echo "RUSTC_WRAPPER=$w: the build failed with it (exit $rc1) and without it (exit $rc): the wrapper was not the cause" >>"$OUT/cargo.log" ;; esac
+fi
 for l in caught missed timeout unviable; do cp "$o/mutants.out/$l.txt" "$OUT/$l.txt" 2>/dev/null || : >"$OUT/$l.txt"; done
 echo "$rc" >"$OUT/rc"
+[ ! -s "$OUT/diag.txt" ] || cat "$OUT/diag.txt"
+[ ! -s "$OUT/note" ] || cat "$OUT/note"
 tail -n 30 "$OUT/cargo.log"
 # 0 clean, 2 missed and 3 timeouts are normal; 4 (red baseline) and everything else are errors
 case $rc in 0|2|3) exit 0 ;; *) exit "$rc" ;; esac

tools/fixtures/mutation-mock/fakebin/cargo

@@ -62,6 +62,8 @@ open=$(perl -MPOSIX -e '$n = 0; for (10 .. 40) { my $d = POSIX::dup($_); if (def
 [ -z "${FAKE_CM_KILLS:-}" ] || touch "$FAKE_CM_KILLS"   # a watchdog kill arriving while a run is going
 echo "stage=$stage filter=$filter harness=${harness[*]} openfds=$open mult=$mult min=$mintimeout" >>"${FAKE_CM_LOG:?}"
 [ "${FAKE_CM_RED:-}" != "$stage" ] || { echo "baseline failed" >&2; exit 4; }
+# FAKE_CM_DEADWRAP=1: a run whose environment still holds RUSTC_WRAPPER dies like cargo with a wrapper that cannot execute.
+[ -z "${FAKE_CM_DEADWRAP:-}" ] || [ -z "${RUSTC_WRAPPER:-}" ] || { echo "  Caused by: could not execute process $RUSTC_WRAPPER rustc (never executed)"; exit 101; }
 mkdir -p "$out/mutants.out/log"
 col=3; [ "$stage" = 2 ] || col=2
 for o in caught missed timeout unviable; do

tools/test-tools.sh

@@ -343,6 +343,13 @@ if grep '^stage=2' "$cm_log" | grep -q 'mult=9 min=33$' && ! grep '^stage=2' "$c
 # slot lease: the job's own slot is held by ygg-verify (an inherited descriptor), others by other jobs
 cm A HOLD_OWN=1 HOLD_OTHERS=1; cmcheck lease-free 'containers=3 |'; cmlocks lease-free
 cm A HOLD_OWN=1 HOLD_OTHERS=3; cmcheck lease-one-free 'containers=2 |'
+# a rustc wrapper that cannot execute: the build retries once without it and the job summary says so; a wrapper-free failure is not blamed on it
+cm C FAKE_CM_DEADWRAP=1 RUSTC_WRAPPER=/nonexistent/sccache
+cmrc C-dead-wrapper 0
+cmcheck C-dead-wrapper "NOTE stage 2 (shard 1 of 1): RUSTC_WRAPPER=/nonexistent/sccache: the build failed with it" "MUTANTS tested=2 caught=2"
+cm C FAKE_CM_RED=2 RUSTC_WRAPPER=/nonexistent/sccache
+[ "$cm_rc" != 0 ] || { echo "FAIL cargo-mutants C-red-with-wrapper: exit 0"; fail=1; }
+cmcheck C-red-with-wrapper "ERROR stage 2 (shard 1 of 1)" "!MUTANTS"
 cm A HOLD_OWN=1 HOLD_OTHERS=4; cmcheck lease-none-free 'containers=1 |' 'tested=16'; cmlocks lease-none-free
 open_none=$(grep -o 'openfds=[0-9]*' "$cm_log" | sort -u | paste -sd, -)
 cm A HOLD_OWN=1 HOLD_OTHERS=1; open_leased=$(grep -o 'openfds=[0-9]*' "$cm_log" | sort -u | paste -sd, -)