Verify containers share cargo's package-cache lock: the shared volume is CARGO_HOME, and the wrapper retry goes

A registry-only mount at /usr/local/cargo/registry left the package-cache lock under CARGO_HOME=/usr/local/cargo in each
container's own layer, so sibling containers (cargo-mutants stages, concurrent ygg-verify jobs) shared no lock and removed
each other's half-unpacked crate directories. Reproduced on Yggdrasil from an empty registry: 15 of 18 containers failed
with a registry mount, 15 of 18 with RUSTC_WRAPPER= too, 0 of 18 with the volume as CARGO_HOME.

ygg-verify.sh and cargo-mutants.sh mount eureka-cargo-home at /cargo-home and set CARGO_HOME and PATH for the rust image.
Removed: the retry without RUSTC_WRAPPER, its note and diag plumbing, and its two tests and the FAKE_CM_DEADWRAP switch.
Kept: container errors tail 25 lines.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tools/cargo-mutants.sh

@@ -135,11 +135,13 @@ if [ "$1" = --host ]; then
   trap cleanup EXIT; trap 'exit 143' TERM INT HUP
   killed() { [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; }
   now() { date +%s; }
+  # The shared volume is CARGO_HOME itself so sibling containers share cargo's package-cache lock; a registry-only mount left
+  # the lock in each container's layer and concurrent unpacks removed each other's crate directories (2026-11-03).
   # Every container carries the job label (the stopgap's watchdog kills by it) and the same caps.
   common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}"
     -v "$YGG_WORK:/seed:ro" -v "$D:/cm:ro" -v /etc/machine-id:/etc/machine-id:ro
-    -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache
-    -e BASH_ENV=/tmp/.ygg-bash-env
+    -v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache
+    -e BASH_ENV=/tmp/.ygg-bash-env -e CARGO_HOME=/cargo-home -e PATH=/usr/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
     -e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "FILTERS=${FILTERS:-}" -e "CM_NAMES=${CM_NAMES:-}" -e "JOBS=$JOBS"
     -e "MIN_TEST_TIMEOUT=$MIN_TEST_TIMEOUT" -e "CONFIRM_TIMEOUT_MULT=$CONFIRM_TIMEOUT_MULT"
     -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*')
@@ -251,7 +253,7 @@ if [ "$1" = --host ]; then
       while IFS= read -r l; do sum "$l"; done
   fi
   for n in $(seq 1 "$s1n"); do
-    o="$D/items/s1-$n/out"; [ ! -s "$o/note" ] || sum "NOTE stage 1 (${S1_FILE[$n]}): $(cat "$o/note")"
+    o="$D/items/s1-$n/out"
     cat "$D/items/s1-$n.names" >>"$D/m/s1.names"
     cat "$o/caught.txt" >>"$D/m/s1.caught"; cat "$o/unviable.txt" >>"$D/m/s1.unviable"
     cat "$o/missed.txt" "$o/timeout.txt" >>"$D/m/s1.surv"
@@ -286,7 +288,7 @@ if [ "$1" = --host ]; then
     ts2=$(( $(now) - ts2 ))
     bad=0
     for k in $(seq 1 "$K2"); do
-      o="$D/items/s2-$k/out"; [ ! -s "$o/note" ] || sum "NOTE stage 2 (shard $k of $K2): $(cat "$o/note")"
+      o="$D/items/s2-$k/out"
       if [ "$(cat "$o/hostrc" 2>/dev/null)" != 0 ]; then
         [ -e "$D/q/s2-$k" ] || continue
         container_error "stage 2 (shard $k of $K2)" "$o"; bad=1; continue
@@ -378,25 +380,10 @@ else
   case $CM_SHARD in */1) ;; *) args+=(--shard "$CM_SHARD") ;; esac
   [ "${#targs[@]}" = 0 ] || tail_args=(-- -- "${targs[@]}")
 fi
-run_cargo() { cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1; rc=$?; }
-run_cargo
-# The image's rustc wrapper (sccache, one cache volume shared by sibling containers) is a cache, never an input. 2026-11-02:
-# builds died mid-way in sibling containers, cargo reporting the wrapper "(never executed)" or a dep-info file of its own
-# target dir missing (ENOENT), while the image's sccache runs fine alone and RUSTC_WRAPPER= builds clean. A failed build
-# with a wrapper set is therefore retried once without it, and the job summary says which way it went. The failure itself
-# is not diagnosed here: the first attempt's log is kept as cargo-wrapper.log in the job's output.
-if [ "$rc" != 0 ] && [ "$rc" != 2 ] && [ "$rc" != 3 ] && [ -n "${RUSTC_WRAPPER:-}" ]; then
-  w=$RUSTC_WRAPPER; rc1=$rc; cp "$OUT/cargo.log" "$OUT/cargo-wrapper.log"
-  { echo "DIAG wrapper=$w"; ls -l "$w"; "$w" --version; df -h /tmp /sccache; free -m; grep -m3 -E "never executed|No such file" "$OUT/cargo.log"; } >"$OUT/diag.txt" 2>&1
-  unset RUSTC_WRAPPER
-  run_cargo
-  case $rc in 0|2|3) echo "RUSTC_WRAPPER=$w: the build failed with it (exit $rc1) and passed without it; the run continued without it" >"$OUT/note" ;;
-    *) echo "RUSTC_WRAPPER=$w: the build failed with it (exit $rc1) and without it (exit $rc): the wrapper was not the cause" >>"$OUT/cargo.log" ;; esac
-fi
+cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1
+rc=$?
 for l in caught missed timeout unviable; do cp "$o/mutants.out/$l.txt" "$OUT/$l.txt" 2>/dev/null || : >"$OUT/$l.txt"; done
 echo "$rc" >"$OUT/rc"
-[ ! -s "$OUT/diag.txt" ] || cat "$OUT/diag.txt"
-[ ! -s "$OUT/note" ] || cat "$OUT/note"
 tail -n 30 "$OUT/cargo.log"
 # 0 clean, 2 missed and 3 timeouts are normal; 4 (red baseline) and everything else are errors
 case $rc in 0|2|3) exit 0 ;; *) exit "$rc" ;; esac

tools/fixtures/mutation-mock/fakebin/cargo

@@ -62,8 +62,6 @@ open=$(perl -MPOSIX -e '$n = 0; for (10 .. 40) { my $d = POSIX::dup($_); if (def
 [ -z "${FAKE_CM_KILLS:-}" ] || touch "$FAKE_CM_KILLS"   # a watchdog kill arriving while a run is going
 echo "stage=$stage filter=$filter harness=${harness[*]} openfds=$open mult=$mult min=$mintimeout" >>"${FAKE_CM_LOG:?}"
 [ "${FAKE_CM_RED:-}" != "$stage" ] || { echo "baseline failed" >&2; exit 4; }
-# FAKE_CM_DEADWRAP=1: a run whose environment still holds RUSTC_WRAPPER dies like cargo with a wrapper that cannot execute.
-[ -z "${FAKE_CM_DEADWRAP:-}" ] || [ -z "${RUSTC_WRAPPER:-}" ] || { echo "  Caused by: could not execute process $RUSTC_WRAPPER rustc (never executed)"; exit 101; }
 mkdir -p "$out/mutants.out/log"
 col=3; [ "$stage" = 2 ] || col=2
 for o in caught missed timeout unviable; do

tools/stopgap/ygg-verify.sh

@@ -20,7 +20,7 @@
 #      the commit out detached. It is a real clone, not a worktree, because a
 #      worktree's .git file points at a host path the container cannot see.
 #   3. Runs <command> in <image> under a CPU and memory cap, niced. Each
-#      toolchain gets one shared registry cache. The build output directory
+#      toolchain gets one shared cargo home (registry and package-cache lock). The build output directory
 #      stays inside the work directory and is never shared between checkouts.
 #      CARGO_TARGET_DIR is deliberately NOT set: cargo's default (<workspace>/
 #      target) is already inside /src, and forcing it made cargo-mutants' --jobs
@@ -405,10 +405,14 @@ else
 # Priority is set inside the container: nice on the docker client does not reach a process
 # containerd starts. Jobs run at nice 10 and idle IO class, so the swarm's daemons and turns
 # on the same disk go first (2026-10-31: three verify jobs held IO pressure at ~75% full).
+# Cargo's package-cache lock lives in $CARGO_HOME, so the shared volume is CARGO_HOME itself: a registry-only mount left the
+# lock in each container's layer and sibling containers unpacking one crate removed each other's directories (2026-11-03).
+# PATH restates the image's; only the rust image needs it.
+case $image in eureka-verify-rust:*) cargo_env=(-e CARGO_HOME=/cargo-home -e PATH=/usr/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin) ;; *) cargo_env=() ;; esac
 sudo docker run --rm --name "$cname" --label "$label" --cpus="$cpus" --memory="$mem" \
   -v "$work:/src" -v /etc/machine-id:/etc/machine-id:ro \
-  -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages \
-  -v eureka-sccache:/sccache \
+  -v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages \
+  -v eureka-sccache:/sccache "${cargo_env[@]}" \
   -e BASH_ENV=/tmp/.ygg-bash-env ${nonce:+-e YGG_NONCE=$nonce} \
   -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0='*' \
   $docker_args \

tools/test-tools.sh

@@ -343,13 +343,6 @@ if grep '^stage=2' "$cm_log" | grep -q 'mult=9 min=33$' && ! grep '^stage=2' "$c
 # slot lease: the job's own slot is held by ygg-verify (an inherited descriptor), others by other jobs
 cm A HOLD_OWN=1 HOLD_OTHERS=1; cmcheck lease-free 'containers=3 |'; cmlocks lease-free
 cm A HOLD_OWN=1 HOLD_OTHERS=3; cmcheck lease-one-free 'containers=2 |'
-# a rustc wrapper that cannot execute: the build retries once without it and the job summary says so; a wrapper-free failure is not blamed on it
-cm C FAKE_CM_DEADWRAP=1 RUSTC_WRAPPER=/nonexistent/sccache
-cmrc C-dead-wrapper 0
-cmcheck C-dead-wrapper "NOTE stage 2 (shard 1 of 1): RUSTC_WRAPPER=/nonexistent/sccache: the build failed with it" "MUTANTS tested=2 caught=2"
-cm C FAKE_CM_RED=2 RUSTC_WRAPPER=/nonexistent/sccache
-[ "$cm_rc" != 0 ] || { echo "FAIL cargo-mutants C-red-with-wrapper: exit 0"; fail=1; }
-cmcheck C-red-with-wrapper "ERROR stage 2 (shard 1 of 1)" "!MUTANTS"
 cm A HOLD_OWN=1 HOLD_OTHERS=4; cmcheck lease-none-free 'containers=1 |' 'tested=16'; cmlocks lease-none-free
 open_none=$(grep -o 'openfds=[0-9]*' "$cm_log" | sort -u | paste -sd, -)
 cm A HOLD_OWN=1 HOLD_OTHERS=1; open_leased=$(grep -o 'openfds=[0-9]*' "$cm_log" | sort -u | paste -sd, -)