diff --git a/tools/cargo-mutants.sh b/tools/cargo-mutants.sh index 9e435f5..b46bd90 100644 --- a/tools/cargo-mutants.sh +++ b/tools/cargo-mutants.sh @@ -135,11 +135,13 @@ if [ "$1" = --host ]; then trap cleanup EXIT; trap 'exit 143' TERM INT HUP killed() { [ -n "${YGG_KILLED:-}" ] && [ -e "$YGG_KILLED" ]; } now() { date +%s; } + # The shared volume is CARGO_HOME itself so sibling containers share cargo's package-cache lock; a registry-only mount left + # the lock in each container's layer and concurrent unpacks removed each other's crate directories (2026-11-03). # Every container carries the job label (the stopgap's watchdog kills by it) and the same caps. common=(--label "$YGG_LABEL" "--cpus=${YGG_CPUS:-4}" "--memory=${YGG_MEM:-6g}" -v "$YGG_WORK:/seed:ro" -v "$D:/cm:ro" -v /etc/machine-id:/etc/machine-id:ro - -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache - -e BASH_ENV=/tmp/.ygg-bash-env + -v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages -v eureka-sccache:/sccache + -e BASH_ENV=/tmp/.ygg-bash-env -e CARGO_HOME=/cargo-home -e PATH=/usr/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -e "WORKDIR=${WORKDIR:-.}" -e "BASE=$BASE" -e "TEST_ARGS=${TEST_ARGS:-}" -e "FILTERS=${FILTERS:-}" -e "CM_NAMES=${CM_NAMES:-}" -e "JOBS=$JOBS" -e "MIN_TEST_TIMEOUT=$MIN_TEST_TIMEOUT" -e "CONFIRM_TIMEOUT_MULT=$CONFIRM_TIMEOUT_MULT" -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*') @@ -251,7 +253,7 @@ if [ "$1" = --host ]; then while IFS= read -r l; do sum "$l"; done fi for n in $(seq 1 "$s1n"); do - o="$D/items/s1-$n/out"; [ ! -s "$o/note" ] || sum "NOTE stage 1 (${S1_FILE[$n]}): $(cat "$o/note")" + o="$D/items/s1-$n/out" cat "$D/items/s1-$n.names" >>"$D/m/s1.names" cat "$o/caught.txt" >>"$D/m/s1.caught"; cat "$o/unviable.txt" >>"$D/m/s1.unviable" cat "$o/missed.txt" "$o/timeout.txt" >>"$D/m/s1.surv" @@ -286,7 +288,7 @@ if [ "$1" = --host ]; then ts2=$(( $(now) - ts2 )) bad=0 for k in $(seq 1 "$K2"); do - o="$D/items/s2-$k/out"; [ ! -s "$o/note" ] || sum "NOTE stage 2 (shard $k of $K2): $(cat "$o/note")" + o="$D/items/s2-$k/out" if [ "$(cat "$o/hostrc" 2>/dev/null)" != 0 ]; then [ -e "$D/q/s2-$k" ] || continue container_error "stage 2 (shard $k of $K2)" "$o"; bad=1; continue @@ -378,25 +380,10 @@ else case $CM_SHARD in */1) ;; *) args+=(--shard "$CM_SHARD") ;; esac [ "${#targs[@]}" = 0 ] || tail_args=(-- -- "${targs[@]}") fi -run_cargo() { cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1; rc=$?; } -run_cargo -# The image's rustc wrapper (sccache, one cache volume shared by sibling containers) is a cache, never an input. 2026-11-02: -# builds died mid-way in sibling containers, cargo reporting the wrapper "(never executed)" or a dep-info file of its own -# target dir missing (ENOENT), while the image's sccache runs fine alone and RUSTC_WRAPPER= builds clean. A failed build -# with a wrapper set is therefore retried once without it, and the job summary says which way it went. The failure itself -# is not diagnosed here: the first attempt's log is kept as cargo-wrapper.log in the job's output. -if [ "$rc" != 0 ] && [ "$rc" != 2 ] && [ "$rc" != 3 ] && [ -n "${RUSTC_WRAPPER:-}" ]; then - w=$RUSTC_WRAPPER; rc1=$rc; cp "$OUT/cargo.log" "$OUT/cargo-wrapper.log" - { echo "DIAG wrapper=$w"; ls -l "$w"; "$w" --version; df -h /tmp /sccache; free -m; grep -m3 -E "never executed|No such file" "$OUT/cargo.log"; } >"$OUT/diag.txt" 2>&1 - unset RUSTC_WRAPPER - run_cargo - case $rc in 0|2|3) echo "RUSTC_WRAPPER=$w: the build failed with it (exit $rc1) and passed without it; the run continued without it" >"$OUT/note" ;; - *) echo "RUSTC_WRAPPER=$w: the build failed with it (exit $rc1) and without it (exit $rc): the wrapper was not the cause" >>"$OUT/cargo.log" ;; esac -fi +cargo "${args[@]}" "${tail_args[@]}" >"$OUT/cargo.log" 2>&1 +rc=$? for l in caught missed timeout unviable; do cp "$o/mutants.out/$l.txt" "$OUT/$l.txt" 2>/dev/null || : >"$OUT/$l.txt"; done echo "$rc" >"$OUT/rc" -[ ! -s "$OUT/diag.txt" ] || cat "$OUT/diag.txt" -[ ! -s "$OUT/note" ] || cat "$OUT/note" tail -n 30 "$OUT/cargo.log" # 0 clean, 2 missed and 3 timeouts are normal; 4 (red baseline) and everything else are errors case $rc in 0|2|3) exit 0 ;; *) exit "$rc" ;; esac diff --git a/tools/fixtures/mutation-mock/fakebin/cargo b/tools/fixtures/mutation-mock/fakebin/cargo index f8ad166..a722cbf 100755 --- a/tools/fixtures/mutation-mock/fakebin/cargo +++ b/tools/fixtures/mutation-mock/fakebin/cargo @@ -62,8 +62,6 @@ open=$(perl -MPOSIX -e '$n = 0; for (10 .. 40) { my $d = POSIX::dup($_); if (def [ -z "${FAKE_CM_KILLS:-}" ] || touch "$FAKE_CM_KILLS" # a watchdog kill arriving while a run is going echo "stage=$stage filter=$filter harness=${harness[*]} openfds=$open mult=$mult min=$mintimeout" >>"${FAKE_CM_LOG:?}" [ "${FAKE_CM_RED:-}" != "$stage" ] || { echo "baseline failed" >&2; exit 4; } -# FAKE_CM_DEADWRAP=1: a run whose environment still holds RUSTC_WRAPPER dies like cargo with a wrapper that cannot execute. -[ -z "${FAKE_CM_DEADWRAP:-}" ] || [ -z "${RUSTC_WRAPPER:-}" ] || { echo " Caused by: could not execute process $RUSTC_WRAPPER rustc (never executed)"; exit 101; } mkdir -p "$out/mutants.out/log" col=3; [ "$stage" = 2 ] || col=2 for o in caught missed timeout unviable; do diff --git a/tools/stopgap/ygg-verify.sh b/tools/stopgap/ygg-verify.sh index aed061b..aeeb461 100644 --- a/tools/stopgap/ygg-verify.sh +++ b/tools/stopgap/ygg-verify.sh @@ -20,7 +20,7 @@ # the commit out detached. It is a real clone, not a worktree, because a # worktree's .git file points at a host path the container cannot see. # 3. Runs in under a CPU and memory cap, niced. Each -# toolchain gets one shared registry cache. The build output directory +# toolchain gets one shared cargo home (registry and package-cache lock). The build output directory # stays inside the work directory and is never shared between checkouts. # CARGO_TARGET_DIR is deliberately NOT set: cargo's default (/ # target) is already inside /src, and forcing it made cargo-mutants' --jobs @@ -405,10 +405,14 @@ else # Priority is set inside the container: nice on the docker client does not reach a process # containerd starts. Jobs run at nice 10 and idle IO class, so the swarm's daemons and turns # on the same disk go first (2026-10-31: three verify jobs held IO pressure at ~75% full). +# Cargo's package-cache lock lives in $CARGO_HOME, so the shared volume is CARGO_HOME itself: a registry-only mount left the +# lock in each container's layer and sibling containers unpacking one crate removed each other's directories (2026-11-03). +# PATH restates the image's; only the rust image needs it. +case $image in eureka-verify-rust:*) cargo_env=(-e CARGO_HOME=/cargo-home -e PATH=/usr/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin) ;; *) cargo_env=() ;; esac sudo docker run --rm --name "$cname" --label "$label" --cpus="$cpus" --memory="$mem" \ -v "$work:/src" -v /etc/machine-id:/etc/machine-id:ro \ - -v eureka-cargo-registry:/usr/local/cargo/registry -v eureka-nuget:/root/.nuget/packages \ - -v eureka-sccache:/sccache \ + -v eureka-cargo-home:/cargo-home -v eureka-nuget:/root/.nuget/packages \ + -v eureka-sccache:/sccache "${cargo_env[@]}" \ -e BASH_ENV=/tmp/.ygg-bash-env ${nonce:+-e YGG_NONCE=$nonce} \ -e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0='*' \ $docker_args \ diff --git a/tools/test-tools.sh b/tools/test-tools.sh index faf43cc..bedbf38 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -343,13 +343,6 @@ if grep '^stage=2' "$cm_log" | grep -q 'mult=9 min=33$' && ! grep '^stage=2' "$c # slot lease: the job's own slot is held by ygg-verify (an inherited descriptor), others by other jobs cm A HOLD_OWN=1 HOLD_OTHERS=1; cmcheck lease-free 'containers=3 |'; cmlocks lease-free cm A HOLD_OWN=1 HOLD_OTHERS=3; cmcheck lease-one-free 'containers=2 |' -# a rustc wrapper that cannot execute: the build retries once without it and the job summary says so; a wrapper-free failure is not blamed on it -cm C FAKE_CM_DEADWRAP=1 RUSTC_WRAPPER=/nonexistent/sccache -cmrc C-dead-wrapper 0 -cmcheck C-dead-wrapper "NOTE stage 2 (shard 1 of 1): RUSTC_WRAPPER=/nonexistent/sccache: the build failed with it" "MUTANTS tested=2 caught=2" -cm C FAKE_CM_RED=2 RUSTC_WRAPPER=/nonexistent/sccache -[ "$cm_rc" != 0 ] || { echo "FAIL cargo-mutants C-red-with-wrapper: exit 0"; fail=1; } -cmcheck C-red-with-wrapper "ERROR stage 2 (shard 1 of 1)" "!MUTANTS" cm A HOLD_OWN=1 HOLD_OTHERS=4; cmcheck lease-none-free 'containers=1 |' 'tested=16'; cmlocks lease-none-free open_none=$(grep -o 'openfds=[0-9]*' "$cm_log" | sort -u | paste -sd, -) cm A HOLD_OWN=1 HOLD_OTHERS=1; open_leased=$(grep -o 'openfds=[0-9]*' "$cm_log" | sort -u | paste -sd, -)