From dbe594e3955fcec9229f1e6e021799dc8e985e35 Mon Sep 17 00:00:00 2001
From: Meta Vens <rwvens@hotmail.com>
Date: Wed, 23 Sep 2026 14:02:05 +0200
Subject: [PATCH] Stop ygg-verify's watchdog from holding a slot after its job
 ends

The watchdog added in 9359e9c ran in a subshell that inherited the slot
lock's file descriptor. When a job finished early, `kill $watchdog`
ended the subshell but not its `sleep` child. That orphaned sleep kept
the slot for up to TIMEOUT seconds. Two slots were held this way on
2026-09-23, and four jobs queued behind them.

The subshell now closes the lock fd before sleeping, and the cleanup
kills the watchdog's children before the watchdog itself.

Tested on Yggdrasil: a fast job exited 0, and a TIMEOUT=15 hang exited
137 with the TIMEOUT line. Afterwards no sleep process held any
slot lock.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 tools/stopgap/ygg-verify.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/stopgap/ygg-verify.sh b/tools/stopgap/ygg-verify.sh
index e18a278..a8de898 100644
--- a/tools/stopgap/ygg-verify.sh
+++ b/tools/stopgap/ygg-verify.sh
@@ -129,7 +129,9 @@ echo "ygg-verify: $name@${sha:0:10} slot $slot, image $image, cpus $cpus, mem $m
 # waiting on it never woke). The watchdog kills the named container; killing
 # the docker client alone would leave the container running.
 cname="eureka-verify-$slot-$$"
-( sleep "$timeout_s"; sudo docker kill "$cname" >/dev/null 2>&1 && echo "ygg-verify: TIMEOUT after ${timeout_s}s, container killed" >&2 ) &
+# The subshell must not inherit the slot lock: its sleep outlives a job that
+# finishes early, and an inherited fd held slot 2 for an hour (2026-09-23).
+( exec {fd}>&-; sleep "$timeout_s"; sudo docker kill "$cname" >/dev/null 2>&1 && echo "ygg-verify: TIMEOUT after ${timeout_s}s, container killed" >&2 ) &
 watchdog=$!
 set +e
 sudo nice -n 10 docker run --rm --name "$cname" --cpus="$cpus" --memory="$mem" \
@@ -141,6 +143,7 @@ sudo nice -n 10 docker run --rm --name "$cname" --cpus="$cpus" --memory="$mem" \
   -w /src "$image" bash -c "$cmd"
 status=$?
 set -e
+pkill -P "$watchdog" 2>/dev/null || true
 kill "$watchdog" 2>/dev/null || true
 echo "ygg-verify: exit $status" >&2
 echo "__YGG_VERDICT__ $status"