diff --git a/tools/INDEX.md b/tools/INDEX.md index 616bd28..1cf2c0e 100644 --- a/tools/INDEX.md +++ b/tools/INDEX.md @@ -9,7 +9,7 @@ gets a Soul pass before merge. | script | purpose | consumer | named by | |---|---|---|---| | `tools/stopgap/ygg-verify.sh` | Pushes an exact revision to a mirror on Yggdrasil and runs one command in a capped, niced container (`RUN_ON_HOST=1`: run a host-side command in the same slot, clone and watchdog instead; the watchdog kills every container carrying the job label). `SUMMARY=` keeps the full log and prints a bounded summary (verdict, counts, ZERO TESTS RAN, failing tests); `wait ` blocks at most 270 s and returns the summary once; call it with a Bash tool `timeout` of at least 280000 ms. Dies with the stopgap when Idunn's verify transaction lands. | Hands, Soul, Imagination probes | `references/briefs.md` (Hands long jobs and verification, Soul long jobs), `SKILL.md` (Yggdrasil) | -| `tools/wait-on.sh` | One general waiter for anything that is not a Yggdrasil job: `pid []`, `log ` (with `WAIT_PID=` a dead process with no match is exit 98), `file `. Same contract as `ygg-verify.sh wait`: blocks at most 270 s, exit 0 done with a tail of at most 60 log lines, `RUNNING s` exit 99, exit 98 gone with no completion evidence; call it with a Bash tool `timeout` of at least 280000 ms. Replaces hand-rolled poll loops and idle `true`/`echo`/`sleep` calls. | Hands, Soul, Imagination probes | `references/briefs.md` (Hands long jobs, Soul long jobs, detached scripts) | +| `tools/wait-on.sh` | One general waiter for anything that is not a Yggdrasil job: `pid []`, `log ` (with `WAIT_PID=` a dead process with no match is exit 98; `WAIT_FROM=` ignores a reused log's stale lines; an invalid ERE is exit 3), `file `. The `pid` form prints `GONE (no completion evidence)`: prefer `log` whenever the job writes a done line. Same contract as `ygg-verify.sh wait`: blocks at most 270 s, exit 0 done with a tail of at most 60 log lines, `RUNNING s` exit 99, exit 98 gone with no completion evidence; call it with a Bash tool `timeout` of at least 280000 ms. Replaces hand-rolled poll loops and idle `true`/`echo`/`sleep` calls. | Hands, Soul, Imagination probes | `references/briefs.md` (Hands long jobs, Soul long jobs, detached scripts) | | `tools/mutation-table.sh` | One verify job that runs a list of hand mutants (file, before, after, test) against a revision and returns a kill table: CAUGHT with the killing test, SURVIVED, UNVIABLE, TIMEOUT, NOAPPLY, ERROR. Refuses a red or zero-test baseline and a red BUILD; rows print live. Isolation is Docker's: the baselines run once in a container that holds the revision, that container is committed to a throwaway image, and every mutant runs in a fresh `--rm` container from it, so nothing a run writes (tracked, untracked or ignored, such as `target/`) reaches the next mutant; the image is removed when the job ends, killed or not. The shared registry and sccache volumes and the network are not isolated. A file-level load failure (vitest `FAIL [ ]`, jest `FAIL ` plus `Test suite failed to run`) reads UNVIABLE. Replaces per-agent runmut/mutrun/strykrun scripts. | Soul, Hands (scoped run) | `references/briefs.md` (Hands verification, Soul reruns) | | `tools/testparse.sh` | Sourced library: failing test names and pass/fail/skip counts from cargo, dotnet, vitest, jest and pytest logs, and test files that failed to load (`LOADFAIL`). Shared by the summary and the kill table. | the two scripts above | not named by a brief; it lives and dies with them | | `tools/test-tools.sh` | Local fixture check of the parsers and the summary over `tools/fixtures/logs` (real cargo and mutation logs, plus hand-written dotnet, vitest, jest, pytest, exit-0-but-failed and no-verdict logs), and of the mutation runner's host driver and container half under a mock docker. Run it before merging a change to the scripts above. | whoever changes those scripts | `SKILL.md` (Shared tools) | diff --git a/tools/test-tools.sh b/tools/test-tools.sh index 12d0056..856fc75 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -216,6 +216,25 @@ r=$(WM=2 wo log "$T/none.log" 'x'); wchk "log missing, no pid, short cap" 99 "$r ( sleep 2; echo ready >"$T/f.txt" ) & r=$(WM=10 wo file "$T/f.txt"); wchk "file appears" 0 "$r" : >"$T/e.txt"; r=$(WM=2 wo file "$T/e.txt"); wchk "file empty is not done" 99 "$r" +# F1: a stale done line in a reused log is ignored from WAIT_FROM on; a new one is seen +printf 'old run +ALL DONE +' >"$T/r.log"; off=$(wc -c <"$T/r.log") +r=$(WAIT_FROM=$off WM=2 wo log "$T/r.log" 'ALL DONE'); wchk "stale done before WAIT_FROM ignored" 99 "$r" +printf 'new run +ALL DONE +' >>"$T/r.log" +r=$(WAIT_FROM=$off WM=4 wo log "$T/r.log" 'ALL DONE'); wchk "new done after WAIT_FROM seen" 0 "$r" +printf 'short +' >"$T/r2.log" +r=$(WAIT_FROM=9999 WM=2 wo log "$T/r2.log" 'short'); wchk "truncated log searched whole" 0 "$r" +# F2: the pid form reports GONE, never anything that reads as success +sleep 1 & gp=$!; wait $gp +r=$(WM=4 wo pid $gp); wchk "pid gone" 0 "$r" +grep -q '^GONE (no completion evidence)' "$T/wo.out" || { echo "FAIL wait-on pid: no GONE line"; fail=1; } +# F4: an invalid ERE is a usage error, not a forever RUNNING +r=$(WM=2 wo log "$T/l.log" '(unclosed'); wchk "invalid regex" 3 "$r" +grep -q 'not a valid extended regex' "$T/wo.out" || { echo "FAIL wait-on: no regex message"; fail=1; } r=$(wo bogus); wchk "bad usage" 2 "$r" wait exit $fail diff --git a/tools/wait-on.sh b/tools/wait-on.sh old mode 100644 new mode 100755 index 907abd8..50852f6 --- a/tools/wait-on.sh +++ b/tools/wait-on.sh @@ -5,16 +5,35 @@ # 0 done; prints a bounded tail (at most 60 lines) of the log when one is given # 99 still going; prints `RUNNING s (...)`: call it again # 98 the thing waited on is gone with no completion evidence +# 3 the done regex is not a valid ERE (usage error, nothing was waited on) +# 2 bad usage # Call it with a Bash tool `timeout` of at least 280000 ms. # # Forms: -# wait-on.sh pid [] done when the process is gone (exit 0) +# wait-on.sh pid [] process gone: prints `GONE (no completion evidence)`, exit 0 # wait-on.sh log done when a log line matches the ERE # wait-on.sh file done when the file exists and is non-empty # `log` accepts WAIT_PID=: if that process dies and the regex never matched # (checked once more after the death), exit 98. Without WAIT_PID a log that does # not exist after 30 s is exit 98 (never started). `pid` cannot tell a finished -# process from one that never existed: both are exit 0, so pass a log and read it. +# process from one that never existed or crashed: exit 0 there means only "gone", +# never "succeeded". Prefer `log ` with WAIT_PID whenever the job +# writes a done line. +# +# Reused log paths: `log` accepts WAIT_FROM=; only bytes after it are +# searched, so a stale done line from an earlier job on the same path is ignored. +# Capture it at launch: `WAIT_FROM=$(wc -c < "$log" 2>/dev/null || echo 0)`. If the +# file is now shorter than WAIT_FROM (truncated and rewritten) the whole file is +# searched. Reading only from the offset also keeps each 2 s check cheap on a +# multi-GB log. Without WAIT_FROM the whole log is searched. +# +# Known limitations (Soul pass 1, PR #7; not fixed): +# F5 WAIT_PID naming a launcher that exits while its child keeps writing gives a +# false 98; pass the pid that writes the log, or none. +# F6 `file` is done at the first byte; a writer that is still writing is not +# waited out. Have the job write the file last (rename into place). +# F7 `pid ` prints a tail only; without WAIT_FROM the `log` form +# greps the whole log each 2 s, which is slow on multi-GB logs. # # PIDs: on Git Bash a Windows PID (PowerShell Start-Process .Id, a task manager # number) is seen by `tasklist`, but `kill -0` does not know it; Git Bash's own @@ -35,6 +54,17 @@ alive() { esac return 1 } +# scan : does the log (from WAIT_FROM, when set and not past EOF) match? +scan() { + [ -f "$1" ] || return 1 + local from=${WAIT_FROM:-0} + case "$from" in ''|*[!0-9]*) from=0 ;; esac + if [ "$from" -gt 0 ] && [ "$(wc -c <"$1")" -ge "$from" ]; then + tail -c +"$((from + 1))" "$1" 2>/dev/null | grep -Eq -- "$2" + else + grep -Eq -- "$2" "$1" 2>/dev/null + fi +} tail60() { [ -n "${1:-}" ] && [ -f "$1" ] && tail -n 60 "$1" | cut -c1-400; return 0; } running() { echo "RUNNING $(( $(date +%s) - t0 ))s ($1)"; exit 99; } elapsed() { echo $(( $(date +%s) - t0 )); } @@ -43,18 +73,20 @@ case "$mode" in pid) pid=${2:-}; log=${3:-}; [ -n "$pid" ] || usage while :; do - if ! alive "$pid"; then tail60 "$log"; exit 0; fi + if ! alive "$pid"; then echo "GONE (no completion evidence)"; tail60 "$log"; exit 0; fi [ "$(elapsed)" -lt "$cap" ] || running "pid $pid alive; last log line: $(tail -n 1 "$log" 2>/dev/null | cut -c1-160)" sleep 2 done ;; log) log=${2:-}; re=${3:-}; [ -n "$log" ] && [ -n "$re" ] || usage wp=${WAIT_PID:-} + rc=0; printf '' | grep -Eq -- "$re" 2>/dev/null || rc=$? + [ "$rc" -le 1 ] || { echo "wait-on: usage - /$re/ is not a valid extended regex" >&2; exit 3; } while :; do - if [ -f "$log" ] && grep -Eq -- "$re" "$log" 2>/dev/null; then tail60 "$log"; exit 0; fi + if scan "$log" "$re"; then tail60 "$log"; exit 0; fi if [ -n "$wp" ] && ! alive "$wp"; then sleep 1 - if [ -f "$log" ] && grep -Eq -- "$re" "$log" 2>/dev/null; then tail60 "$log"; exit 0; fi + if scan "$log" "$re"; then tail60 "$log"; exit 0; fi echo "wait-on: DEAD - pid $wp is gone and /$re/ never matched in $log"; tail60 "$log"; exit 98 fi if [ -z "$wp" ] && [ ! -f "$log" ] && [ "$(elapsed)" -ge 30 ]; then