diff --git a/tools/INDEX.md b/tools/INDEX.md index 84b0647..25efd0c 100644 --- a/tools/INDEX.md +++ b/tools/INDEX.md @@ -2,9 +2,13 @@ One line per script. Briefs name the exact script they want; agents do not browse. A line whose consuming brief no longer names the script is dead and is -deleted. Contributions come by branch and pull request. A script that verdicts -depend on gets a Soul pass before merge. +deleted. Contributions come by branch and Forgejo pull request (this repo lives on +Yggdrasil's Forgejo), never a push to main. A script that verdicts depend on +gets a Soul pass before merge. | script | purpose | consumer | named by | |---|---|---|---| -| `tools/stopgap/ygg-verify.sh` | Pushes an exact revision to a mirror on Yggdrasil and runs one command in a capped, niced container. Dies with the stopgap when Idunn's verify transaction lands. | Hands, Soul, Imagination probes | `references/briefs.md` (Hands and Soul verification), `SKILL.md` (Yggdrasil) | +| `tools/stopgap/ygg-verify.sh` | Pushes an exact revision to a mirror on Yggdrasil and runs one command in a capped, niced container. `SUMMARY=` keeps the full log and prints a bounded summary (verdict, counts, ZERO TESTS RAN, failing tests); `wait ` blocks at most 270 s and returns the summary once. Dies with the stopgap when Idunn's verify transaction lands. | Hands, Soul, Imagination probes | `references/briefs.md` (Hands long jobs and verification, Soul long jobs), `SKILL.md` (Yggdrasil) | +| `tools/mutation-table.sh` | One verify job that runs a list of hand mutants (file, before, after, test) against a revision and returns a kill table: CAUGHT with the killing test, SURVIVED, UNVIABLE, TIMEOUT, NOAPPLY, ERROR. Refuses a red or zero-test baseline. Replaces per-agent runmut/mutrun/strykrun scripts. | Soul, Hands (scoped run) | `references/briefs.md` (Hands verification, Soul reruns) | +| `tools/testparse.sh` | Sourced library: failing test names and pass/fail/skip counts from cargo, dotnet, vitest and pytest logs. Shared by the summary and the kill table. | the two scripts above | not named by a brief; it lives and dies with them | +| `tools/test-tools.sh` | Local fixture check of the parsers and the summary over `tools/fixtures/logs` (real cargo and mutation logs, plus hand-written dotnet, vitest, pytest, exit-0-but-failed and no-verdict logs). Run it before merging a change to the scripts above. | whoever changes those scripts | `SKILL.md` (Shared tools) | diff --git a/tools/fixtures/logs/cargo-green.log b/tools/fixtures/logs/cargo-green.log new file mode 100644 index 0000000..06afc5c --- /dev/null +++ b/tools/fixtures/logs/cargo-green.log @@ -0,0 +1,20 @@ +warning: You appear to have cloned an empty repository. +ygg-verify: probe@7d3a1f5284 slot 1, image eureka-verify-rust:b99d30ec441e, cpus 4, mem 6g, work /home/gamecultadmin/eureka-verify/work/probe-7d3a1f5284-Pvbq + Compiling tiny-crate v0.0.0 (/src) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.35s + Running unittests src/lib.rs (target/debug/deps/tiny_crate-7b09195d0b4b7f00) + Doc-tests tiny_crate + +running 3 tests +test tests::add_adds ... ok +test tests::even_detects ... ok +test tests::clamp_negative ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +ygg-verify: exit 0 diff --git a/tools/fixtures/logs/cargo-red.log b/tools/fixtures/logs/cargo-red.log new file mode 100644 index 0000000..c98c3e0 --- /dev/null +++ b/tools/fixtures/logs/cargo-red.log @@ -0,0 +1,29 @@ +warning: You appear to have cloned an empty repository. +ygg-verify: probe@7d3a1f5284 slot 2, image eureka-verify-rust:b99d30ec441e, cpus 4, mem 6g, work /home/gamecultadmin/eureka-verify/work/probe-7d3a1f5284-axtF + Compiling tiny-crate v0.0.0 (/src) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.34s + Running unittests src/lib.rs (target/debug/deps/tiny_crate-7b09195d0b4b7f00) + +running 3 tests +test tests::add_adds ... FAILED +error: test failed, to rerun pass `--lib` +test tests::clamp_negative ... ok +test tests::even_detects ... ok + +failures: + +---- tests::add_adds stdout ---- + +thread 'tests::add_adds' (113) panicked at src/lib.rs:19:9: +assertion `left == right` failed + left: 5 + right: 6 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + tests::add_adds + +test result: FAILED. 2 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +ygg-verify: exit 101 diff --git a/tools/fixtures/logs/cargo-zero-tests.log b/tools/fixtures/logs/cargo-zero-tests.log new file mode 100644 index 0000000..cbd303f --- /dev/null +++ b/tools/fixtures/logs/cargo-zero-tests.log @@ -0,0 +1,11 @@ +warning: You appear to have cloned an empty repository. +ygg-verify: probe@7d3a1f5284 slot 1, image eureka-verify-rust:b99d30ec441e, cpus 4, mem 6g, work /home/gamecultadmin/eureka-verify/work/probe-7d3a1f5284-OoDx + Compiling tiny-crate v0.0.0 (/src) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.32s + Running unittests src/lib.rs (target/debug/deps/tiny_crate-7b09195d0b4b7f00) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s + +ygg-verify: exit 0 diff --git a/tools/fixtures/logs/dotnet-red.log b/tools/fixtures/logs/dotnet-red.log new file mode 100644 index 0000000..183a623 --- /dev/null +++ b/tools/fixtures/logs/dotnet-red.log @@ -0,0 +1,6 @@ +SYNTHETIC: hand-written in the format dotnet test prints; not captured from a run. + Passed MyNs.AddTests.Adds [2 ms] + Failed MyNs.AddTests.Subtracts [12 ms] + Error Message: + Assert.Equal() Failure: Expected 1, Actual 2 +Failed! - Failed: 1, Passed: 11, Skipped: 2, Total: 14, Duration: 1 s - MyNs.Tests.dll (net10.0) diff --git a/tools/fixtures/logs/exit0-but-failed.log b/tools/fixtures/logs/exit0-but-failed.log new file mode 100644 index 0000000..f6ca9b5 --- /dev/null +++ b/tools/fixtures/logs/exit0-but-failed.log @@ -0,0 +1,3 @@ +SYNTHETIC: a command whose last statement hid a failing test run (the exit status is 0 in the test). +test tests::add_adds ... FAILED +test result: FAILED. 2 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s diff --git a/tools/fixtures/logs/mutation-table.log b/tools/fixtures/logs/mutation-table.log new file mode 100644 index 0000000..71b01b3 --- /dev/null +++ b/tools/fixtures/logs/mutation-table.log @@ -0,0 +1,9 @@ +warning: You appear to have cloned an empty repository. +ygg-verify: probe@7d3a1f5284 slot 2, image eureka-verify-rust:b99d30ec441e, cpus 4, mem 6g, work /home/gamecultadmin/eureka-verify/work/probe-7d3a1f5284-Elfg +YGG-SUMMARY: MUTATION 5 mutants: caught=2 survived=1 unviable=1 timeout=0 noapply=1 error=0 (baseline green, tests ran) +YGG-SUMMARY: clamp-positive | SURVIVED | tests passed with the mutation applied: cargo test +YGG-SUMMARY: add-typo | UNVIABLE | build exit 101 +YGG-SUMMARY: ghost | NOAPPLY | before text not found +YGG-SUMMARY: add-minus | CAUGHT | tests::add_adds +YGG-SUMMARY: even-flip | CAUGHT | tests::even_detects +ygg-verify: exit 1 diff --git a/tools/fixtures/logs/no-verdict.log b/tools/fixtures/logs/no-verdict.log new file mode 100644 index 0000000..9bcb259 --- /dev/null +++ b/tools/fixtures/logs/no-verdict.log @@ -0,0 +1,2 @@ +SYNTHETIC: the remote script died before printing its verdict line. +ygg-verify: NO VERDICT from the remote job - treating as failure diff --git a/tools/fixtures/logs/pytest-red.log b/tools/fixtures/logs/pytest-red.log new file mode 100644 index 0000000..2e8603b --- /dev/null +++ b/tools/fixtures/logs/pytest-red.log @@ -0,0 +1,3 @@ +SYNTHETIC: hand-written in the format pytest prints; not captured from a run. +FAILED tests/test_add.py::test_carries - assert 4 == 5 +========================= 1 failed, 5 passed, 1 skipped in 0.12s ========================= diff --git a/tools/fixtures/logs/vitest-red.log b/tools/fixtures/logs/vitest-red.log new file mode 100644 index 0000000..fabf42d --- /dev/null +++ b/tools/fixtures/logs/vitest-red.log @@ -0,0 +1,5 @@ +SYNTHETIC: hand-written in the format vitest prints; not captured from a run. + FAIL src/add.test.ts > add > carries +AssertionError: expected 4 to be 5 + Test Files 1 failed | 2 passed (3) + Tests 1 failed | 10 passed | 1 skipped (12) diff --git a/tools/fixtures/tiny-crate/Cargo.toml b/tools/fixtures/tiny-crate/Cargo.toml new file mode 100644 index 0000000..1e7a91d --- /dev/null +++ b/tools/fixtures/tiny-crate/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "tiny-crate" +version = "0.0.0" +edition = "2021" diff --git a/tools/fixtures/tiny-crate/mutants.txt b/tools/fixtures/tiny-crate/mutants.txt new file mode 100644 index 0000000..c611989 --- /dev/null +++ b/tools/fixtures/tiny-crate/mutants.txt @@ -0,0 +1,40 @@ +Fixture for tools/mutation-table.sh. Expected: add-minus CAUGHT by add_adds, +even-flip CAUGHT by even_detects, clamp-positive SURVIVED (no test reads the +positive branch), add-typo UNVIABLE (with BUILD='cargo test --no-run'), +ghost NOAPPLY. Run with TEST='cargo test' BUILD='cargo test --no-run'. + +=== add-minus +file: src/lib.rs +--- before + a + b +--- after + a - b + +=== even-flip +file: src/lib.rs +test: cargo test even_ +--- before +n % 2 == 0 +--- after +n % 2 == 1 + +=== clamp-positive +file: src/lib.rs +--- before +else { n } +--- after +else { n + 1 } + +=== add-typo +file: src/lib.rs +--- before + a + b +--- after + a + c + +=== ghost +file: src/lib.rs +--- before +this text is nowhere +--- after +x diff --git a/tools/fixtures/tiny-crate/src/lib.rs b/tools/fixtures/tiny-crate/src/lib.rs new file mode 100644 index 0000000..1bc490c --- /dev/null +++ b/tools/fixtures/tiny-crate/src/lib.rs @@ -0,0 +1,32 @@ +pub fn add(a: i32, b: i32) -> i32 { + a + b +} + +pub fn is_even(n: i32) -> bool { + n % 2 == 0 +} + +pub fn clamp_pos(n: i32) -> i32 { + if n < 0 { 0 } else { n } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn add_adds() { + assert_eq!(add(2, 3), 5); + } + + #[test] + fn even_detects() { + assert!(is_even(4)); + assert!(!is_even(3)); + } + + #[test] + fn clamp_negative() { + assert_eq!(clamp_pos(-5), 0); + } +} diff --git a/tools/mutation-table.sh b/tools/mutation-table.sh new file mode 100644 index 0000000..99bb764 --- /dev/null +++ b/tools/mutation-table.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Hand-mutant kill table: run a list of mutants against a revision on +# Yggdrasil, in ONE verify job, and get one table back. +# +# tools/mutation-table.sh [workdir] +# workdir directory inside the repo where the commands run (e.g. +# packages/cultcache-rs; CultLib has no Cargo.toml at its root). +# env TEST default test command for mutants with no `test:` line +# BUILD optional compile-only command; when it fails the mutant +# is UNVIABLE. Without it a mutant that does not compile +# has no failing test and reads ERROR, never CAUGHT. +# MUT_TIMEOUT seconds per build or test run (default 600) +# everything ygg-verify.sh reads (SUMMARY=, TIMEOUT, MEM, ...) +# Typical, from an agent: launch as ONE background shell with +# SUMMARY=/mut1.log TEST='cargo test' tools/mutation-table.sh ... +# then `tools/stopgap/ygg-verify.sh wait /mut1.log` in the foreground. +# +# Mutants file (plain text; the text before the first `===` is ignored): +# === label-unique-in-the-file +# file: src/lib.rs (relative to workdir) +# test: cargo test add_ (optional; else $TEST) +# --- before +# a + b (exact text, may span lines; must occur ONCE) +# --- after +# a - b +# +# Each mutant is applied in the container's throwaway clone and reverted from +# git before the next, so no tree you care about is touched. The output is one +# row per mutant: label | outcome | killing test or detail. Outcomes: +# CAUGHT the test command failed AND a failing test name was parsed from it +# SURVIVED the test command passed with the mutation applied +# UNVIABLE BUILD failed +# TIMEOUT a run hit MUT_TIMEOUT (not a kill) +# NOAPPLY before text absent, ambiguous, or equal to after (no mutation made) +# ERROR the test command failed but no failing test was parsed (a build +# error without BUILD, a crash): not a kill +# Before any mutant, every distinct test command runs UNMUTATED. If one fails or +# runs zero tests the job aborts with `BASELINE RED`, because a red or empty +# baseline would make every mutant look killed or every one survive. +# Exit 0 means every mutant was CAUGHT or UNVIABLE and nothing else happened. +# +# DELETION LINE: this runner replaces the per-agent runmut/mutrun/strykrun +# scripts. It dies with tools/stopgap/ygg-verify.sh when `idunn verify` ships a +# typed mutation step. It reads tools/testparse.sh for failure names, the same +# parser the summary uses. +set -euo pipefail + +if [ "${1:-}" != --in-container ]; then + here=$(cd "$(dirname "$0")" && pwd) + repo=${1:?usage: mutation-table.sh [workdir]} + rev=${2:?revision}; image=${3:?image}; mfile=${4:?mutants file}; workdir=${5:-.} + [ -r "$mfile" ] || { echo "mutation-table: cannot read $mfile" >&2; exit 2; } + stage=$(mktemp -d) + trap 'rm -rf "$stage"' EXIT + cp "$here/testparse.sh" "$here/mutation-table.sh" "$stage/" + tr -d '\r' <"$mfile" >"$stage/mutants.txt" + bundle=$(tar -C "$stage" -cz testparse.sh mutation-table.sh mutants.txt | base64 | tr -d '\n') + cmd=$(printf 'mkdir -p /tmp/mt && echo %s | base64 -d | tar -xz -C /tmp/mt && WORKDIR=%q BUILD=%q TEST=%q MUT_TIMEOUT=%q bash /tmp/mt/mutation-table.sh --in-container' \ + "$bundle" "$workdir" "${BUILD:-}" "${TEST:-}" "${MUT_TIMEOUT:-600}") + "$here/stopgap/ygg-verify.sh" "$repo" "$rev" "$image" "$cmd" + exit $? +fi + +# ---- container side --------------------------------------------------------- +set +e +# shellcheck source=testparse.sh +. /tmp/mt/testparse.sh +cd "/src/${WORKDIR:-.}" || { echo "YGG-SUMMARY: BAD WORKDIR ${WORKDIR:-.}"; exit 2; } +M=/tmp/mt/m; rm -rf "$M"; mkdir -p "$M" +tmo=${MUT_TIMEOUT:-600} + +if ! perl -e ' + my ($mf, $out) = @ARGV; my ($n, $sec, %c, %seen) = (0, ""); + open my $f, "<", $mf or die "cannot open $mf\n"; + sub flush { + return unless exists $c{label}; + for my $k (qw(file before after)) { die "mutant $c{label}: no $k\n" unless defined $c{$k} } + my $d = "$out/$n"; mkdir $d; + for my $k (qw(label file test before after)) { + my $v = join("\n", ref $c{$k} ? @{$c{$k}} : ($c{$k} // "")); + open my $o, ">:raw", "$d/$k" or die; print $o $v; close $o; + } + } + while (<$f>) { + chomp; + if (/^=== (.+?)\s*$/) { flush(); die "duplicate label $1\n" if $seen{$1}++; %c = (label => $1); $n++; $sec = ""; next } + next unless exists $c{label}; + if ($sec eq "" && /^file: (.+?)\s*$/) { $c{file} = $1; next } + if ($sec eq "" && /^test: (.+?)\s*$/) { $c{test} = $1; next } + if (/^--- (before|after)\s*$/) { $sec = $1; $c{$sec} = []; next } + push @{$c{$sec}}, $_ if $sec; + } + flush(); die "no mutants in $mf\n" unless $n; + print "$n\n";' /tmp/mt/mutants.txt "$M" >"$M/count" 2>"$M/err"; then + echo "YGG-SUMMARY: BAD MUTANT FILE: $(cat "$M/err")"; exit 2 +fi +count=$(cat "$M/count") + +# run : prints the exit status; 124 means timeout +run() { timeout "$tmo" bash -o pipefail -c "$1" >"$2" 2>&1; echo $?; } + +declare -A based +for n in $(seq 1 "$count"); do + t=$(cat "$M/$n/test"); t=${t:-$TEST} + [ -n "$t" ] || { echo "YGG-SUMMARY: mutant $(cat "$M/$n/label") has no test: line and TEST is unset"; exit 2; } + [ -z "${based[$t]:-}" ] || continue + based[$t]=1 + rc=$(run "$t" "$M/base.log") + scan=$(tp_scan "$M/base.log") + tests=$(printf '%s\n' "$scan" | awk '/^FRAMEWORK/{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed"||a[1]=="failed")s+=a[2]}}END{print s+0}') + if [ "$rc" != 0 ]; then + echo "YGG-SUMMARY: BASELINE RED (exit $rc) for: $t"; tail -n 15 "$M/base.log" | sed 's/^/YGG-SUMMARY: /'; exit 2 + fi + if [ "$tests" -eq 0 ]; then + echo "YGG-SUMMARY: BASELINE RAN ZERO TESTS (or printed counts no parser reads) for: $t"; exit 2 + fi +done + +: >"$M/rows" +for n in $(seq 1 "$count"); do + label=$(cat "$M/$n/label"); file=$(cat "$M/$n/file"); t=$(cat "$M/$n/test"); t=${t:-$TEST} + out=''; detail='' + if ! detail=$(perl -e ' + local $/; my ($file, $bf, $af) = @ARGV; + open my $h, "<:raw", $file or do { print "cannot read $file\n"; exit 3 }; my $c = <$h>; close $h; + open my $b, "<:raw", $bf; my $before = <$b> // ""; open my $a, "<:raw", $af; my $after = <$a> // ""; + if (length($before) == 0) { print "empty before text\n"; exit 3 } + if ($before eq $after) { print "after equals before\n"; exit 3 } + my ($cnt, $p) = (0, 0); while (($p = index($c, $before, $p)) >= 0) { $cnt++; $p += length($before) } + if ($cnt == 0) { print "before text not found\n"; exit 3 } + if ($cnt > 1) { print "before text occurs $cnt times; make it unique\n"; exit 3 } + substr($c, index($c, $before), length($before)) = $after; + open my $o, ">:raw", $file or exit 3; print $o $c; close $o;' "$file" "$M/$n/before" "$M/$n/after" 2>&1); then + out=NOAPPLY + elif git diff --quiet -- "$file"; then + out=NOAPPLY; detail="applied, but git sees no change in $file" + else + out='' + if [ -n "${BUILD:-}" ]; then + rc=$(run "$BUILD" "$M/run.log") + if [ "$rc" = 124 ]; then out=TIMEOUT; detail="build exceeded ${tmo}s" + elif [ "$rc" != 0 ]; then out=UNVIABLE; detail="build exit $rc"; fi + fi + if [ -z "$out" ]; then + rc=$(run "$t" "$M/run.log") + if [ "$rc" = 0 ]; then out=SURVIVED; detail="tests passed with the mutation applied: $t" + elif [ "$rc" = 124 ]; then out=TIMEOUT; detail="test exceeded ${tmo}s: $t" + else + names=$(tp_scan "$M/run.log" | grep '^FAILED ' | awk '!s[$0]++' | sed 's/^FAILED //') + if [ -n "$names" ]; then out=CAUGHT; detail=$(printf '%s\n' "$names" | head -n 1); k=$(printf '%s\n' "$names" | wc -l); [ "$k" -le 1 ] || detail="$detail (+$((k - 1)) more)" + else out=ERROR; detail="exit $rc, no failing test parsed: $(tail -n 1 "$M/run.log" | cut -c1-120)"; fi + fi + fi + fi + printf '%s | %s | %s\n' "$label" "$out" "$detail" >>"$M/rows" + git checkout -q -- "$file" + if ! git diff --quiet -- "$file"; then echo "YGG-SUMMARY: RESTORE FAILED for $file after $label; stopping"; cat "$M/rows" | sed 's/^/YGG-SUMMARY: /'; exit 2; fi +done + +tally() { grep -c " | $1 | " "$M/rows" || true; } +bad=$(grep -vc -E ' \| (CAUGHT|UNVIABLE) \| ' "$M/rows" || true) +{ + echo "MUTATION $count mutants: caught=$(tally CAUGHT) survived=$(tally SURVIVED) unviable=$(tally UNVIABLE) timeout=$(tally TIMEOUT) noapply=$(tally NOAPPLY) error=$(tally ERROR) (baseline green, tests ran)" + grep -v ' | CAUGHT | ' "$M/rows" + grep ' | CAUGHT | ' "$M/rows" +} | sed 's/^/YGG-SUMMARY: /' +[ "$bad" -eq 0 ] diff --git a/tools/stopgap/ygg-verify.sh b/tools/stopgap/ygg-verify.sh index 8e35fb3..6f9ce10 100644 --- a/tools/stopgap/ygg-verify.sh +++ b/tools/stopgap/ygg-verify.sh @@ -83,6 +83,28 @@ # (e.g. DOCKER_ARGS='--sysctl net.ipv6.conf.all.disable_ipv6=0' or # DOCKER_ARGS='--network host'). Word-split by the remote shell, so quote a # flag's own value inside the string if it needs one. +# +# Summary and wait modes (agents use these; they save the polling calls): +# SUMMARY= ygg-verify.sh '' +# Runs the same job but keeps the whole output in and prints only +# a bounded summary (at most ~80 lines): the exit verdict, per-framework +# pass/fail/skip counts (cargo test, dotnet test, vitest, pytest), a ZERO +# TESTS RAN line when counts exist and are all zero, failing test names, and +# for a red job the failure sections and the log tail. When the job ends it +# writes .summary and then .done (holding the exit +# status). Use a fresh per job. Launch it as ONE background shell +# (run_in_background), then: +# ygg-verify.sh wait +# Blocks at most 270 s (WAIT_MAX can only lower that), prints the summary and +# exits with the job's status when the job is done. Otherwise prints +# `RUNNING s` and exits 99: call it again. Exit 98 means the +# launcher is gone with no verdict (killed outright) or never started. It +# checks a local file every 2 s; no remote call, no model turn, one wake. +# A job's own lines starting `YGG-SUMMARY: ` are always carried into the +# summary (tools/mutation-table.sh prints its table that way). The full log +# stays at ; open it only at a named failing test, with grep -n -A 40. +# Killing the launcher does not stop the remote container: its own TIMEOUT and +# IDLE watchdog does. # Images: # rust eureka-verify-rust: (rust 1.95, cargo-mutants, sccache; # every job shares the eureka-sccache volume at /sccache, keyed by @@ -97,11 +119,103 @@ # 'cargo test --workspace && git diff BASE HEAD > /tmp/d && cargo mutants --in-diff /tmp/d' set -euo pipefail +here=$(cd "$(dirname "$0")" && pwd) + +# Bounded summary of one job's log. $1 log, $2 exit status, $3 optional note. +ygg_summarize() { + local log=$1 status=$2 note=${3:-} word=RED scan fw names p f nf own=0 + [ "$status" = 0 ] && word=GREEN + # shellcheck source=../testparse.sh + . "$here/../testparse.sh" + grep -q '^YGG-SUMMARY: ' "$log" && own=1 + scan=$(tp_scan "$log") + fw=$(printf '%s\n' "$scan" | grep '^FRAMEWORK' || true) + names=$(printf '%s\n' "$scan" | grep '^FAILED ' | awk '!seen[$0]++' || true) + read -r p f < <(printf '%s\n' "$fw" | awk '{for(i=1;i<=NF;i++){split($i,a,"=");if(a[1]=="passed")p+=a[2];if(a[1]=="failed")f+=a[2]}}END{print p+0, f+0}') + nf=0; [ -z "$names" ] || nf=$(printf '%s\n' "$names" | wc -l) + { + echo "YGG-VERIFY exit $status $word${note:+ ($note)} log: $log" + grep -E '^ygg-verify: (TIMEOUT|IDLE|NO VERDICT)' "$log" || true + if [ "$status" = 0 ] && { [ "$nf" -gt 0 ] || [ "$f" -gt 0 ]; }; then + echo "MISMATCH: exit 0 but the log shows $nf failing test names and $f failed counts; do not trust the exit" + fi + [ -z "$fw" ] || printf '%s\n' "$fw" + if [ -z "$fw" ]; then [ "$own" = 1 ] || echo "no test counts recognised in the log (cargo, dotnet, vitest, pytest)" + elif [ $((p + f)) -eq 0 ]; then echo "ZERO TESTS RAN"; fi + if [ "$nf" -gt 0 ]; then + printf '%s\n' "$names" | awk 'NR<=15{print "FAILED " substr($0,8)} END{if(NR>15)print "... " NR-15 " more failing tests"}' + fi + { grep '^YGG-SUMMARY: ' "$log" || true; } | sed 's/^YGG-SUMMARY: //' + { grep -E '^(MISSED|TIMEOUT) |mutants tested' "$log" || true; } | awk 'NR<=20' + if [ "$status" != 0 ]; then + if [ "$own" = 1 ]; then echo "--- log tail"; { tail -n 8 "$log" | grep -v '^YGG-SUMMARY: '; } || true + else + [ "$nf" -gt 0 ] || [ "$f" -gt 0 ] || echo "exit non-zero with no failing test parsed: build error, crash, timeout or a non-test command" + secs=$(tp_sections "$log" 30 | awk 'NR<=40') + [ -z "$secs" ] || printf '%s\n' "$secs" + echo "--- log tail" + if [ -n "$secs" ]; then tail -n 8 "$log"; else tail -n 20 "$log"; fi + fi + elif [ -z "$fw" ] && [ "$own" = 0 ]; then + echo "--- log tail" + tail -n 5 "$log" + fi + } | awk 'NR<=80{print} NR==81{print "[summary truncated at 80 lines; full log is the path above]"}' +} + +if [ "${1:-}" = summarize ]; then # re-read a finished log: summarize + ygg_summarize "${2:?log}" "${3:?exit status}"; exit 0 +fi + +if [ "${1:-}" = wait ]; then + wlog=${2:?usage: ygg-verify.sh wait } + cap=${WAIT_MAX:-270}; [ "$cap" -le 270 ] || cap=270 + t0=$(date +%s) + while :; do + if [ -e "$wlog.done" ]; then + st=$(cat "$wlog.done") + if [ -s "$wlog.summary" ]; then cat "$wlog.summary"; else echo "ygg-verify: exit $st, no summary file; see $wlog"; fi + exit "$st" + fi + wpid=$(cat "$wlog.pid" 2>/dev/null || true) + el=$(( $(date +%s) - t0 )) + if [ -n "$wpid" ] && ! kill -0 "$wpid" 2>/dev/null && [ ! -e "$wlog.done" ]; then + echo "ygg-verify: DEAD - launcher $wpid is gone without a verdict; the job's outcome is unknown (log $wlog)"; exit 98 + fi + if [ -z "$wpid" ] && [ "$el" -ge 30 ]; then + echo "ygg-verify: NOT STARTED - no $wlog.pid after 30 s; was the job launched with SUMMARY=$wlog?"; exit 98 + fi + if [ "$el" -ge "$cap" ]; then + started=$(cat "$wlog.start" 2>/dev/null || echo "$t0") + echo "RUNNING $(( $(date +%s) - started ))s (log $wlog; last line: $(tail -n 1 "$wlog" 2>/dev/null | cut -c1-160))" + exit 99 + fi + sleep 2 + done +fi + +if [ -n "${SUMMARY:-}" ] && [ -z "${YGG_SUMMARY_INNER:-}" ]; then + slog=$SUMMARY; child='' + mkdir -p "$(dirname "$slog")" + rm -f "$slog.done" "$slog.summary" + : >"$slog"; date +%s >"$slog.start"; echo $$ >"$slog.pid" + ygg_finish() { # summary first, done-file last: a visible .done always has its summary + ygg_summarize "$slog" "$1" "${2:-}" >"$slog.summary.tmp" && mv "$slog.summary.tmp" "$slog.summary" || true + echo "$1" >"$slog.done.tmp"; mv "$slog.done.tmp" "$slog.done" + if [ -s "$slog.summary" ]; then cat "$slog.summary"; fi + exit "$1" + } + trap '[ -z "$child" ] || kill "$child" 2>/dev/null; ygg_finish 143 "launcher killed by a signal; the remote job may still be running"' TERM INT HUP + YGG_SUMMARY_INNER=1 "$0" "$@" >"$slog" 2>&1 & + child=$! + sst=0; wait "$child" || sst=$? + ygg_finish "$sst" +fi + repo=${1:?local repo}; rev=${2:?revision}; image=${3:?image}; cmd=${4:?command} host=${YGG_HOST:-ygg}; cpus=${CPUS:-4}; mem=${MEM:-6g}; slots=${SLOTS:-5} timeout_s=${TIMEOUT:-3600}; idle_s=${IDLE:-900} docker_args=${DOCKER_ARGS:-} -here=$(cd "$(dirname "$0")" && pwd) sshopts=(-o BatchMode=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=4) sha=$(git -C "$repo" rev-parse --verify "$rev^{commit}") diff --git a/tools/test-tools.sh b/tools/test-tools.sh new file mode 100644 index 0000000..d6621b7 --- /dev/null +++ b/tools/test-tools.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Local, cheap checks of the verdict-bearing text parsers: tools/testparse.sh and +# the summary mode of tools/stopgap/ygg-verify.sh, over tools/fixtures/logs. +# cargo-*.log and mutation-table.log are real Yggdrasil runs (the tiny-crate +# fixture, 2026-10-01); dotnet, vitest, pytest, exit0 and no-verdict are +# hand-written. Run: bash tools/test-tools.sh (no network, no builds) +set -uo pipefail +here=$(cd "$(dirname "$0")" && pwd) +L=$here/fixtures/logs; V=$here/stopgap/ygg-verify.sh +fail=0 +# check [] +check() { + out=$(bash "$V" summarize "$L/$1" "$2") + printf '%s\n' "$out" | grep -qF -- "$3" || { echo "FAIL $1: missing: $3"; fail=1; return; } + if [ -n "${4:-}" ] && printf '%s\n' "$out" | grep -qF -- "$4"; then echo "FAIL $1: must not contain: $4"; fail=1; return; fi + [ "$(printf '%s\n' "$out" | wc -l)" -le 81 ] || { echo "FAIL $1: summary over 81 lines"; fail=1; return; } + echo "ok $1: $3" +} +check cargo-green.log 0 "exit 0 GREEN" "RED" +check cargo-green.log 0 "FRAMEWORK cargo passed=3 failed=0" +check cargo-red.log 101 "exit 101 RED" +check cargo-red.log 101 "FAILED tests::add_adds" +check cargo-red.log 101 "left: 5" +check cargo-zero-tests.log 0 "ZERO TESTS RAN" +check mutation-table.log 1 "clamp-positive | SURVIVED" +check mutation-table.log 1 "add-typo | UNVIABLE" +check mutation-table.log 1 "add-minus | CAUGHT | tests::add_adds" +check mutation-table.log 1 "caught=2 survived=1 unviable=1 timeout=0 noapply=1 error=0" "no test counts recognised" +check dotnet-red.log 1 "FRAMEWORK dotnet passed=11 failed=1 skipped=2" +check dotnet-red.log 1 "FAILED MyNs.AddTests.Subtracts" +check vitest-red.log 1 "FRAMEWORK vitest passed=10 failed=1 skipped=1" +check vitest-red.log 1 "FAILED src/add.test.ts > add > carries" +check pytest-red.log 1 "FRAMEWORK pytest passed=5 failed=1 skipped=1" +check pytest-red.log 1 "FAILED tests/test_add.py::test_carries" +check exit0-but-failed.log 0 "MISMATCH: exit 0 but the log shows" +check no-verdict.log 97 "NO VERDICT" +check no-verdict.log 97 "RED" "GREEN" +exit $fail diff --git a/tools/testparse.sh b/tools/testparse.sh new file mode 100644 index 0000000..11628de --- /dev/null +++ b/tools/testparse.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Sourced, not run. Reads a test-run log and says what it contains. +# +# tp_scan one line per fact: +# FRAMEWORK passed= failed= skipped= (summed over the log) +# FAILED (one per failing test) +# tp_sections [max-lines] cargo failure sections ("---- name stdout +# ----"), each cut to its last lines (default 30) +# +# Frameworks: cargo test, dotnet test, vitest/jest, pytest. Anything else is +# simply not recognised; the callers treat "recognised nothing" as a fact to +# print, never as success. +# +# Owner: tools/stopgap/ygg-verify.sh (summary mode) and tools/mutation-table.sh +# (kill attribution). One parser, so a summary and a kill table cannot disagree +# about what failed. Written for mawk: no match() groups, no gensub. + +tp_scan() { + awk ' + function num(s, a) { split(s, a, " "); return a[1] + 0 } + /^test result: / { # cargo, one line per test binary + line = $0; sub(/^test result: [A-Za-z]+\. /, "", line) + n = split(line, parts, "; ") + for (i = 1; i <= n; i++) { + v = num(parts[i]) + if (parts[i] ~ /passed/) cp += v + if (parts[i] ~ /failed/) cf += v + if (parts[i] ~ /ignored/) cs += v + } + cargo = 1; next + } + /^test .* \.\.\. FAILED$/ { name = $0; sub(/^test /, "", name); sub(/ \.\.\. FAILED$/, "", name); print "FAILED " name; next } + /^(Passed|Failed)! - Failed: *[0-9]+, Passed: *[0-9]+, Skipped: *[0-9]+, Total:/ { # dotnet + line = $0; sub(/^[A-Za-z]+! - /, "", line) + n = split(line, parts, ", ") + for (i = 1; i <= n; i++) { + v = parts[i]; sub(/^[A-Za-z]+: */, "", v); v += 0 + if (parts[i] ~ /^Failed:/) df += v + if (parts[i] ~ /^Passed:/) dp += v + if (parts[i] ~ /^Skipped:/) ds += v + } + dotnet = 1; next + } + /^[ \t]+Failed [^ ]+ \[/ { name = $0; sub(/^[ \t]+Failed /, "", name); sub(/ \[.*$/, "", name); print "FAILED " name; next } + /^ *Tests +[0-9]+ (failed|passed|skipped)/ { # vitest / jest + line = $0; sub(/^ *Tests +/, "", line); sub(/ *\(.*$/, "", line) + n = split(line, parts, " \\| ") + for (i = 1; i <= n; i++) { + v = num(parts[i]) + if (parts[i] ~ /failed/) vf += v + if (parts[i] ~ /passed/) vp += v + if (parts[i] ~ /skipped/) vs += v + } + vitest = 1; next + } + /^ *FAIL / { name = $0; sub(/^ *FAIL +/, "", name); print "FAILED " name; next } + /^=+ .*(passed|failed|no tests ran).* in [0-9.]+s/ { # pytest summary line + line = $0; gsub(/=/, "", line); sub(/ in [0-9.]+s.*$/, "", line) + n = split(line, parts, ", ") + for (i = 1; i <= n; i++) { + v = num(parts[i]) + if (parts[i] ~ /failed/) pf += v + if (parts[i] ~ /passed/) pp += v + if (parts[i] ~ /(skipped|deselected)/) ps += v + if (parts[i] ~ /errors?/) pf += v + } + pytest = 1; next + } + /^FAILED [^ ]+::/ { name = $2; print "FAILED " name; next } + END { + if (cargo) printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n", cp, cf, cs + if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds + if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs + if (pytest) printf "FRAMEWORK pytest passed=%d failed=%d skipped=%d\n", pp, pf, ps + }' "$1" +} + +tp_sections() { + awk -v max="${2:-30}" ' + function flush( i, start) { + if (!n) return + start = n > max ? n - max + 1 : 1 + for (i = start; i <= n; i++) print buf[i] + n = 0 + } + /^---- .* stdout ----$/ { flush(); n = 0; buf[++n] = $0; inside = 1; next } + inside && (/^failures:$/ || /^test result: /) { flush(); inside = 0; next } + inside { buf[++n] = $0 } + END { flush() }' "$1" +}