f68bd4099f
Jest without a TTY prints "FAIL jt/add.test.js" with one space; the parser matched two, so a real jest load failure read ERROR, not UNVIABLE. The old `^ +● ` denylist also named process.exit's "● process.exit called with "1"" as a killing test (CAUGHT). Names now need a FAIL file header above them; a crash with no named failing test is ERROR. The "no test counts recognised" line names jest. The synthetic jest logs and mock are replaced by real jest 29.7.0 output captured on Yggdrasil from tools/fixtures/mutation-probe/jsr (green, wrong, load, exit runs in logs/jest-real.log). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
131 lines
6.4 KiB
Shell
131 lines
6.4 KiB
Shell
#!/usr/bin/env bash
|
||
# Sourced, not run. Reads a test-run log and says what it contains.
|
||
#
|
||
# tp_scan <log> one line per fact:
|
||
# FRAMEWORK <name> passed=<n> failed=<n> skipped=<n> (summed over the log)
|
||
# FAILED <test name> (one per failing test)
|
||
# LOADFAIL <file> (a test FILE that failed, with no test name:
|
||
# a module that did not load; jest also prints one for a file with failing tests)
|
||
# tp_sections <log> [max-lines] cargo failure sections ("---- name stdout
|
||
# ----"), each cut to its last <max-lines> lines (default 30)
|
||
#
|
||
# Frameworks: cargo test, dotnet test, vitest, jest, pytest. Anything else is
|
||
# simply not recognised; the callers treat "recognised nothing" as a fact to
|
||
# print, never as success.
|
||
#
|
||
# Owner: tools/stopgap/ygg-verify.sh (summary mode) and tools/mutation-table.sh
|
||
# (kill attribution). One parser, so a summary and a kill table cannot disagree
|
||
# about what failed. Written for mawk: no match() groups, no gensub.
|
||
|
||
tp_scan() {
|
||
awk '
|
||
function num(s, a) { split(s, a, " "); return a[1] + 0 }
|
||
function pyname(s, p, sp, d, i, c) { # pytest short-summary name: a [param id] may hold spaces
|
||
p = index(s, "["); sp = index(s, " ")
|
||
if (p == 0 || (sp > 0 && sp < p)) return sp ? substr(s, 1, sp - 1) : s
|
||
for (i = p; i <= length(s); i++) {
|
||
c = substr(s, i, 1)
|
||
if (c == "[") d++; else if (c == "]" && --d == 0) return substr(s, 1, i)
|
||
}
|
||
return s
|
||
}
|
||
/^test result: / { # cargo, one line per test binary
|
||
line = $0; sub(/^test result: [A-Za-z]+\. /, "", line)
|
||
n = split(line, parts, "; ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = num(parts[i])
|
||
if (parts[i] ~ /passed/) cp += v
|
||
if (parts[i] ~ /failed/) cf += v
|
||
if (parts[i] ~ /ignored/) cs += v
|
||
}
|
||
cargo = 1; next
|
||
}
|
||
/^test .* \.\.\. FAILED$/ { name = $0; sub(/^test /, "", name); sub(/ \.\.\. FAILED$/, "", name); print "FAILED " name; next }
|
||
/^(Passed|Failed)! - Failed: *[0-9]+, Passed: *[0-9]+, Skipped: *[0-9]+, Total:/ { # dotnet
|
||
line = $0; sub(/^[A-Za-z]+! - /, "", line)
|
||
n = split(line, parts, ", ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = parts[i]; sub(/^[A-Za-z]+: */, "", v); v += 0
|
||
if (parts[i] ~ /^Failed:/) df += v
|
||
if (parts[i] ~ /^Passed:/) dp += v
|
||
if (parts[i] ~ /^Skipped:/) ds += v
|
||
}
|
||
dotnet = 1; next
|
||
}
|
||
/^No test (matches the given testcase filter|is available in)/ { dotnet = 1; next } # dotnet, zero tests
|
||
/^[ \t]+Failed .+ \[[<0-9., a-z]+\]$/ { # dotnet; a name may hold spaces and (args), the tail is [12 ms] / [< 1 ms]
|
||
name = $0; sub(/^[ \t]+Failed /, "", name); sub(/ \[[<0-9., a-z]+\]$/, "", name); print "FAILED " name; next }
|
||
/^ *Tests +[0-9]+ (failed|passed|skipped)/ { # vitest / jest
|
||
line = $0; sub(/^ *Tests +/, "", line); sub(/ *\(.*$/, "", line)
|
||
n = split(line, parts, " \\| ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = num(parts[i])
|
||
if (parts[i] ~ /failed/) vf += v
|
||
if (parts[i] ~ /passed/) vp += v
|
||
if (parts[i] ~ /skipped/) vs += v
|
||
}
|
||
vitest = 1; next
|
||
}
|
||
/^ *Tests: +[0-9]+ / { # jest: "Tests: 1 failed, 2 passed, 3 total"
|
||
line = $0; sub(/^ *Tests: +/, "", line)
|
||
n = split(line, parts, ", ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = num(parts[i])
|
||
if (parts[i] ~ /failed/) jf += v
|
||
if (parts[i] ~ /passed/) jp += v
|
||
if (parts[i] ~ /skipped/) js += v
|
||
}
|
||
jest = 1; next
|
||
}
|
||
# vitest and jest print "FAIL <file>" for a file and " FAIL <file> > <suite> > <test>" (vitest,
|
||
# optionally with a "|project|" prefix) for a test. Vitest and a jest TTY pad the word with two spaces;
|
||
# jest without a TTY, which is how every job runs it, prints ONE ("FAIL jt/add.test.js"). A line with
|
||
# no " > " names a FILE: a vitest load failure ("<file> [ <file> ]"), or a jest file header whose
|
||
# tests are named by the "●" lines below it.
|
||
/^ *FAIL +/ { infail = 1; name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next }
|
||
/^ *PASS +/ || /^Test Suites:/ { infail = 0 }
|
||
# jest failure block header " ● suite › test". It names a test only under a FAIL file header: a "●"
|
||
# elsewhere is output of the run itself (process.exit prints "● process.exit called with ..." with no
|
||
# FAIL header at all), and a crash that names no test stays unnamed, so callers read it as ERROR.
|
||
/^ +● / {
|
||
name = $0; sub(/^ +● +/, "", name)
|
||
if (infail && name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name
|
||
next
|
||
}
|
||
/^(=+ )?(no tests ran|[0-9]+ [a-z]+(, [0-9]+ [a-z]+)*) in [0-9.]+s( \([0-9:]+\))?( =+)?$/ { # pytest summary, with or without -q
|
||
line = $0; sub(/^=+ /, "", line); sub(/ =+$/, "", line); sub(/ in [0-9.]+s.*$/, "", line)
|
||
n = split(line, parts, ", "); ok = 1; a = b = c = 0
|
||
for (i = 1; i <= n; i++) {
|
||
split(parts[i], w, " "); v = w[1] + 0
|
||
if (w[2] == "failed" || w[2] == "error" || w[2] == "errors") b += v
|
||
else if (w[2] == "passed" || w[2] == "xpassed") a += v
|
||
else if (w[2] == "skipped" || w[2] == "deselected" || w[2] == "xfailed") c += v
|
||
else if (w[2] != "warning" && w[2] != "warnings") ok = 0
|
||
}
|
||
if (line == "no tests ran") ok = 1
|
||
if (ok) { pp += a; pf += b; ps += c; pytest = 1 }
|
||
next
|
||
}
|
||
/^FAILED [^ ]+::/ { print "FAILED " pyname(substr($0, 8)); next }
|
||
END {
|
||
if (cargo) printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n", cp, cf, cs
|
||
if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds
|
||
if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs
|
||
if (jest) printf "FRAMEWORK jest passed=%d failed=%d skipped=%d\n", jp, jf, js
|
||
if (pytest) printf "FRAMEWORK pytest passed=%d failed=%d skipped=%d\n", pp, pf, ps
|
||
}' "$1"
|
||
}
|
||
|
||
tp_sections() {
|
||
awk -v max="${2:-30}" '
|
||
function flush( i, start) {
|
||
if (!n) return
|
||
start = n > max ? n - max + 1 : 1
|
||
for (i = start; i <= n; i++) print buf[i]
|
||
n = 0
|
||
}
|
||
/^---- .* stdout ----$/ { flush(); n = 0; buf[++n] = $0; inside = 1; next }
|
||
inside && (/^failures:$/ || /^test result: /) { flush(); inside = 0; next }
|
||
inside { buf[++n] = $0 }
|
||
END { flush() }' "$1"
|
||
}
|