Eureka
655357806e
dotnet 10 at normal verbosity ends a run with Total tests: N / Passed: N / Failed: N / Skipped: N (one count per indented line, zero counts omitted) instead of the one-line "Passed! - Failed: ..." form, so mutation-table.sh baselines read as "ran zero tests". tp_scan now parses both forms. A count the block does not account for (Total - Passed - Failed - Skipped) is counted as failed, so a cut-off block cannot read as clean; a "Total tests:" line that is not a count recognises nothing. Fixtures: dotnet10-green/red/zero.log are real dotnet 10.0.403 MSTest runs (-v n); dotnet10-all-failed/truncated.log are hand-written. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
148 lines
7.4 KiB
Shell
Executable file
148 lines
7.4 KiB
Shell
Executable file
#!/usr/bin/env bash
|
||
# Sourced, not run. Reads a test-run log and says what it contains.
|
||
#
|
||
# tp_scan <log> one line per fact:
|
||
# FRAMEWORK <name> passed=<n> failed=<n> skipped=<n> (summed over the log)
|
||
# FAILED <test name> (one per failing test)
|
||
# LOADFAIL <file> (a test FILE that failed, with no test name:
|
||
# a module that did not load; jest also prints one for a file with failing tests)
|
||
# tp_sections <log> [max-lines] cargo failure sections ("---- name stdout
|
||
# ----"), each cut to its last <max-lines> lines (default 30)
|
||
#
|
||
# Frameworks: cargo test, dotnet test, vitest, jest, pytest. Anything else is
|
||
# simply not recognised; the callers treat "recognised nothing" as a fact to
|
||
# print, never as success.
|
||
#
|
||
# Owner: tools/stopgap/ygg-verify.sh (summary mode) and tools/mutation-table.sh
|
||
# (kill attribution). One parser, so a summary and a kill table cannot disagree
|
||
# about what failed. Written for mawk: no match() groups, no gensub.
|
||
|
||
tp_scan() {
|
||
awk '
|
||
function num(s, a) { split(s, a, " "); return a[1] + 0 }
|
||
function pyname(s, p, sp, d, i, c) { # pytest short-summary name: a [param id] may hold spaces
|
||
p = index(s, "["); sp = index(s, " ")
|
||
if (p == 0 || (sp > 0 && sp < p)) return sp ? substr(s, 1, sp - 1) : s
|
||
for (i = p; i <= length(s); i++) {
|
||
c = substr(s, i, 1)
|
||
if (c == "[") d++; else if (c == "]" && --d == 0) return substr(s, 1, i)
|
||
}
|
||
return s
|
||
}
|
||
function dflush() { # close a dotnet 10 "Total tests:" block
|
||
if (!dblk) return
|
||
# dotnet omits a zero line, so Failed may be absent; a count the block does not account for is a
|
||
# failure, never a pass (a truncated block must not read as clean)
|
||
rem = dtot - bp - bf - bs; if (rem < 0) rem = 0
|
||
dp += bp; df += bf + rem; ds += bs; dotnet = 1; dblk = 0
|
||
}
|
||
/^Total tests: *[0-9]+ *$/ { # dotnet 10, normal verbosity: a block, one count per indented line
|
||
dflush(); dtot = $3 + 0; bp = bf = bs = 0; dblk = 1; next
|
||
}
|
||
dblk && /^ +(Passed|Failed|Skipped): +[0-9]+ *$/ {
|
||
v = $2 + 0
|
||
if ($1 == "Passed:") bp += v; else if ($1 == "Failed:") bf += v; else bs += v
|
||
next
|
||
}
|
||
dblk { dflush() } # any other line ends the block, and is then scanned as usual
|
||
/^test result: / { # cargo, one line per test binary
|
||
line = $0; sub(/^test result: [A-Za-z]+\. /, "", line)
|
||
n = split(line, parts, "; ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = num(parts[i])
|
||
if (parts[i] ~ /passed/) cp += v
|
||
if (parts[i] ~ /failed/) cf += v
|
||
if (parts[i] ~ /ignored/) cs += v
|
||
}
|
||
cargo = 1; next
|
||
}
|
||
/^test .* \.\.\. FAILED$/ { name = $0; sub(/^test /, "", name); sub(/ \.\.\. FAILED$/, "", name); print "FAILED " name; next }
|
||
/^(Passed|Failed)! - Failed: *[0-9]+, Passed: *[0-9]+, Skipped: *[0-9]+, Total:/ { # dotnet
|
||
line = $0; sub(/^[A-Za-z]+! - /, "", line) # dotnet, one-line summary (quiet verbosity)
|
||
n = split(line, parts, ", ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = parts[i]; sub(/^[A-Za-z]+: */, "", v); v += 0
|
||
if (parts[i] ~ /^Failed:/) df += v
|
||
if (parts[i] ~ /^Passed:/) dp += v
|
||
if (parts[i] ~ /^Skipped:/) ds += v
|
||
}
|
||
dotnet = 1; next
|
||
}
|
||
/^No test (matches the given testcase filter|is available in)/ { dotnet = 1; next } # dotnet, zero tests
|
||
/^[ \t]+Failed .+ \[[<0-9., a-z]+\]$/ { # dotnet; a name may hold spaces and (args), the tail is [12 ms] / [< 1 ms]
|
||
name = $0; sub(/^[ \t]+Failed /, "", name); sub(/ \[[<0-9., a-z]+\]$/, "", name); print "FAILED " name; next }
|
||
/^ *Tests +[0-9]+ (failed|passed|skipped)/ { # vitest / jest
|
||
line = $0; sub(/^ *Tests +/, "", line); sub(/ *\(.*$/, "", line)
|
||
n = split(line, parts, " \\| ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = num(parts[i])
|
||
if (parts[i] ~ /failed/) vf += v
|
||
if (parts[i] ~ /passed/) vp += v
|
||
if (parts[i] ~ /skipped/) vs += v
|
||
}
|
||
vitest = 1; next
|
||
}
|
||
/^ *Tests: +[0-9]+ / { # jest: "Tests: 1 failed, 2 passed, 3 total"
|
||
line = $0; sub(/^ *Tests: +/, "", line)
|
||
n = split(line, parts, ", ")
|
||
for (i = 1; i <= n; i++) {
|
||
v = num(parts[i])
|
||
if (parts[i] ~ /failed/) jf += v
|
||
if (parts[i] ~ /passed/) jp += v
|
||
if (parts[i] ~ /skipped/) js += v
|
||
}
|
||
jest = 1; next
|
||
}
|
||
# vitest and jest print "FAIL <file>" for a file and " FAIL <file> > <suite> > <test>" (vitest,
|
||
# optionally with a "|project|" prefix) for a test. Vitest and a jest TTY pad the word with two spaces;
|
||
# jest without a TTY, which is how every job runs it, prints ONE ("FAIL jt/add.test.js"). A line with
|
||
# no " > " names a FILE: a vitest load failure ("<file> [ <file> ]"), or a jest file header whose
|
||
# tests are named by the "●" lines below it.
|
||
/^ *FAIL +/ { infail = 1; name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next }
|
||
/^ *PASS +/ || /^Test Suites:/ { infail = 0 }
|
||
# jest failure block header " ● suite › test". It names a test only under a FAIL file header: a "●"
|
||
# elsewhere is output of the run itself (process.exit prints "● process.exit called with ..." with no
|
||
# FAIL header at all), and a crash that names no test stays unnamed, so callers read it as ERROR.
|
||
/^ +● / {
|
||
name = $0; sub(/^ +● +/, "", name)
|
||
if (infail && name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name
|
||
next
|
||
}
|
||
/^(=+ )?(no tests ran|[0-9]+ [a-z]+(, [0-9]+ [a-z]+)*) in [0-9.]+s( \([0-9:]+\))?( =+)?$/ { # pytest summary, with or without -q
|
||
line = $0; sub(/^=+ /, "", line); sub(/ =+$/, "", line); sub(/ in [0-9.]+s.*$/, "", line)
|
||
n = split(line, parts, ", "); ok = 1; a = b = c = 0
|
||
for (i = 1; i <= n; i++) {
|
||
split(parts[i], w, " "); v = w[1] + 0
|
||
if (w[2] == "failed" || w[2] == "error" || w[2] == "errors") b += v
|
||
else if (w[2] == "passed" || w[2] == "xpassed") a += v
|
||
else if (w[2] == "skipped" || w[2] == "deselected" || w[2] == "xfailed") c += v
|
||
else if (w[2] != "warning" && w[2] != "warnings") ok = 0
|
||
}
|
||
if (line == "no tests ran") ok = 1
|
||
if (ok) { pp += a; pf += b; ps += c; pytest = 1 }
|
||
next
|
||
}
|
||
/^FAILED [^ ]+::/ { print "FAILED " pyname(substr($0, 8)); next }
|
||
END {
|
||
dflush()
|
||
if (cargo) printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n", cp, cf, cs
|
||
if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds
|
||
if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs
|
||
if (jest) printf "FRAMEWORK jest passed=%d failed=%d skipped=%d\n", jp, jf, js
|
||
if (pytest) printf "FRAMEWORK pytest passed=%d failed=%d skipped=%d\n", pp, pf, ps
|
||
}' "$1"
|
||
}
|
||
|
||
tp_sections() {
|
||
awk -v max="${2:-30}" '
|
||
function flush( i, start) {
|
||
if (!n) return
|
||
start = n > max ? n - max + 1 : 1
|
||
for (i = start; i <= n; i++) print buf[i]
|
||
n = 0
|
||
}
|
||
/^---- .* stdout ----$/ { flush(); n = 0; buf[++n] = $0; inside = 1; next }
|
||
inside && (/^failures:$/ || /^test result: /) { flush(); inside = 0; next }
|
||
inside { buf[++n] = $0 }
|
||
END { flush() }' "$1"
|
||
}
|