#!/usr/bin/env bash # Sourced, not run. Reads a test-run log and says what it contains. # # tp_scan one line per fact: # FRAMEWORK passed= failed= skipped= (summed over the log) # FAILED (one per failing test) # LOADFAIL (a test FILE that failed, with no test name: # a module that did not load; jest also prints one for a file with failing tests) # tp_sections [max-lines] cargo failure sections ("---- name stdout # ----"), each cut to its last lines (default 30) # # Frameworks: cargo test, dotnet test, vitest, jest, pytest. Anything else is # simply not recognised; the callers treat "recognised nothing" as a fact to # print, never as success. # # Owner: tools/stopgap/ygg-verify.sh (summary mode) and tools/mutation-table.sh # (kill attribution). One parser, so a summary and a kill table cannot disagree # about what failed. Written for mawk: no match() groups, no gensub. tp_scan() { awk ' function num(s, a) { split(s, a, " "); return a[1] + 0 } function pyname(s, p, sp, d, i, c) { # pytest short-summary name: a [param id] may hold spaces p = index(s, "["); sp = index(s, " ") if (p == 0 || (sp > 0 && sp < p)) return sp ? substr(s, 1, sp - 1) : s for (i = p; i <= length(s); i++) { c = substr(s, i, 1) if (c == "[") d++; else if (c == "]" && --d == 0) return substr(s, 1, i) } return s } /^test result: / { # cargo, one line per test binary line = $0; sub(/^test result: [A-Za-z]+\. /, "", line) n = split(line, parts, "; ") for (i = 1; i <= n; i++) { v = num(parts[i]) if (parts[i] ~ /passed/) cp += v if (parts[i] ~ /failed/) cf += v if (parts[i] ~ /ignored/) cs += v } cargo = 1; next } /^test .* \.\.\. FAILED$/ { name = $0; sub(/^test /, "", name); sub(/ \.\.\. FAILED$/, "", name); print "FAILED " name; next } /^(Passed|Failed)! - Failed: *[0-9]+, Passed: *[0-9]+, Skipped: *[0-9]+, Total:/ { # dotnet line = $0; sub(/^[A-Za-z]+! - /, "", line) n = split(line, parts, ", ") for (i = 1; i <= n; i++) { v = parts[i]; sub(/^[A-Za-z]+: */, "", v); v += 0 if (parts[i] ~ /^Failed:/) df += v if (parts[i] ~ /^Passed:/) dp += v if (parts[i] ~ /^Skipped:/) ds += v } dotnet = 1; next } /^No test (matches the given testcase filter|is available in)/ { dotnet = 1; next } # dotnet, zero tests /^[ \t]+Failed .+ \[[<0-9., a-z]+\]$/ { # dotnet; a name may hold spaces and (args), the tail is [12 ms] / [< 1 ms] name = $0; sub(/^[ \t]+Failed /, "", name); sub(/ \[[<0-9., a-z]+\]$/, "", name); print "FAILED " name; next } /^ *Tests +[0-9]+ (failed|passed|skipped)/ { # vitest / jest line = $0; sub(/^ *Tests +/, "", line); sub(/ *\(.*$/, "", line) n = split(line, parts, " \\| ") for (i = 1; i <= n; i++) { v = num(parts[i]) if (parts[i] ~ /failed/) vf += v if (parts[i] ~ /passed/) vp += v if (parts[i] ~ /skipped/) vs += v } vitest = 1; next } /^ *Tests: +[0-9]+ / { # jest: "Tests: 1 failed, 2 passed, 3 total" line = $0; sub(/^ *Tests: +/, "", line) n = split(line, parts, ", ") for (i = 1; i <= n; i++) { v = num(parts[i]) if (parts[i] ~ /failed/) jf += v if (parts[i] ~ /passed/) jp += v if (parts[i] ~ /skipped/) js += v } jest = 1; next } # vitest and jest print "FAIL " for a file and " FAIL > > " (vitest, # optionally with a "|project|" prefix) for a test. Vitest and a jest TTY pad the word with two spaces; # jest without a TTY, which is how every job runs it, prints ONE ("FAIL jt/add.test.js"). A line with # no " > " names a FILE: a vitest load failure (" [ ]"), or a jest file header whose # tests are named by the "●" lines below it. /^ *FAIL +/ { infail = 1; name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next } /^ *PASS +/ || /^Test Suites:/ { infail = 0 } # jest failure block header " ● suite › test". It names a test only under a FAIL file header: a "●" # elsewhere is output of the run itself (process.exit prints "● process.exit called with ..." with no # FAIL header at all), and a crash that names no test stays unnamed, so callers read it as ERROR. /^ +● / { name = $0; sub(/^ +● +/, "", name) if (infail && name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name next } /^(=+ )?(no tests ran|[0-9]+ [a-z]+(, [0-9]+ [a-z]+)*) in [0-9.]+s( \([0-9:]+\))?( =+)?$/ { # pytest summary, with or without -q line = $0; sub(/^=+ /, "", line); sub(/ =+$/, "", line); sub(/ in [0-9.]+s.*$/, "", line) n = split(line, parts, ", "); ok = 1; a = b = c = 0 for (i = 1; i <= n; i++) { split(parts[i], w, " "); v = w[1] + 0 if (w[2] == "failed" || w[2] == "error" || w[2] == "errors") b += v else if (w[2] == "passed" || w[2] == "xpassed") a += v else if (w[2] == "skipped" || w[2] == "deselected" || w[2] == "xfailed") c += v else if (w[2] != "warning" && w[2] != "warnings") ok = 0 } if (line == "no tests ran") ok = 1 if (ok) { pp += a; pf += b; ps += c; pytest = 1 } next } /^FAILED [^ ]+::/ { print "FAILED " pyname(substr($0, 8)); next } END { if (cargo) printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n", cp, cf, cs if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs if (jest) printf "FRAMEWORK jest passed=%d failed=%d skipped=%d\n", jp, jf, js if (pytest) printf "FRAMEWORK pytest passed=%d failed=%d skipped=%d\n", pp, pf, ps }' "$1" } tp_sections() { awk -v max="${2:-30}" ' function flush( i, start) { if (!n) return start = n > max ? n - max + 1 : 1 for (i = start; i <= n; i++) print buf[i] n = 0 } /^---- .* stdout ----$/ { flush(); n = 0; buf[++n] = $0; inside = 1; next } inside && (/^failures:$/ || /^test result: /) { flush(); inside = 0; next } inside { buf[++n] = $0 } END { flush() }' "$1" }