Eureka/tools/testparse.sh

#!/usr/bin/env bash
# Sourced, not run. Reads a test-run log and says what it contains.
#
#   tp_scan <log>   one line per fact:
#       FRAMEWORK <name> passed=<n> failed=<n> skipped=<n>   (summed over the log)
#       FAILED <test name>                                   (one per failing test)
#       LOADFAIL <file>                                      (a test FILE that failed, with no test name:
#                                    a module that did not load; jest also prints one for a file with failing tests)
#   tp_sections <log> [max-lines]   cargo failure sections ("---- name stdout
#       ----"), each cut to its last <max-lines> lines (default 30)
#
# Frameworks: cargo test, dotnet test, vitest, jest, pytest. Anything else is
# simply not recognised; the callers treat "recognised nothing" as a fact to
# print, never as success.
#
# Owner: tools/stopgap/ygg-verify.sh (summary mode) and tools/mutation-table.sh
# (kill attribution). One parser, so a summary and a kill table cannot disagree
# about what failed. Written for mawk: no match() groups, no gensub.

tp_scan() {
  awk '
    function num(s,   a) { split(s, a, " "); return a[1] + 0 }
    function pyname(s,   p, sp, d, i, c) {   # pytest short-summary name: a [param id] may hold spaces
      p = index(s, "["); sp = index(s, " ")
      if (p == 0 || (sp > 0 && sp < p)) return sp ? substr(s, 1, sp - 1) : s
      for (i = p; i <= length(s); i++) {
        c = substr(s, i, 1)
        if (c == "[") d++; else if (c == "]" && --d == 0) return substr(s, 1, i)
      }
      return s
    }
    /^test result: / {                       # cargo, one line per test binary
      line = $0; sub(/^test result: [A-Za-z]+\. /, "", line)
      n = split(line, parts, "; ")
      for (i = 1; i <= n; i++) {
        v = num(parts[i])
        if (parts[i] ~ /passed/)  cp += v
        if (parts[i] ~ /failed/)  cf += v
        if (parts[i] ~ /ignored/) cs += v
      }
      cargo = 1; next
    }
    /^test .* \.\.\. FAILED$/ { name = $0; sub(/^test /, "", name); sub(/ \.\.\. FAILED$/, "", name); print "FAILED " name; next }
    /^(Passed|Failed)!  - Failed: *[0-9]+, Passed: *[0-9]+, Skipped: *[0-9]+, Total:/ {   # dotnet
      line = $0; sub(/^[A-Za-z]+!  - /, "", line)
      n = split(line, parts, ", ")
      for (i = 1; i <= n; i++) {
        v = parts[i]; sub(/^[A-Za-z]+: */, "", v); v += 0
        if (parts[i] ~ /^Failed:/)  df += v
        if (parts[i] ~ /^Passed:/)  dp += v
        if (parts[i] ~ /^Skipped:/) ds += v
      }
      dotnet = 1; next
    }
    /^No test (matches the given testcase filter|is available in)/ { dotnet = 1; next }   # dotnet, zero tests
    /^[ \t]+Failed .+ \[[<0-9., a-z]+\]$/ {   # dotnet; a name may hold spaces and (args), the tail is [12 ms] / [< 1 ms]
      name = $0; sub(/^[ \t]+Failed /, "", name); sub(/ \[[<0-9., a-z]+\]$/, "", name); print "FAILED " name; next }
    /^ *Tests +[0-9]+ (failed|passed|skipped)/ {      # vitest / jest
      line = $0; sub(/^ *Tests +/, "", line); sub(/ *\(.*$/, "", line)
      n = split(line, parts, " \\| ")
      for (i = 1; i <= n; i++) {
        v = num(parts[i])
        if (parts[i] ~ /failed/)  vf += v
        if (parts[i] ~ /passed/)  vp += v
        if (parts[i] ~ /skipped/) vs += v
      }
      vitest = 1; next
    }
    /^ *Tests: +[0-9]+ /  {                          # jest: "Tests:       1 failed, 2 passed, 3 total"
      line = $0; sub(/^ *Tests: +/, "", line)
      n = split(line, parts, ", ")
      for (i = 1; i <= n; i++) {
        v = num(parts[i])
        if (parts[i] ~ /failed/)  jf += v
        if (parts[i] ~ /passed/)  jp += v
        if (parts[i] ~ /skipped/) js += v
      }
      jest = 1; next
    }
    # vitest and jest print "FAIL <file>" for a file and " FAIL  <file> > <suite> > <test>" (vitest,
    # optionally with a "|project|" prefix) for a test. Vitest and a jest TTY pad the word with two spaces;
    # jest without a TTY, which is how every job runs it, prints ONE ("FAIL jt/add.test.js"). A line with
    # no " > " names a FILE: a vitest load failure ("<file> [ <file> ]"), or a jest file header whose
    # tests are named by the "●" lines below it.
    /^ *FAIL +/ { infail = 1; name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next }
    /^ *PASS +/ || /^Test Suites:/ { infail = 0 }
    # jest failure block header "  ● suite › test". It names a test only under a FAIL file header: a "●"
    # elsewhere is output of the run itself (process.exit prints "●  process.exit called with ..." with no
    # FAIL header at all), and a crash that names no test stays unnamed, so callers read it as ERROR.
    /^ +● / {
      name = $0; sub(/^ +● +/, "", name)
      if (infail && name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name
      next
    }
    /^(=+ )?(no tests ran|[0-9]+ [a-z]+(, [0-9]+ [a-z]+)*) in [0-9.]+s( \([0-9:]+\))?( =+)?$/ {   # pytest summary, with or without -q
      line = $0; sub(/^=+ /, "", line); sub(/ =+$/, "", line); sub(/ in [0-9.]+s.*$/, "", line)
      n = split(line, parts, ", "); ok = 1; a = b = c = 0
      for (i = 1; i <= n; i++) {
        split(parts[i], w, " "); v = w[1] + 0
        if (w[2] == "failed" || w[2] == "error" || w[2] == "errors") b += v
        else if (w[2] == "passed" || w[2] == "xpassed") a += v
        else if (w[2] == "skipped" || w[2] == "deselected" || w[2] == "xfailed") c += v
        else if (w[2] != "warning" && w[2] != "warnings") ok = 0
      }
      if (line == "no tests ran") ok = 1
      if (ok) { pp += a; pf += b; ps += c; pytest = 1 }
      next
    }
    /^FAILED [^ ]+::/ { print "FAILED " pyname(substr($0, 8)); next }
    END {
      if (cargo)  printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n",  cp, cf, cs
      if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds
      if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs
      if (jest)   printf "FRAMEWORK jest passed=%d failed=%d skipped=%d\n",   jp, jf, js
      if (pytest) printf "FRAMEWORK pytest passed=%d failed=%d skipped=%d\n", pp, pf, ps
    }' "$1"
}

tp_sections() {
  awk -v max="${2:-30}" '
    function flush(   i, start) {
      if (!n) return
      start = n > max ? n - max + 1 : 1
      for (i = start; i <= n; i++) print buf[i]
      n = 0
    }
    /^---- .* stdout ----$/ { flush(); n = 0; buf[++n] = $0; inside = 1; next }
    inside && (/^failures:$/ || /^test result: /) { flush(); inside = 0; next }
    inside { buf[++n] = $0 }
    END { flush() }' "$1"
}