testparse: read dotnet 10's "Total tests:" summary block
dotnet 10 at normal verbosity ends a run with Total tests: N / Passed: N / Failed: N / Skipped: N (one count per indented line, zero counts omitted) instead of the one-line "Passed! - Failed: ..." form, so mutation-table.sh baselines read as "ran zero tests". tp_scan now parses both forms. A count the block does not account for (Total - Passed - Failed - Skipped) is counted as failed, so a cut-off block cannot read as clean; a "Total tests:" line that is not a count recognises nothing. Fixtures: dotnet10-green/red/zero.log are real dotnet 10.0.403 MSTest runs (-v n); dotnet10-all-failed/truncated.log are hand-written. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
0faa16184e
commit
655357806e
7 changed files with 81 additions and 1 deletions
4
tools/fixtures/logs/dotnet10-all-failed.log
Normal file
4
tools/fixtures/logs/dotnet10-all-failed.log
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
Test Run Failed.
|
||||
Total tests: 3
|
||||
Failed: 3
|
||||
Total time: 0.5 Seconds
|
||||
12
tools/fixtures/logs/dotnet10-green.log
Normal file
12
tools/fixtures/logs/dotnet10-green.log
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
Test run for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (.NETCoreApp,Version=v10.0)
|
||||
A total of 1 test files matched the specified pattern.
|
||||
Test Parallelization enabled for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (Workers: 16, Scope: MethodLevel)
|
||||
Skipped Skipped
|
||||
Passed Bad [5 ms]
|
||||
Passed Ok [5 ms]
|
||||
|
||||
Test Run Successful.
|
||||
Total tests: 3
|
||||
Passed: 2
|
||||
Skipped: 1
|
||||
Total time: 0.5415 Seconds
|
||||
22
tools/fixtures/logs/dotnet10-red.log
Normal file
22
tools/fixtures/logs/dotnet10-red.log
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
Test run for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (.NETCoreApp,Version=v10.0)
|
||||
A total of 1 test files matched the specified pattern.
|
||||
Test Parallelization enabled for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (Workers: 16, Scope: MethodLevel)
|
||||
Skipped Skipped
|
||||
Passed Ok [5 ms]
|
||||
Failed Bad [12 ms]
|
||||
Error Message:
|
||||
Assert.AreEqual failed. Expected:<1>. Actual:<2>. 'expected' expression: '1', 'actual' expression: '2'.
|
||||
Stack Trace:
|
||||
at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowAssertAreEqualFailed(Object expected, Object actual, String userMessage) in /_/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs:line 665
|
||||
at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AreEqual[T](T expected, T actual, IEqualityComparer`1 comparer, String message, String expectedExpression, String actualExpression) in /_/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs:line 492
|
||||
at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AreEqual[T](T expected, T actual, String message, String expectedExpression, String actualExpression) in /_/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs:line 405
|
||||
at T.Test1.Bad() in /tmp/dn/proj/T/Test1.cs:line 6
|
||||
at System.Reflection.MethodBaseInvoker.InterpretedInvoke_Method(Object obj, IntPtr* args)
|
||||
at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr)
|
||||
|
||||
Test Run Failed.
|
||||
Total tests: 3
|
||||
Passed: 1
|
||||
Failed: 1
|
||||
Skipped: 1
|
||||
Total time: 0.5395 Seconds
|
||||
3
tools/fixtures/logs/dotnet10-truncated.log
Normal file
3
tools/fixtures/logs/dotnet10-truncated.log
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Test Run Failed.
|
||||
Total tests: 5
|
||||
Passed: 2
|
||||
3
tools/fixtures/logs/dotnet10-zero.log
Normal file
3
tools/fixtures/logs/dotnet10-zero.log
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Test run for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (.NETCoreApp,Version=v10.0)
|
||||
A total of 1 test files matched the specified pattern.
|
||||
No test matches the given testcase filter `Name=Nope` in /tmp/dn/proj/T/bin/Debug/net10.0/T.dll
|
||||
|
|
@ -40,6 +40,25 @@ check mutation-table-nonce.log 1 "caught=2 survived=1 unviable=0 timeout=0 noapp
|
|||
NONCE=
|
||||
check dotnet-red.log 1 "FRAMEWORK dotnet passed=11 failed=1 skipped=2"
|
||||
check dotnet-red.log 1 "FAILED MyNs.AddTests.Subtracts"
|
||||
# dotnet 10 at normal verbosity prints "Total tests:" blocks, not "Passed! - Failed:". dotnet10-green/red/zero.log are
|
||||
# real runs (dotnet 10.0.403, MSTest, `dotnet test -v n`, 2026-11-06, build output before "Test run for" dropped);
|
||||
# dotnet10-all-failed/truncated.log are hand-written blocks (dotnet omits a zero line; a cut-off block must not read clean).
|
||||
check dotnet10-green.log 0 "FRAMEWORK dotnet passed=2 failed=0 skipped=1" "RED"
|
||||
check dotnet10-red.log 1 "FRAMEWORK dotnet passed=1 failed=1 skipped=1"
|
||||
check dotnet10-red.log 1 "FAILED Bad"
|
||||
check dotnet10-red.log 0 "MISMATCH: exit 0 but the log shows"
|
||||
check dotnet10-zero.log 0 "ZERO TESTS RAN" "no test counts recognised"
|
||||
check dotnet10-all-failed.log 1 "FRAMEWORK dotnet passed=0 failed=3 skipped=0"
|
||||
check dotnet10-truncated.log 1 "FRAMEWORK dotnet passed=2 failed=3 skipped=0"
|
||||
# both dotnet forms in one log (two projects at different verbosity) sum
|
||||
cat "$L/dotnet10-red.log" "$L/dotnet-red.log" >"$T/dotnet-both.log"
|
||||
check "$T/dotnet-both.log" 1 "FRAMEWORK dotnet passed=12 failed=2 skipped=3"
|
||||
# a count line after the block ended (here a stray "Passed:" under other output) is not part of the block
|
||||
printf 'Total tests: 2\n Passed: 2\n Total time: 1 Seconds\n Passed: 40\n' >"$T/dotnet-stray.log"
|
||||
check "$T/dotnet-stray.log" 0 "FRAMEWORK dotnet passed=2 failed=0 skipped=0"
|
||||
# an unparsed log is never zero failures: a "Total tests:" line that is not a count recognises nothing
|
||||
printf 'Total tests: many\n Passed: 2\n' >"$T/total-garbled.log"
|
||||
check "$T/total-garbled.log" 0 "no test counts recognised"
|
||||
check vitest-red.log 1 "FRAMEWORK vitest passed=10 failed=1 skipped=1"
|
||||
check vitest-red.log 1 "FAILED src/add.test.ts > add > carries"
|
||||
check pytest-red.log 1 "FRAMEWORK pytest passed=5 failed=1 skipped=1"
|
||||
|
|
|
|||
19
tools/testparse.sh
Normal file → Executable file
19
tools/testparse.sh
Normal file → Executable file
|
|
@ -29,6 +29,22 @@ tp_scan() {
|
|||
}
|
||||
return s
|
||||
}
|
||||
function dflush() { # close a dotnet 10 "Total tests:" block
|
||||
if (!dblk) return
|
||||
# dotnet omits a zero line, so Failed may be absent; a count the block does not account for is a
|
||||
# failure, never a pass (a truncated block must not read as clean)
|
||||
rem = dtot - bp - bf - bs; if (rem < 0) rem = 0
|
||||
dp += bp; df += bf + rem; ds += bs; dotnet = 1; dblk = 0
|
||||
}
|
||||
/^Total tests: *[0-9]+ *$/ { # dotnet 10, normal verbosity: a block, one count per indented line
|
||||
dflush(); dtot = $3 + 0; bp = bf = bs = 0; dblk = 1; next
|
||||
}
|
||||
dblk && /^ +(Passed|Failed|Skipped): +[0-9]+ *$/ {
|
||||
v = $2 + 0
|
||||
if ($1 == "Passed:") bp += v; else if ($1 == "Failed:") bf += v; else bs += v
|
||||
next
|
||||
}
|
||||
dblk { dflush() } # any other line ends the block, and is then scanned as usual
|
||||
/^test result: / { # cargo, one line per test binary
|
||||
line = $0; sub(/^test result: [A-Za-z]+\. /, "", line)
|
||||
n = split(line, parts, "; ")
|
||||
|
|
@ -42,7 +58,7 @@ tp_scan() {
|
|||
}
|
||||
/^test .* \.\.\. FAILED$/ { name = $0; sub(/^test /, "", name); sub(/ \.\.\. FAILED$/, "", name); print "FAILED " name; next }
|
||||
/^(Passed|Failed)! - Failed: *[0-9]+, Passed: *[0-9]+, Skipped: *[0-9]+, Total:/ { # dotnet
|
||||
line = $0; sub(/^[A-Za-z]+! - /, "", line)
|
||||
line = $0; sub(/^[A-Za-z]+! - /, "", line) # dotnet, one-line summary (quiet verbosity)
|
||||
n = split(line, parts, ", ")
|
||||
for (i = 1; i <= n; i++) {
|
||||
v = parts[i]; sub(/^[A-Za-z]+: */, "", v); v += 0
|
||||
|
|
@ -108,6 +124,7 @@ tp_scan() {
|
|||
}
|
||||
/^FAILED [^ ]+::/ { print "FAILED " pyname(substr($0, 8)); next }
|
||||
END {
|
||||
dflush()
|
||||
if (cargo) printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n", cp, cf, cs
|
||||
if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds
|
||||
if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs
|
||||
|
|
|
|||
Loading…
Reference in a new issue