From 655357806efa9e3960d4832bdb51287a28767495 Mon Sep 17 00:00:00 2001 From: Eureka Date: Fri, 6 Nov 2026 12:34:46 +0000 Subject: [PATCH] testparse: read dotnet 10's "Total tests:" summary block dotnet 10 at normal verbosity ends a run with Total tests: N / Passed: N / Failed: N / Skipped: N (one count per indented line, zero counts omitted) instead of the one-line "Passed! - Failed: ..." form, so mutation-table.sh baselines read as "ran zero tests". tp_scan now parses both forms. A count the block does not account for (Total - Passed - Failed - Skipped) is counted as failed, so a cut-off block cannot read as clean; a "Total tests:" line that is not a count recognises nothing. Fixtures: dotnet10-green/red/zero.log are real dotnet 10.0.403 MSTest runs (-v n); dotnet10-all-failed/truncated.log are hand-written. Co-Authored-By: Claude Sonnet 5.5 --- tools/fixtures/logs/dotnet10-all-failed.log | 4 ++++ tools/fixtures/logs/dotnet10-green.log | 12 +++++++++++ tools/fixtures/logs/dotnet10-red.log | 22 +++++++++++++++++++++ tools/fixtures/logs/dotnet10-truncated.log | 3 +++ tools/fixtures/logs/dotnet10-zero.log | 3 +++ tools/test-tools.sh | 19 ++++++++++++++++++ tools/testparse.sh | 19 +++++++++++++++++- 7 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tools/fixtures/logs/dotnet10-all-failed.log create mode 100644 tools/fixtures/logs/dotnet10-green.log create mode 100644 tools/fixtures/logs/dotnet10-red.log create mode 100644 tools/fixtures/logs/dotnet10-truncated.log create mode 100644 tools/fixtures/logs/dotnet10-zero.log mode change 100644 => 100755 tools/testparse.sh diff --git a/tools/fixtures/logs/dotnet10-all-failed.log b/tools/fixtures/logs/dotnet10-all-failed.log new file mode 100644 index 0000000..6f67afa --- /dev/null +++ b/tools/fixtures/logs/dotnet10-all-failed.log @@ -0,0 +1,4 @@ +Test Run Failed. +Total tests: 3 + Failed: 3 + Total time: 0.5 Seconds diff --git a/tools/fixtures/logs/dotnet10-green.log b/tools/fixtures/logs/dotnet10-green.log new file mode 100644 index 0000000..5b091a4 --- /dev/null +++ b/tools/fixtures/logs/dotnet10-green.log @@ -0,0 +1,12 @@ +Test run for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (.NETCoreApp,Version=v10.0) +A total of 1 test files matched the specified pattern. +Test Parallelization enabled for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (Workers: 16, Scope: MethodLevel) + Skipped Skipped + Passed Bad [5 ms] + Passed Ok [5 ms] + +Test Run Successful. +Total tests: 3 + Passed: 2 + Skipped: 1 + Total time: 0.5415 Seconds diff --git a/tools/fixtures/logs/dotnet10-red.log b/tools/fixtures/logs/dotnet10-red.log new file mode 100644 index 0000000..72af692 --- /dev/null +++ b/tools/fixtures/logs/dotnet10-red.log @@ -0,0 +1,22 @@ +Test run for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (.NETCoreApp,Version=v10.0) +A total of 1 test files matched the specified pattern. +Test Parallelization enabled for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (Workers: 16, Scope: MethodLevel) + Skipped Skipped + Passed Ok [5 ms] + Failed Bad [12 ms] + Error Message: + Assert.AreEqual failed. Expected:<1>. Actual:<2>. 'expected' expression: '1', 'actual' expression: '2'. + Stack Trace: + at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.ThrowAssertAreEqualFailed(Object expected, Object actual, String userMessage) in /_/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs:line 665 + at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AreEqual[T](T expected, T actual, IEqualityComparer`1 comparer, String message, String expectedExpression, String actualExpression) in /_/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs:line 492 + at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.AreEqual[T](T expected, T actual, String message, String expectedExpression, String actualExpression) in /_/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs:line 405 + at T.Test1.Bad() in /tmp/dn/proj/T/Test1.cs:line 6 + at System.Reflection.MethodBaseInvoker.InterpretedInvoke_Method(Object obj, IntPtr* args) + at System.Reflection.MethodBaseInvoker.InvokeWithNoArgs(Object obj, BindingFlags invokeAttr) + +Test Run Failed. +Total tests: 3 + Passed: 1 + Failed: 1 + Skipped: 1 + Total time: 0.5395 Seconds diff --git a/tools/fixtures/logs/dotnet10-truncated.log b/tools/fixtures/logs/dotnet10-truncated.log new file mode 100644 index 0000000..7835443 --- /dev/null +++ b/tools/fixtures/logs/dotnet10-truncated.log @@ -0,0 +1,3 @@ +Test Run Failed. +Total tests: 5 + Passed: 2 diff --git a/tools/fixtures/logs/dotnet10-zero.log b/tools/fixtures/logs/dotnet10-zero.log new file mode 100644 index 0000000..ed1ae5c --- /dev/null +++ b/tools/fixtures/logs/dotnet10-zero.log @@ -0,0 +1,3 @@ +Test run for /tmp/dn/proj/T/bin/Debug/net10.0/T.dll (.NETCoreApp,Version=v10.0) +A total of 1 test files matched the specified pattern. +No test matches the given testcase filter `Name=Nope` in /tmp/dn/proj/T/bin/Debug/net10.0/T.dll diff --git a/tools/test-tools.sh b/tools/test-tools.sh index bedbf38..df37c05 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -40,6 +40,25 @@ check mutation-table-nonce.log 1 "caught=2 survived=1 unviable=0 timeout=0 noapp NONCE= check dotnet-red.log 1 "FRAMEWORK dotnet passed=11 failed=1 skipped=2" check dotnet-red.log 1 "FAILED MyNs.AddTests.Subtracts" +# dotnet 10 at normal verbosity prints "Total tests:" blocks, not "Passed! - Failed:". dotnet10-green/red/zero.log are +# real runs (dotnet 10.0.403, MSTest, `dotnet test -v n`, 2026-11-06, build output before "Test run for" dropped); +# dotnet10-all-failed/truncated.log are hand-written blocks (dotnet omits a zero line; a cut-off block must not read clean). +check dotnet10-green.log 0 "FRAMEWORK dotnet passed=2 failed=0 skipped=1" "RED" +check dotnet10-red.log 1 "FRAMEWORK dotnet passed=1 failed=1 skipped=1" +check dotnet10-red.log 1 "FAILED Bad" +check dotnet10-red.log 0 "MISMATCH: exit 0 but the log shows" +check dotnet10-zero.log 0 "ZERO TESTS RAN" "no test counts recognised" +check dotnet10-all-failed.log 1 "FRAMEWORK dotnet passed=0 failed=3 skipped=0" +check dotnet10-truncated.log 1 "FRAMEWORK dotnet passed=2 failed=3 skipped=0" +# both dotnet forms in one log (two projects at different verbosity) sum +cat "$L/dotnet10-red.log" "$L/dotnet-red.log" >"$T/dotnet-both.log" +check "$T/dotnet-both.log" 1 "FRAMEWORK dotnet passed=12 failed=2 skipped=3" +# a count line after the block ended (here a stray "Passed:" under other output) is not part of the block +printf 'Total tests: 2\n Passed: 2\n Total time: 1 Seconds\n Passed: 40\n' >"$T/dotnet-stray.log" +check "$T/dotnet-stray.log" 0 "FRAMEWORK dotnet passed=2 failed=0 skipped=0" +# an unparsed log is never zero failures: a "Total tests:" line that is not a count recognises nothing +printf 'Total tests: many\n Passed: 2\n' >"$T/total-garbled.log" +check "$T/total-garbled.log" 0 "no test counts recognised" check vitest-red.log 1 "FRAMEWORK vitest passed=10 failed=1 skipped=1" check vitest-red.log 1 "FAILED src/add.test.ts > add > carries" check pytest-red.log 1 "FRAMEWORK pytest passed=5 failed=1 skipped=1" diff --git a/tools/testparse.sh b/tools/testparse.sh old mode 100644 new mode 100755 index 4bae433..c4150cb --- a/tools/testparse.sh +++ b/tools/testparse.sh @@ -29,6 +29,22 @@ tp_scan() { } return s } + function dflush() { # close a dotnet 10 "Total tests:" block + if (!dblk) return + # dotnet omits a zero line, so Failed may be absent; a count the block does not account for is a + # failure, never a pass (a truncated block must not read as clean) + rem = dtot - bp - bf - bs; if (rem < 0) rem = 0 + dp += bp; df += bf + rem; ds += bs; dotnet = 1; dblk = 0 + } + /^Total tests: *[0-9]+ *$/ { # dotnet 10, normal verbosity: a block, one count per indented line + dflush(); dtot = $3 + 0; bp = bf = bs = 0; dblk = 1; next + } + dblk && /^ +(Passed|Failed|Skipped): +[0-9]+ *$/ { + v = $2 + 0 + if ($1 == "Passed:") bp += v; else if ($1 == "Failed:") bf += v; else bs += v + next + } + dblk { dflush() } # any other line ends the block, and is then scanned as usual /^test result: / { # cargo, one line per test binary line = $0; sub(/^test result: [A-Za-z]+\. /, "", line) n = split(line, parts, "; ") @@ -42,7 +58,7 @@ tp_scan() { } /^test .* \.\.\. FAILED$/ { name = $0; sub(/^test /, "", name); sub(/ \.\.\. FAILED$/, "", name); print "FAILED " name; next } /^(Passed|Failed)! - Failed: *[0-9]+, Passed: *[0-9]+, Skipped: *[0-9]+, Total:/ { # dotnet - line = $0; sub(/^[A-Za-z]+! - /, "", line) + line = $0; sub(/^[A-Za-z]+! - /, "", line) # dotnet, one-line summary (quiet verbosity) n = split(line, parts, ", ") for (i = 1; i <= n; i++) { v = parts[i]; sub(/^[A-Za-z]+: */, "", v); v += 0 @@ -108,6 +124,7 @@ tp_scan() { } /^FAILED [^ ]+::/ { print "FAILED " pyname(substr($0, 8)); next } END { + dflush() if (cargo) printf "FRAMEWORK cargo passed=%d failed=%d skipped=%d\n", cp, cf, cs if (dotnet) printf "FRAMEWORK dotnet passed=%d failed=%d skipped=%d\n", dp, df, ds if (vitest) printf "FRAMEWORK vitest passed=%d failed=%d skipped=%d\n", vp, vf, vs -- 2.47.3