From f68bd4099fdd127a253cf3bed8bb8fbcc4a2bb06 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 18:14:45 +0200 Subject: [PATCH] =?UTF-8?q?testparse:=20jest=20names=20come=20only=20from?= =?UTF-8?q?=20"=E2=97=8F"=20lines=20under=20a=20FAIL=20header;=20real=20je?= =?UTF-8?q?st=20fixtures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jest without a TTY prints "FAIL jt/add.test.js" with one space; the parser matched two, so a real jest load failure read ERROR, not UNVIABLE. The old `^ +● ` denylist also named process.exit's "● process.exit called with "1"" as a killing test (CAUGHT). Names now need a FAIL file header above them; a crash with no named failing test is ERROR. The "no test counts recognised" line names jest. The synthetic jest logs and mock are replaced by real jest 29.7.0 output captured on Yggdrasil from tools/fixtures/mutation-probe/jsr (green, wrong, load, exit runs in logs/jest-real.log). Co-Authored-By: Claude Sonnet 5.5 --- tools/fixtures/logs/jest-real.log | 170 +++++++++++++++++++++ tools/fixtures/mutation-mock/jtest.sh | 19 ++- tools/fixtures/mutation-mock/mutants-h.txt | 6 + tools/stopgap/ygg-verify.sh | 2 +- tools/test-tools.sh | 21 ++- tools/testparse.sh | 18 ++- 6 files changed, 212 insertions(+), 24 deletions(-) create mode 100644 tools/fixtures/logs/jest-real.log diff --git a/tools/fixtures/logs/jest-real.log b/tools/fixtures/logs/jest-real.log new file mode 100644 index 0000000..c163cd8 --- /dev/null +++ b/tools/fixtures/logs/jest-real.log @@ -0,0 +1,170 @@ +# Real jest 29.7.0 output (node:22, no TTY, --ci) from tools/fixtures/mutation-probe/jsr run through tools/stopgap/ygg-verify.sh on Yggdrasil, 2026-10-01. Runs split by '=== MARK '; a Console-free header is one space: 'FAIL jt/add.test.js'. +=== MARK green +PASS jt/add.test.js + add + ✓ adds (2 ms) + ✓ adds zero (1 ms) + +Test Suites: 1 passed, 1 total +Tests: 2 passed, 2 total +Snapshots: 0 total +Time: 0.391 s +Ran all test suites. +=== MARK wrong +FAIL jt/add.test.js + add + ✕ adds (4 ms) + ✓ adds zero (1 ms) + + ● add › adds + + expect(received).toBe(expected) // Object.is equality + + Expected: 5 + Received: -1 + + 1 | const { add } = require('./add'); + 2 | describe('add', () => { + > 3 | test('adds', () => { expect(add(2, 3)).toBe(5); }); + | ^ + 4 | test('adds zero', () => { expect(add(0, 0)).toBe(0); }); + 5 | }); + 6 | + + at Object.toBe (jt/add.test.js:3:42) + +Test Suites: 1 failed, 1 total +Tests: 1 failed, 1 passed, 2 total +Snapshots: 0 total +Time: 0.385 s, estimated 1 s +Ran all test suites. +=== MARK load +FAIL jt/add.test.js + ● Test suite failed to run + + Jest encountered an unexpected token + + Jest failed to parse a file. This happens e.g. when your code or its dependencies use non-standard JavaScript syntax, or when Jest is not configured to support such syntax. + + Out of the box Jest supports Babel, which will be used to transform your files into valid JS based on your Babel configuration. + + By default "node_modules" folder is ignored by transformers. + + Here's what you can do: + • If you are trying to use ECMAScript Modules, see https://jestjs.io/docs/ecmascript-modules for how to enable it. + • If you are trying to use TypeScript, see https://jestjs.io/docs/getting-started#using-typescript + • To have some of your "node_modules" files transformed, you can specify a custom "transformIgnorePatterns" in your config. + • If you need a custom transformation specify a "transform" option in your config. + • If you simply want to mock your non-JS modules (e.g. binary assets) you can stub them out with the "moduleNameMapper" config option. + + You'll find more details and examples of these config options in the docs: + https://jestjs.io/docs/configuration + For information about custom transformations, see: + https://jestjs.io/docs/code-transformation + + Details: + + SyntaxError: /src/tools/fixtures/mutation-probe/jsr/jt/add.js: Unexpected token (2:21) + + 1 | module.exports = { + > 2 | add: (a, b) => a + , + | ^ + 3 | untested: () => 1, + 4 | }; + 5 | + + > 1 | const { add } = require('./add'); + | ^ + 2 | describe('add', () => { + 3 | test('adds', () => { expect(add(2, 3)).toBe(5); }); + 4 | test('adds zero', () => { expect(add(0, 0)).toBe(0); }); + + at constructor (node_modules/@babel/parser/src/parse-error.ts:96:45) + at Parser.toParseError [as raise] (node_modules/@babel/parser/src/tokenizer/index.ts:1504:19) + at Parser.raise [as unexpected] (node_modules/@babel/parser/src/tokenizer/index.ts:1544:16) + at Parser.unexpected [as parseExprAtom] (node_modules/@babel/parser/src/parser/expression.ts:1385:22) + at Parser.parseExprAtom [as parseExprSubscripts] (node_modules/@babel/parser/src/parser/expression.ts:742:23) + at Parser.parseExprSubscripts [as parseUpdate] (node_modules/@babel/parser/src/parser/expression.ts:721:21) + at Parser.parseUpdate [as parseMaybeUnary] (node_modules/@babel/parser/src/parser/expression.ts:683:23) + at Parser.parseMaybeUnary [as parseMaybeUnaryOrPrivate] (node_modules/@babel/parser/src/parser/expression.ts:417:14) + at Parser.parseMaybeUnaryOrPrivate (node_modules/@babel/parser/src/parser/expression.ts:597:12) + at Parser.parseExprOpBaseRightExpr [as parseExprOpRightExpr] (node_modules/@babel/parser/src/parser/expression.ts:582:21) + at Parser.parseExprOpRightExpr [as parseExprOp] (node_modules/@babel/parser/src/parser/expression.ts:513:27) + at Parser.parseExprOp [as parseExprOps] (node_modules/@babel/parser/src/parser/expression.ts:435:17) + at Parser.parseExprOps [as parseMaybeConditional] (node_modules/@babel/parser/src/parser/expression.ts:384:23) + at Parser.parseMaybeConditional [as parseMaybeAssign] (node_modules/@babel/parser/src/parser/expression.ts:301:21) + at Parser.parseMaybeAssign [as parseFunctionBody] (node_modules/@babel/parser/src/parser/expression.ts:2637:14) + at Parser.parseFunctionBody [as parseArrowExpression] (node_modules/@babel/parser/src/parser/expression.ts:2592:10) + at Parser.parseArrowExpression [as parseParenAndDistinguishExpression] (node_modules/@babel/parser/src/parser/expression.ts:1877:12) + at Parser.parseParenAndDistinguishExpression [as parseExprAtom] (node_modules/@babel/parser/src/parser/expression.ts:1197:21) + at Parser.parseExprAtom [as parseExprSubscripts] (node_modules/@babel/parser/src/parser/expression.ts:742:23) + at Parser.parseExprSubscripts [as parseUpdate] (node_modules/@babel/parser/src/parser/expression.ts:721:21) + at Parser.parseUpdate [as parseMaybeUnary] (node_modules/@babel/parser/src/parser/expression.ts:683:23) + at Parser.parseMaybeUnary [as parseMaybeUnaryOrPrivate] (node_modules/@babel/parser/src/parser/expression.ts:417:14) + at Parser.parseMaybeUnaryOrPrivate [as parseExprOps] (node_modules/@babel/parser/src/parser/expression.ts:429:23) + at Parser.parseExprOps [as parseMaybeConditional] (node_modules/@babel/parser/src/parser/expression.ts:384:23) + at Parser.parseMaybeConditional [as parseMaybeAssign] (node_modules/@babel/parser/src/parser/expression.ts:301:21) + at parseMaybeAssign (node_modules/@babel/parser/src/parser/expression.ts:257:12) + at Parser.callback [as allowInAnd] (node_modules/@babel/parser/src/parser/expression.ts:3227:12) + at Parser.allowInAnd [as parseMaybeAssignAllowIn] (node_modules/@babel/parser/src/parser/expression.ts:256:17) + at Parser.parseMaybeAssignAllowIn [as parseMaybeAssignAllowInOrVoidPattern] (node_modules/@babel/parser/src/parser/expression.ts:3341:17) + at Parser.parseMaybeAssignAllowInOrVoidPattern [as parseObjectProperty] (node_modules/@babel/parser/src/parser/expression.ts:2358:16) + at Parser.parseObjectProperty [as parseObjPropValue] (node_modules/@babel/parser/src/parser/expression.ts:2421:12) + at Parser.parseObjPropValue [as parsePropertyDefinition] (node_modules/@babel/parser/src/parser/expression.ts:2261:17) + at Parser.parsePropertyDefinition [as parseObjectLike] (node_modules/@babel/parser/src/parser/expression.ts:2128:21) + at Parser.parseObjectLike [as parseExprAtom] (node_modules/@babel/parser/src/parser/expression.ts:1208:21) + at Parser.parseExprAtom [as parseExprSubscripts] (node_modules/@babel/parser/src/parser/expression.ts:742:23) + at Parser.parseExprSubscripts [as parseUpdate] (node_modules/@babel/parser/src/parser/expression.ts:721:21) + at Parser.parseUpdate [as parseMaybeUnary] (node_modules/@babel/parser/src/parser/expression.ts:683:23) + at Parser.parseMaybeUnary [as parseMaybeUnaryOrPrivate] (node_modules/@babel/parser/src/parser/expression.ts:417:14) + at Parser.parseMaybeUnaryOrPrivate [as parseExprOps] (node_modules/@babel/parser/src/parser/expression.ts:429:23) + at Parser.parseExprOps [as parseMaybeConditional] (node_modules/@babel/parser/src/parser/expression.ts:384:23) + at Parser.parseMaybeConditional [as parseMaybeAssign] (node_modules/@babel/parser/src/parser/expression.ts:301:21) + at Parser.parseMaybeAssign (node_modules/@babel/parser/src/parser/expression.ts:345:25) + at Parser.parseMaybeAssign [as parseExpressionBase] (node_modules/@babel/parser/src/parser/expression.ts:226:23) + at parseExpressionBase (node_modules/@babel/parser/src/parser/expression.ts:217:39) + at Parser.callback [as allowInAnd] (node_modules/@babel/parser/src/parser/expression.ts:3222:16) + at Parser.allowInAnd [as parseExpression] (node_modules/@babel/parser/src/parser/expression.ts:217:17) + at Parser.parseExpression [as parseStatementContent] (node_modules/@babel/parser/src/parser/statement.ts:688:23) + at Parser.parseStatementContent [as parseStatementLike] (node_modules/@babel/parser/src/parser/statement.ts:482:17) + at Parser.parseStatementLike [as parseModuleItem] (node_modules/@babel/parser/src/parser/statement.ts:419:17) + at Parser.parseModuleItem [as parseBlockOrModuleBlockBody] (node_modules/@babel/parser/src/parser/statement.ts:1443:16) + at Parser.parseBlockOrModuleBlockBody [as parseBlockBody] (node_modules/@babel/parser/src/parser/statement.ts:1417:10) + at Parser.parseBlockBody [as parseProgram] (node_modules/@babel/parser/src/parser/statement.ts:229:10) + at Parser.parseProgram [as parseTopLevel] (node_modules/@babel/parser/src/parser/statement.ts:203:25) + at Parser.parseTopLevel [as parse] (node_modules/@babel/parser/src/parser/index.ts:83:25) + at parse (node_modules/@babel/parser/src/index.ts:86:38) + at parser (node_modules/@babel/core/src/parser/index.ts:29:19) + at parser.next () + at normalizeFile (node_modules/@babel/core/src/transformation/normalize-file.ts:49:24) + at normalizeFile.next () + at run (node_modules/@babel/core/src/transformation/index.ts:41:36) + at run.next () + at transform (node_modules/@babel/core/src/transform.ts:29:20) + at transform.next () + at evaluateSync (node_modules/gensync/index.js:251:28) + at sync (node_modules/gensync/index.js:89:14) + at fn (node_modules/@babel/core/src/errors/rewrite-stack-trace.ts:99:14) + at transformSync (node_modules/@babel/core/src/transform.ts:66:52) + at ScriptTransformer.transformSource (node_modules/@jest/transform/build/ScriptTransformer.js:545:31) + at ScriptTransformer._transformAndBuildScript (node_modules/@jest/transform/build/ScriptTransformer.js:674:40) + at ScriptTransformer.transform (node_modules/@jest/transform/build/ScriptTransformer.js:726:19) + at Object.require (jt/add.test.js:1:17) + +Test Suites: 1 failed, 1 total +Tests: 0 total +Snapshots: 0 total +Time: 0.44 s, estimated 1 s +Ran all test suites. +=== MARK exit + ● process.exit called with "1" + + 1 | module.exports = { + > 2 | add: (a, b) => { process.exit(1); }, + | ^ + 3 | untested: () => 1, + 4 | }; + 5 | + + at exit (jt/add.js:2:28) + at Object.add (jt/add.test.js:3:31) diff --git a/tools/fixtures/mutation-mock/jtest.sh b/tools/fixtures/mutation-mock/jtest.sh index a1d7662..3586153 100644 --- a/tools/fixtures/mutation-mock/jtest.sh +++ b/tools/fixtures/mutation-mock/jtest.sh @@ -1,10 +1,9 @@ -# Mock jest run (ignore for the log shapes: see tools/fixtures/logs/jest-*.log). BROKEN in -# add.js is a module that does not load (file header, "Test suite failed to run", no test -# named); WRONG is a real assertion failure with its "●" block. -if grep -q BROKEN add.js; then - echo " FAIL add.js"; echo " ● Test suite failed to run"; echo; echo "Test Suites: 1 failed, 1 total"; echo "Tests: 0 total"; exit 1 -fi -if grep -q WRONG add.js; then - echo " FAIL add.js"; echo " ✕ adds (2 ms)"; echo; echo " ● add › adds"; echo; echo "Tests: 1 failed, 1 passed, 2 total"; exit 1 -fi -echo "Tests: 2 passed, 2 total" +# Mock jest run: prints a segment of jest-real.log (a real jest 29.7.0 run, see tools/fixtures/logs/jest-real.log) +# picked by what add.js says. BROKEN: a module that does not load; WRONG: an assertion failure; EXIT: a test run +# that calls process.exit(1); otherwise green. +m=green +grep -q BROKEN add.js && m=load +grep -q WRONG add.js && m=wrong +grep -q EXIT add.js && m=exit +awk -v m="=== MARK $m" '/^=== MARK /{on=($0==m);next} on' jest-real.log +[ "$m" = green ] diff --git a/tools/fixtures/mutation-mock/mutants-h.txt b/tools/fixtures/mutation-mock/mutants-h.txt index de263ef..f0d2309 100644 --- a/tools/fixtures/mutation-mock/mutants-h.txt +++ b/tools/fixtures/mutation-mock/mutants-h.txt @@ -10,3 +10,9 @@ file: add.js add = 1 --- after add = WRONG +=== jest-exit +file: add.js +--- before +add = 1 +--- after +add = EXIT diff --git a/tools/stopgap/ygg-verify.sh b/tools/stopgap/ygg-verify.sh index 5513b53..5d1fea6 100644 --- a/tools/stopgap/ygg-verify.sh +++ b/tools/stopgap/ygg-verify.sh @@ -169,7 +169,7 @@ ygg_summarize() { echo "MISMATCH: exit 0 but the log shows $nf failing test names and $f failed counts; do not trust the exit" fi [ -z "$fw" ] || printf '%s\n' "$fw" - if [ -z "$fw" ]; then [ "$own" -gt 0 ] || echo "no test counts recognised in the log (cargo, dotnet, vitest, pytest)" + if [ -z "$fw" ]; then [ "$own" -gt 0 ] || echo "no test counts recognised in the log (cargo, dotnet, vitest, jest, pytest)" elif [ $((p + f)) -eq 0 ]; then echo "ZERO TESTS RAN"; fi if [ "$nf" -gt 0 ]; then printf '%s\n' "$names" | awk 'NR<=15{print "FAILED " substr($0,8)} END{if(NR>15)print "... " NR-15 " more failing tests"}' diff --git a/tools/test-tools.sh b/tools/test-tools.sh index d08b979..401e0a8 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -53,7 +53,7 @@ check no-verdict.log 97 "RED" "GREEN" # One case per finding of the 2026-10-01 Soul passes; each fails if it regresses. mt() { # mt [BUILD]: sets mt_out, mt_rc, mt_repo, mt_fd local root; root=$(mktemp -d); mt_repo=$root/repo; mt_fd=$root/fd; mkdir -p "$mt_repo/pkg" "$root/mt" "$mt_fd" - cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/"; rm -rf "$mt_repo/pkg/fakebin" + cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/"; rm -rf "$mt_repo/pkg/fakebin"; cp "$L/jest-real.log" "$mt_repo/pkg/" git -C "$mt_repo" init -q && git -C "$mt_repo" add -A && git -C "$mt_repo" -c user.name=t -c user.email=t@t commit -qm mock cp "$here/testparse.sh" "$here/mutation-table.sh" "$root/mt/"; cp "$here/fixtures/mutation-mock/$1" "$root/mt/mutants.txt" mt_out=$(PATH="$here/fixtures/mutation-mock/fakebin:$PATH" FAKE_DOCKER=$mt_fd YGG_WORK=$mt_repo YGG_IMAGE=base YGG_CNAME=job1 YGG_LABEL=eureka-verify-job=job1 \ @@ -100,9 +100,10 @@ mtclean C # D: a test file that fails to load is not a kill; a named failing test is mt mutants-d.txt 'bash vtest.sh' mtcheck D-import-failure "syntax-break | UNVIABLE" "wrong-result | CAUGHT | add.js > adds" "caught=1 survived=0 unviable=1" -# D2 (jest): "FAIL " with "Test suite failed to run" is a load failure; the "●" block names the test +# D2 (jest, real output): "FAIL " with "Test suite failed to run" is a load failure; the "●" block names the +# test; a test run that calls process.exit prints a "●" line under no FAIL header, which names nothing (ERROR, not CAUGHT) mt mutants-h.txt 'bash jtest.sh' -mtcheck D2-jest "jest-load | UNVIABLE" "jest-wrong | CAUGHT | add › adds" "caught=1 survived=0 unviable=1" +mtcheck D2-jest "jest-load | UNVIABLE" "jest-wrong | CAUGHT | add › adds" "jest-exit | ERROR | exit 1, no failing test parsed" "!process.exit called | CAUGHT" "caught=1 survived=0 unviable=1 timeout=0 noapply=0 error=1" # E: rows print as mutants finish, so a job timeout keeps them; the TERM the timeout sends still cleans up MT_WALL=8 mt mutants-e.txt 'bash test.sh' mtcheck E-live-rows "[1/2] quick | CAUGHT" "!MUTATION" @@ -180,8 +181,14 @@ else echo "FAIL wait foreign done: exit $st, output: $out"; fail=1; fi check vitest-project-load-failure.log 1 "FAILED TO LOAD |unit| src/add.test.ts" "FAILED |unit|" check vitest-spaces-load-failure.log 1 "FAILED TO LOAD src/my tests/add.test.ts" "FAILED src/my" check vitest-project-red.log 1 "FAILED |unit| src/add.test.ts > add > carries" "FAILED TO LOAD" -check jest-red.log 1 "FRAMEWORK jest passed=1 failed=1 skipped=1" -check jest-red.log 1 "FAILED add › carries" "FAILED TO LOAD" -check jest-load-failure.log 1 "FAILED TO LOAD src/add.test.js" "FAILED src/add.test.js" -check jest-load-failure.log 1 "FRAMEWORK jest passed=0 failed=0 skipped=0" +# jest, real output (jest-real.log): one space after FAIL, no TTY. Names come from the "●" lines under a FAIL header only. +check "$(seg jest-real.log green)" 0 "FRAMEWORK jest passed=2 failed=0 skipped=0" "no test counts recognised" +check "$(seg jest-real.log wrong)" 1 "FRAMEWORK jest passed=1 failed=1 skipped=0" +check "$(seg jest-real.log wrong)" 1 "FAILED add › adds" "FAILED TO LOAD" +check "$(seg jest-real.log load)" 1 "FAILED TO LOAD jt/add.test.js" "FAILED jt/add.test.js" +check "$(seg jest-real.log load)" 1 "FRAMEWORK jest passed=0 failed=0 skipped=0" "FAILED Test suite" +# F3: process.exit(1) in a test run prints a "●" line under no FAIL header; it is a crash, not a named kill +check "$(seg jest-real.log exit)" 1 "exit non-zero with no failing test parsed" "FAILED process.exit" +# F4: the "nothing recognised" line names every framework the parser reads +check "$(seg jest-real.log exit)" 1 "no test counts recognised in the log (cargo, dotnet, vitest, jest, pytest)" exit $fail diff --git a/tools/testparse.sh b/tools/testparse.sh index a60b308..4bae433 100644 --- a/tools/testparse.sh +++ b/tools/testparse.sh @@ -77,13 +77,19 @@ tp_scan() { } jest = 1; next } - # vitest and jest print " FAIL " for a file and " FAIL > > " (vitest, - # optionally with a "|project|" prefix) for a test. A line with no " > " names a FILE: a vitest load - # failure (" [ ]"), or a jest file header whose tests are named by the "●" lines below. - /^ *FAIL / { name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next } - /^ +● / { # jest failure block header: " ● suite › test" + # vitest and jest print "FAIL " for a file and " FAIL > > " (vitest, + # optionally with a "|project|" prefix) for a test. Vitest and a jest TTY pad the word with two spaces; + # jest without a TTY, which is how every job runs it, prints ONE ("FAIL jt/add.test.js"). A line with + # no " > " names a FILE: a vitest load failure (" [ ]"), or a jest file header whose + # tests are named by the "●" lines below it. + /^ *FAIL +/ { infail = 1; name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next } + /^ *PASS +/ || /^Test Suites:/ { infail = 0 } + # jest failure block header " ● suite › test". It names a test only under a FAIL file header: a "●" + # elsewhere is output of the run itself (process.exit prints "● process.exit called with ..." with no + # FAIL header at all), and a crash that names no test stays unnamed, so callers read it as ERROR. + /^ +● / { name = $0; sub(/^ +● +/, "", name) - if (name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name + if (infail && name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name next } /^(=+ )?(no tests ran|[0-9]+ [a-z]+(, [0-9]+ [a-z]+)*) in [0-9.]+s( \([0-9:]+\))?( =+)?$/ { # pytest summary, with or without -q