diff --git a/tools/fixtures/logs/jest-real.log b/tools/fixtures/logs/jest-real.log new file mode 100644 index 0000000..c163cd8 --- /dev/null +++ b/tools/fixtures/logs/jest-real.log @@ -0,0 +1,170 @@ +# Real jest 29.7.0 output (node:22, no TTY, --ci) from tools/fixtures/mutation-probe/jsr run through tools/stopgap/ygg-verify.sh on Yggdrasil, 2026-10-01. Runs split by '=== MARK '; a Console-free header is one space: 'FAIL jt/add.test.js'. +=== MARK green +PASS jt/add.test.js + add + ✓ adds (2 ms) + ✓ adds zero (1 ms) + +Test Suites: 1 passed, 1 total +Tests: 2 passed, 2 total +Snapshots: 0 total +Time: 0.391 s +Ran all test suites. +=== MARK wrong +FAIL jt/add.test.js + add + ✕ adds (4 ms) + ✓ adds zero (1 ms) + + ● add › adds + + expect(received).toBe(expected) // Object.is equality + + Expected: 5 + Received: -1 + + 1 | const { add } = require('./add'); + 2 | describe('add', () => { + > 3 | test('adds', () => { expect(add(2, 3)).toBe(5); }); + | ^ + 4 | test('adds zero', () => { expect(add(0, 0)).toBe(0); }); + 5 | }); + 6 | + + at Object.toBe (jt/add.test.js:3:42) + +Test Suites: 1 failed, 1 total +Tests: 1 failed, 1 passed, 2 total +Snapshots: 0 total +Time: 0.385 s, estimated 1 s +Ran all test suites. +=== MARK load +FAIL jt/add.test.js + ● Test suite failed to run + + Jest encountered an unexpected token + + Jest failed to parse a file. This happens e.g. when your code or its dependencies use non-standard JavaScript syntax, or when Jest is not configured to support such syntax. + + Out of the box Jest supports Babel, which will be used to transform your files into valid JS based on your Babel configuration. + + By default "node_modules" folder is ignored by transformers. + + Here's what you can do: + • If you are trying to use ECMAScript Modules, see https://jestjs.io/docs/ecmascript-modules for how to enable it. + • If you are trying to use TypeScript, see https://jestjs.io/docs/getting-started#using-typescript + • To have some of your "node_modules" files transformed, you can specify a custom "transformIgnorePatterns" in your config. + • If you need a custom transformation specify a "transform" option in your config. + • If you simply want to mock your non-JS modules (e.g. binary assets) you can stub them out with the "moduleNameMapper" config option. + + You'll find more details and examples of these config options in the docs: + https://jestjs.io/docs/configuration + For information about custom transformations, see: + https://jestjs.io/docs/code-transformation + + Details: + + SyntaxError: /src/tools/fixtures/mutation-probe/jsr/jt/add.js: Unexpected token (2:21) + + 1 | module.exports = { + > 2 | add: (a, b) => a + , + | ^ + 3 | untested: () => 1, + 4 | }; + 5 | + + > 1 | const { add } = require('./add'); + | ^ + 2 | describe('add', () => { + 3 | test('adds', () => { expect(add(2, 3)).toBe(5); }); + 4 | test('adds zero', () => { expect(add(0, 0)).toBe(0); }); + + at constructor (node_modules/@babel/parser/src/parse-error.ts:96:45) + at Parser.toParseError [as raise] (node_modules/@babel/parser/src/tokenizer/index.ts:1504:19) + at Parser.raise [as unexpected] (node_modules/@babel/parser/src/tokenizer/index.ts:1544:16) + at Parser.unexpected [as parseExprAtom] (node_modules/@babel/parser/src/parser/expression.ts:1385:22) + at Parser.parseExprAtom [as parseExprSubscripts] (node_modules/@babel/parser/src/parser/expression.ts:742:23) + at Parser.parseExprSubscripts [as parseUpdate] (node_modules/@babel/parser/src/parser/expression.ts:721:21) + at Parser.parseUpdate [as parseMaybeUnary] (node_modules/@babel/parser/src/parser/expression.ts:683:23) + at Parser.parseMaybeUnary [as parseMaybeUnaryOrPrivate] (node_modules/@babel/parser/src/parser/expression.ts:417:14) + at Parser.parseMaybeUnaryOrPrivate (node_modules/@babel/parser/src/parser/expression.ts:597:12) + at Parser.parseExprOpBaseRightExpr [as parseExprOpRightExpr] (node_modules/@babel/parser/src/parser/expression.ts:582:21) + at Parser.parseExprOpRightExpr [as parseExprOp] (node_modules/@babel/parser/src/parser/expression.ts:513:27) + at Parser.parseExprOp [as parseExprOps] (node_modules/@babel/parser/src/parser/expression.ts:435:17) + at Parser.parseExprOps [as parseMaybeConditional] (node_modules/@babel/parser/src/parser/expression.ts:384:23) + at Parser.parseMaybeConditional [as parseMaybeAssign] (node_modules/@babel/parser/src/parser/expression.ts:301:21) + at Parser.parseMaybeAssign [as parseFunctionBody] (node_modules/@babel/parser/src/parser/expression.ts:2637:14) + at Parser.parseFunctionBody [as parseArrowExpression] (node_modules/@babel/parser/src/parser/expression.ts:2592:10) + at Parser.parseArrowExpression [as parseParenAndDistinguishExpression] (node_modules/@babel/parser/src/parser/expression.ts:1877:12) + at Parser.parseParenAndDistinguishExpression [as parseExprAtom] (node_modules/@babel/parser/src/parser/expression.ts:1197:21) + at Parser.parseExprAtom [as parseExprSubscripts] (node_modules/@babel/parser/src/parser/expression.ts:742:23) + at Parser.parseExprSubscripts [as parseUpdate] (node_modules/@babel/parser/src/parser/expression.ts:721:21) + at Parser.parseUpdate [as parseMaybeUnary] (node_modules/@babel/parser/src/parser/expression.ts:683:23) + at Parser.parseMaybeUnary [as parseMaybeUnaryOrPrivate] (node_modules/@babel/parser/src/parser/expression.ts:417:14) + at Parser.parseMaybeUnaryOrPrivate [as parseExprOps] (node_modules/@babel/parser/src/parser/expression.ts:429:23) + at Parser.parseExprOps [as parseMaybeConditional] (node_modules/@babel/parser/src/parser/expression.ts:384:23) + at Parser.parseMaybeConditional [as parseMaybeAssign] (node_modules/@babel/parser/src/parser/expression.ts:301:21) + at parseMaybeAssign (node_modules/@babel/parser/src/parser/expression.ts:257:12) + at Parser.callback [as allowInAnd] (node_modules/@babel/parser/src/parser/expression.ts:3227:12) + at Parser.allowInAnd [as parseMaybeAssignAllowIn] (node_modules/@babel/parser/src/parser/expression.ts:256:17) + at Parser.parseMaybeAssignAllowIn [as parseMaybeAssignAllowInOrVoidPattern] (node_modules/@babel/parser/src/parser/expression.ts:3341:17) + at Parser.parseMaybeAssignAllowInOrVoidPattern [as parseObjectProperty] (node_modules/@babel/parser/src/parser/expression.ts:2358:16) + at Parser.parseObjectProperty [as parseObjPropValue] (node_modules/@babel/parser/src/parser/expression.ts:2421:12) + at Parser.parseObjPropValue [as parsePropertyDefinition] (node_modules/@babel/parser/src/parser/expression.ts:2261:17) + at Parser.parsePropertyDefinition [as parseObjectLike] (node_modules/@babel/parser/src/parser/expression.ts:2128:21) + at Parser.parseObjectLike [as parseExprAtom] (node_modules/@babel/parser/src/parser/expression.ts:1208:21) + at Parser.parseExprAtom [as parseExprSubscripts] (node_modules/@babel/parser/src/parser/expression.ts:742:23) + at Parser.parseExprSubscripts [as parseUpdate] (node_modules/@babel/parser/src/parser/expression.ts:721:21) + at Parser.parseUpdate [as parseMaybeUnary] (node_modules/@babel/parser/src/parser/expression.ts:683:23) + at Parser.parseMaybeUnary [as parseMaybeUnaryOrPrivate] (node_modules/@babel/parser/src/parser/expression.ts:417:14) + at Parser.parseMaybeUnaryOrPrivate [as parseExprOps] (node_modules/@babel/parser/src/parser/expression.ts:429:23) + at Parser.parseExprOps [as parseMaybeConditional] (node_modules/@babel/parser/src/parser/expression.ts:384:23) + at Parser.parseMaybeConditional [as parseMaybeAssign] (node_modules/@babel/parser/src/parser/expression.ts:301:21) + at Parser.parseMaybeAssign (node_modules/@babel/parser/src/parser/expression.ts:345:25) + at Parser.parseMaybeAssign [as parseExpressionBase] (node_modules/@babel/parser/src/parser/expression.ts:226:23) + at parseExpressionBase (node_modules/@babel/parser/src/parser/expression.ts:217:39) + at Parser.callback [as allowInAnd] (node_modules/@babel/parser/src/parser/expression.ts:3222:16) + at Parser.allowInAnd [as parseExpression] (node_modules/@babel/parser/src/parser/expression.ts:217:17) + at Parser.parseExpression [as parseStatementContent] (node_modules/@babel/parser/src/parser/statement.ts:688:23) + at Parser.parseStatementContent [as parseStatementLike] (node_modules/@babel/parser/src/parser/statement.ts:482:17) + at Parser.parseStatementLike [as parseModuleItem] (node_modules/@babel/parser/src/parser/statement.ts:419:17) + at Parser.parseModuleItem [as parseBlockOrModuleBlockBody] (node_modules/@babel/parser/src/parser/statement.ts:1443:16) + at Parser.parseBlockOrModuleBlockBody [as parseBlockBody] (node_modules/@babel/parser/src/parser/statement.ts:1417:10) + at Parser.parseBlockBody [as parseProgram] (node_modules/@babel/parser/src/parser/statement.ts:229:10) + at Parser.parseProgram [as parseTopLevel] (node_modules/@babel/parser/src/parser/statement.ts:203:25) + at Parser.parseTopLevel [as parse] (node_modules/@babel/parser/src/parser/index.ts:83:25) + at parse (node_modules/@babel/parser/src/index.ts:86:38) + at parser (node_modules/@babel/core/src/parser/index.ts:29:19) + at parser.next () + at normalizeFile (node_modules/@babel/core/src/transformation/normalize-file.ts:49:24) + at normalizeFile.next () + at run (node_modules/@babel/core/src/transformation/index.ts:41:36) + at run.next () + at transform (node_modules/@babel/core/src/transform.ts:29:20) + at transform.next () + at evaluateSync (node_modules/gensync/index.js:251:28) + at sync (node_modules/gensync/index.js:89:14) + at fn (node_modules/@babel/core/src/errors/rewrite-stack-trace.ts:99:14) + at transformSync (node_modules/@babel/core/src/transform.ts:66:52) + at ScriptTransformer.transformSource (node_modules/@jest/transform/build/ScriptTransformer.js:545:31) + at ScriptTransformer._transformAndBuildScript (node_modules/@jest/transform/build/ScriptTransformer.js:674:40) + at ScriptTransformer.transform (node_modules/@jest/transform/build/ScriptTransformer.js:726:19) + at Object.require (jt/add.test.js:1:17) + +Test Suites: 1 failed, 1 total +Tests: 0 total +Snapshots: 0 total +Time: 0.44 s, estimated 1 s +Ran all test suites. +=== MARK exit + ● process.exit called with "1" + + 1 | module.exports = { + > 2 | add: (a, b) => { process.exit(1); }, + | ^ + 3 | untested: () => 1, + 4 | }; + 5 | + + at exit (jt/add.js:2:28) + at Object.add (jt/add.test.js:3:31) diff --git a/tools/fixtures/mutation-mock/jtest.sh b/tools/fixtures/mutation-mock/jtest.sh index a1d7662..3586153 100644 --- a/tools/fixtures/mutation-mock/jtest.sh +++ b/tools/fixtures/mutation-mock/jtest.sh @@ -1,10 +1,9 @@ -# Mock jest run (ignore for the log shapes: see tools/fixtures/logs/jest-*.log). BROKEN in -# add.js is a module that does not load (file header, "Test suite failed to run", no test -# named); WRONG is a real assertion failure with its "●" block. -if grep -q BROKEN add.js; then - echo " FAIL add.js"; echo " ● Test suite failed to run"; echo; echo "Test Suites: 1 failed, 1 total"; echo "Tests: 0 total"; exit 1 -fi -if grep -q WRONG add.js; then - echo " FAIL add.js"; echo " ✕ adds (2 ms)"; echo; echo " ● add › adds"; echo; echo "Tests: 1 failed, 1 passed, 2 total"; exit 1 -fi -echo "Tests: 2 passed, 2 total" +# Mock jest run: prints a segment of jest-real.log (a real jest 29.7.0 run, see tools/fixtures/logs/jest-real.log) +# picked by what add.js says. BROKEN: a module that does not load; WRONG: an assertion failure; EXIT: a test run +# that calls process.exit(1); otherwise green. +m=green +grep -q BROKEN add.js && m=load +grep -q WRONG add.js && m=wrong +grep -q EXIT add.js && m=exit +awk -v m="=== MARK $m" '/^=== MARK /{on=($0==m);next} on' jest-real.log +[ "$m" = green ] diff --git a/tools/fixtures/mutation-mock/mutants-h.txt b/tools/fixtures/mutation-mock/mutants-h.txt index de263ef..f0d2309 100644 --- a/tools/fixtures/mutation-mock/mutants-h.txt +++ b/tools/fixtures/mutation-mock/mutants-h.txt @@ -10,3 +10,9 @@ file: add.js add = 1 --- after add = WRONG +=== jest-exit +file: add.js +--- before +add = 1 +--- after +add = EXIT diff --git a/tools/stopgap/ygg-verify.sh b/tools/stopgap/ygg-verify.sh index 5513b53..5d1fea6 100644 --- a/tools/stopgap/ygg-verify.sh +++ b/tools/stopgap/ygg-verify.sh @@ -169,7 +169,7 @@ ygg_summarize() { echo "MISMATCH: exit 0 but the log shows $nf failing test names and $f failed counts; do not trust the exit" fi [ -z "$fw" ] || printf '%s\n' "$fw" - if [ -z "$fw" ]; then [ "$own" -gt 0 ] || echo "no test counts recognised in the log (cargo, dotnet, vitest, pytest)" + if [ -z "$fw" ]; then [ "$own" -gt 0 ] || echo "no test counts recognised in the log (cargo, dotnet, vitest, jest, pytest)" elif [ $((p + f)) -eq 0 ]; then echo "ZERO TESTS RAN"; fi if [ "$nf" -gt 0 ]; then printf '%s\n' "$names" | awk 'NR<=15{print "FAILED " substr($0,8)} END{if(NR>15)print "... " NR-15 " more failing tests"}' diff --git a/tools/test-tools.sh b/tools/test-tools.sh index d08b979..401e0a8 100644 --- a/tools/test-tools.sh +++ b/tools/test-tools.sh @@ -53,7 +53,7 @@ check no-verdict.log 97 "RED" "GREEN" # One case per finding of the 2026-10-01 Soul passes; each fails if it regresses. mt() { # mt [BUILD]: sets mt_out, mt_rc, mt_repo, mt_fd local root; root=$(mktemp -d); mt_repo=$root/repo; mt_fd=$root/fd; mkdir -p "$mt_repo/pkg" "$root/mt" "$mt_fd" - cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/"; rm -rf "$mt_repo/pkg/fakebin" + cp -r "$here/fixtures/mutation-mock/." "$mt_repo/pkg/"; rm -rf "$mt_repo/pkg/fakebin"; cp "$L/jest-real.log" "$mt_repo/pkg/" git -C "$mt_repo" init -q && git -C "$mt_repo" add -A && git -C "$mt_repo" -c user.name=t -c user.email=t@t commit -qm mock cp "$here/testparse.sh" "$here/mutation-table.sh" "$root/mt/"; cp "$here/fixtures/mutation-mock/$1" "$root/mt/mutants.txt" mt_out=$(PATH="$here/fixtures/mutation-mock/fakebin:$PATH" FAKE_DOCKER=$mt_fd YGG_WORK=$mt_repo YGG_IMAGE=base YGG_CNAME=job1 YGG_LABEL=eureka-verify-job=job1 \ @@ -100,9 +100,10 @@ mtclean C # D: a test file that fails to load is not a kill; a named failing test is mt mutants-d.txt 'bash vtest.sh' mtcheck D-import-failure "syntax-break | UNVIABLE" "wrong-result | CAUGHT | add.js > adds" "caught=1 survived=0 unviable=1" -# D2 (jest): "FAIL " with "Test suite failed to run" is a load failure; the "●" block names the test +# D2 (jest, real output): "FAIL " with "Test suite failed to run" is a load failure; the "●" block names the +# test; a test run that calls process.exit prints a "●" line under no FAIL header, which names nothing (ERROR, not CAUGHT) mt mutants-h.txt 'bash jtest.sh' -mtcheck D2-jest "jest-load | UNVIABLE" "jest-wrong | CAUGHT | add › adds" "caught=1 survived=0 unviable=1" +mtcheck D2-jest "jest-load | UNVIABLE" "jest-wrong | CAUGHT | add › adds" "jest-exit | ERROR | exit 1, no failing test parsed" "!process.exit called | CAUGHT" "caught=1 survived=0 unviable=1 timeout=0 noapply=0 error=1" # E: rows print as mutants finish, so a job timeout keeps them; the TERM the timeout sends still cleans up MT_WALL=8 mt mutants-e.txt 'bash test.sh' mtcheck E-live-rows "[1/2] quick | CAUGHT" "!MUTATION" @@ -180,8 +181,14 @@ else echo "FAIL wait foreign done: exit $st, output: $out"; fail=1; fi check vitest-project-load-failure.log 1 "FAILED TO LOAD |unit| src/add.test.ts" "FAILED |unit|" check vitest-spaces-load-failure.log 1 "FAILED TO LOAD src/my tests/add.test.ts" "FAILED src/my" check vitest-project-red.log 1 "FAILED |unit| src/add.test.ts > add > carries" "FAILED TO LOAD" -check jest-red.log 1 "FRAMEWORK jest passed=1 failed=1 skipped=1" -check jest-red.log 1 "FAILED add › carries" "FAILED TO LOAD" -check jest-load-failure.log 1 "FAILED TO LOAD src/add.test.js" "FAILED src/add.test.js" -check jest-load-failure.log 1 "FRAMEWORK jest passed=0 failed=0 skipped=0" +# jest, real output (jest-real.log): one space after FAIL, no TTY. Names come from the "●" lines under a FAIL header only. +check "$(seg jest-real.log green)" 0 "FRAMEWORK jest passed=2 failed=0 skipped=0" "no test counts recognised" +check "$(seg jest-real.log wrong)" 1 "FRAMEWORK jest passed=1 failed=1 skipped=0" +check "$(seg jest-real.log wrong)" 1 "FAILED add › adds" "FAILED TO LOAD" +check "$(seg jest-real.log load)" 1 "FAILED TO LOAD jt/add.test.js" "FAILED jt/add.test.js" +check "$(seg jest-real.log load)" 1 "FRAMEWORK jest passed=0 failed=0 skipped=0" "FAILED Test suite" +# F3: process.exit(1) in a test run prints a "●" line under no FAIL header; it is a crash, not a named kill +check "$(seg jest-real.log exit)" 1 "exit non-zero with no failing test parsed" "FAILED process.exit" +# F4: the "nothing recognised" line names every framework the parser reads +check "$(seg jest-real.log exit)" 1 "no test counts recognised in the log (cargo, dotnet, vitest, jest, pytest)" exit $fail diff --git a/tools/testparse.sh b/tools/testparse.sh index a60b308..4bae433 100644 --- a/tools/testparse.sh +++ b/tools/testparse.sh @@ -77,13 +77,19 @@ tp_scan() { } jest = 1; next } - # vitest and jest print " FAIL " for a file and " FAIL > > " (vitest, - # optionally with a "|project|" prefix) for a test. A line with no " > " names a FILE: a vitest load - # failure (" [ ]"), or a jest file header whose tests are named by the "●" lines below. - /^ *FAIL / { name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next } - /^ +● / { # jest failure block header: " ● suite › test" + # vitest and jest print "FAIL " for a file and " FAIL > > " (vitest, + # optionally with a "|project|" prefix) for a test. Vitest and a jest TTY pad the word with two spaces; + # jest without a TTY, which is how every job runs it, prints ONE ("FAIL jt/add.test.js"). A line with + # no " > " names a FILE: a vitest load failure (" [ ]"), or a jest file header whose + # tests are named by the "●" lines below it. + /^ *FAIL +/ { infail = 1; name = $0; sub(/^ *FAIL +/, "", name); print (name ~ / > / ? "FAILED " : "LOADFAIL ") name; next } + /^ *PASS +/ || /^Test Suites:/ { infail = 0 } + # jest failure block header " ● suite › test". It names a test only under a FAIL file header: a "●" + # elsewhere is output of the run itself (process.exit prints "● process.exit called with ..." with no + # FAIL header at all), and a crash that names no test stays unnamed, so callers read it as ERROR. + /^ +● / { name = $0; sub(/^ +● +/, "", name) - if (name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name + if (infail && name !~ /^(Test suite failed to run|Console|Validation Warning|Deprecation)/) print "FAILED " name next } /^(=+ )?(no tests ran|[0-9]+ [a-z]+(, [0-9]+ [a-z]+)*) in [0-9.]+s( \([0-9:]+\))?( =+)?$/ { # pytest summary, with or without -q