From e01b71495a78bcc9c17679b9049da1045b52653f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 9 Jul 2026 12:17:51 +0100 Subject: [PATCH] Export screenshot comparison metrics --- docs/eve-migration-roadmap.md | 7 + docs/parity-testing-harness.md | 15 +- ...cult.eve.conformance_export.v1.schema.json | 71 ++++++++ scripts/run-conformance-consumer-smoke.ps1 | 5 + ...ntime-owner-conformance-consumer-smoke.ps1 | 7 + tools/conformance/consume-export.mjs | 56 +++++++ tools/parity/run-parity.mjs | 156 ++++++++++++++++++ 7 files changed, 314 insertions(+), 3 deletions(-) diff --git a/docs/eve-migration-roadmap.md b/docs/eve-migration-roadmap.md index 8afa1b7..f9d48a6 100644 --- a/docs/eve-migration-roadmap.md +++ b/docs/eve-migration-roadmap.md @@ -286,6 +286,13 @@ Exit criteria: Recently cut: +- The conformance export now carries `screenshotComparisonMetrics[]` records. + Runtime owners get structure, color-token, bounding-box, and text-presence + metrics per runtime/fixture pair without relying on byte-identical screenshots + or Eve-local prose. Structure and token metrics are surface-contract evidence; + text presence is authored text evidence; bounding boxes stay + `pending-runtime-probe` or `pending-capture` until a runtime-owned capture or + layout probe attaches real visual evidence. - Runtime capability manifests now declare world-surface lowering targets. `web/eve-runtime-capability.json` claims `web-reference`, and `packages/org.gamecult.eve.unity-uitoolkit/eve-runtime-capability.json` diff --git a/docs/parity-testing-harness.md b/docs/parity-testing-harness.md index 9f177bf..41fc2d4 100644 --- a/docs/parity-testing-harness.md +++ b/docs/parity-testing-harness.md @@ -419,12 +419,21 @@ The first pass is semantic and fixture-driven: - check required bindings; - check retained slider skins and control parts. - record the responsive viewport matrix used by screenshot smoke. +- export `screenshotComparisonMetrics[]` records for structure, color tokens, + bounding boxes, and text presence. This already catches the most embarrassing class of parity lie: a runtime or fixture claiming to render CultUI while the retained tree no longer contains the partitions, bindings, tokens, or slider anatomy the renderer is supposed to lower. +Screenshot comparison metrics are intentionally not byte-identical image +checks. Structure and color-token scores come from the authored surface +contract; text-presence scores come from authored text-bearing nodes; bounding +box records name whether authored layout boxes exist or whether a runtime still +owes a capture/layout probe. A missing Unity, Electron, TUI, or Direct2D capture +body therefore appears as `pending-capture`, not as fake visual parity. + The fixture matrix also includes `embedded-surface`, which proves that nested CultMesh document slots survive the shared surface contract. The web reference, Flutter parity body, Unity UI Toolkit lowerer, iOS UIKit renderer, and native @@ -492,9 +501,9 @@ Pending runtimes are allowed. Silent fake parity is not. ## Next Cuts -1. Add screenshot comparison metrics that score structure, color tokens, - bounding boxes, and text presence without pretending byte-identical pixels - are the goal. +1. Attach runtime-owned image/layout probes to the screenshot comparison metric + lane so bounding-box records can move from `pending-runtime-probe` or + `pending-capture` to measured runtime evidence. 2. Normalize text scale and font loading across web, Flutter Android, Flutter desktop, and iOS. 3. Give iOS a real `vn.stage` scene compositor instead of compact stacked diff --git a/schemas/gamecult.eve.conformance_export.v1.schema.json b/schemas/gamecult.eve.conformance_export.v1.schema.json index e92e64f..c969631 100644 --- a/schemas/gamecult.eve.conformance_export.v1.schema.json +++ b/schemas/gamecult.eve.conformance_export.v1.schema.json @@ -19,6 +19,7 @@ "interactiveWorldSurfaces", "worldSurfaceLoweringCoverage", "commandBoundaryCoverage", + "screenshotComparisonMetrics", "splitTargetBlockers", "splitHandoffMoveCoverage", "splitTargets" @@ -684,6 +685,76 @@ "additionalProperties": true } }, + "screenshotComparisonMetrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "runtimeId", + "runtimeStatus", + "runtimeOwnerRepo", + "fixtureId", + "fixturePack", + "captureStatus", + "metricKind", + "status", + "score", + "evidenceLayer", + "detail" + ], + "properties": { + "runtimeId": { + "type": "string", + "minLength": 1 + }, + "runtimeStatus": { + "type": "string" + }, + "runtimeOwnerRepo": { + "type": "string", + "minLength": 1 + }, + "splitTarget": { + "type": "string" + }, + "fixtureId": { + "type": "string", + "minLength": 1 + }, + "fixturePack": { + "type": "string" + }, + "fixtureOwnerRepo": { + "type": "string" + }, + "captureStatus": { + "type": "string" + }, + "metricKind": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "minLength": 1 + }, + "score": { + "type": "number" + }, + "expected": {}, + "actual": {}, + "evidenceLayer": { + "type": "string", + "minLength": 1 + }, + "detail": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": true + } + }, "plugins": { "type": "array", "items": { diff --git a/scripts/run-conformance-consumer-smoke.ps1 b/scripts/run-conformance-consumer-smoke.ps1 index 44b6c25..9f89975 100644 --- a/scripts/run-conformance-consumer-smoke.ps1 +++ b/scripts/run-conformance-consumer-smoke.ps1 @@ -62,6 +62,11 @@ try { --expect-command-boundary-coverage aetheria:aetheria.daemon.game:tui:covered:EveTui:tui ` --expect-command-boundary-coverage aetheria:aetheria.daemon.editor:electron-shell:covered:EveElectron:electron-shell ` --expect-command-boundary-coverage aetheria:aetheria.daemon.editor:tui:covered:EveTui:tui ` + --expect-screenshot-metric web:embedded-surface:structure:pass ` + --expect-screenshot-metric web:embedded-surface:color-tokens:pass ` + --expect-screenshot-metric web:embedded-surface:text-presence:pass ` + --expect-screenshot-metric web:embedded-surface:bounding-boxes:pending-runtime-probe ` + --expect-screenshot-metric electron-shell:aetheria-world:bounding-boxes:pending-capture ` --expect-capability-gap "split-target:EveUnity:EveUnity:proof:Tagged UPM release" ` --expect-conformance-handoff if ($LASTEXITCODE -ne 0) { diff --git a/scripts/run-runtime-owner-conformance-consumer-smoke.ps1 b/scripts/run-runtime-owner-conformance-consumer-smoke.ps1 index 27b92f2..caa374b 100644 --- a/scripts/run-runtime-owner-conformance-consumer-smoke.ps1 +++ b/scripts/run-runtime-owner-conformance-consumer-smoke.ps1 @@ -123,6 +123,13 @@ node $consumerScript $consumerExport ` --expect-runtime-capture-status electron-shell:missing ` --expect-runtime-capture-status tui:missing ` --expect-runtime-capture-status direct2d:missing ` + --expect-screenshot-metric web:embedded-surface:structure:pass ` + --expect-screenshot-metric web:embedded-surface:color-tokens:pass ` + --expect-screenshot-metric web:embedded-surface:text-presence:pass ` + --expect-screenshot-metric web:embedded-surface:bounding-boxes:pending-runtime-probe ` + --expect-screenshot-metric unity-scene:aetheria-world:bounding-boxes:pending-capture ` + --expect-screenshot-metric electron-shell:aetheria-world:bounding-boxes:pending-capture ` + --expect-screenshot-metric tui:aetheria-world:bounding-boxes:pending-capture ` --expect-runtime-lifecycle-status unity-uitoolkit:release:incubating-upm-package ` --expect-runtime-lifecycle-status unity-uitoolkit:test:batchmode-editmode-tests-and-consumer-build-smoke ` --expect-runtime-lifecycle-status unity-uitoolkit:capture:pending-editor-capture ` diff --git a/tools/conformance/consume-export.mjs b/tools/conformance/consume-export.mjs index 27469b4..4f9acf4 100644 --- a/tools/conformance/consume-export.mjs +++ b/tools/conformance/consume-export.mjs @@ -25,6 +25,7 @@ if (!exportDirectory) { " --expect-world-lowering-coverage ", " --expect-command-boundary-coverage ", " --expect-world-lowering-gap ", + " --expect-screenshot-metric ", " --expect-conformance-handoff", " --expect-plugin-operation ", " --expect-plugin-capability ", @@ -142,6 +143,7 @@ function validateIndex(index, directory, expectations, errors) { const interactiveWorldSurfaces = Array.isArray(index.interactiveWorldSurfaces) ? index.interactiveWorldSurfaces : []; const worldSurfaceLoweringCoverage = Array.isArray(index.worldSurfaceLoweringCoverage) ? index.worldSurfaceLoweringCoverage : []; const commandBoundaryCoverage = Array.isArray(index.commandBoundaryCoverage) ? index.commandBoundaryCoverage : []; + const screenshotComparisonMetrics = Array.isArray(index.screenshotComparisonMetrics) ? index.screenshotComparisonMetrics : []; const splitTargetBlockers = Array.isArray(index.splitTargetBlockers) ? index.splitTargetBlockers : []; const splitHandoffMoveCoverage = Array.isArray(index.splitHandoffMoveCoverage) ? index.splitHandoffMoveCoverage : []; const worldSurfaceLoweringGaps = Array.isArray(index.worldSurfaceLoweringGaps) ? index.worldSurfaceLoweringGaps : []; @@ -171,6 +173,7 @@ function validateIndex(index, directory, expectations, errors) { validateSplitHandoffMoveCoverage(index.splitHandoffMoveCoverage, errors); validateWorldSurfaceLoweringCoverage(index.worldSurfaceLoweringCoverage, errors); validateCommandBoundaryCoverage(index.commandBoundaryCoverage, errors); + validateScreenshotComparisonMetrics(index.screenshotComparisonMetrics, errors); validateWorldSurfaceLoweringGaps(index.worldSurfaceLoweringGaps, errors); for (const expectedGap of expectations.capabilityGaps) { if (!capabilityGaps.some(gap => capabilityGapText(gap).includes(expectedGap))) { @@ -278,6 +281,19 @@ function validateIndex(index, directory, expectations, errors) { errors.push(`worldSurfaceLoweringGaps:${expectation.providerId}:${expectation.surfaceId}:${expectation.targetId}:runtimeId:expected ${expectation.runtimeId} got ${gap.runtimeId || ""}`); } } + for (const expectation of expectations.screenshotMetrics) { + const metric = screenshotComparisonMetrics.find(candidate => + candidate.runtimeId === expectation.runtimeId && + candidate.fixtureId === expectation.fixtureId && + candidate.metricKind === expectation.metricKind); + if (!metric) { + errors.push(`screenshotComparisonMetrics:${expectation.runtimeId}:${expectation.fixtureId}:${expectation.metricKind}:missing`); + continue; + } + if (metric.status !== expectation.status) { + errors.push(`screenshotComparisonMetrics:${expectation.runtimeId}:${expectation.fixtureId}:${expectation.metricKind}:status:expected ${expectation.status} got ${metric.status || ""}`); + } + } for (const expectedFixture of expectations.fixtures) { if (!fixtureIds.has(expectedFixture)) errors.push(`fixture:${expectedFixture}:missing`); @@ -875,6 +891,28 @@ function validateCommandBoundaryCoverage(records, errors) { } } +function validateScreenshotComparisonMetrics(records, errors) { + if (!Array.isArray(records)) { + errors.push("screenshotComparisonMetrics:missing"); + return; + } + if (!records.length) { + errors.push("screenshotComparisonMetrics:empty"); + return; + } + for (const [index, record] of records.entries()) { + for (const field of ["runtimeId", "runtimeOwnerRepo", "fixtureId", "metricKind", "status", "evidenceLayer", "detail"]) { + if (!record?.[field]) errors.push(`screenshotComparisonMetrics:${index}:${field}:missing`); + } + if (typeof record?.score !== "number") { + errors.push(`screenshotComparisonMetrics:${index}:score:expected number`); + } + if (!["structure", "color-tokens", "bounding-boxes", "text-presence"].includes(record?.metricKind)) { + errors.push(`screenshotComparisonMetrics:${index}:metricKind:unexpected ${record?.metricKind || ""}`); + } + } +} + function validateWorldSurfaceLoweringGaps(gaps, errors) { if (!Array.isArray(gaps)) { errors.push("worldSurfaceLoweringGaps:missing"); @@ -1080,6 +1118,7 @@ function parseArguments(args) { worldLoweringCoverage: [], commandBoundaryCoverage: [], worldLoweringGaps: [], + screenshotMetrics: [], conformanceHandoff: false, }; const optionTargets = new Map([ @@ -1130,6 +1169,7 @@ function parseArguments(args) { ["--expect-world-lowering-coverage", expectations.worldLoweringCoverage], ["--expect-command-boundary-coverage", expectations.commandBoundaryCoverage], ["--expect-world-lowering-gap", expectations.worldLoweringGaps], + ["--expect-screenshot-metric", expectations.screenshotMetrics], ]); for (let index = 1; index < args.length; index += 1) { @@ -1222,6 +1262,8 @@ function parseArguments(args) { target.push(parseWorldLoweringCoverageExpectation(value)); } else if (option === "--expect-command-boundary-coverage") { target.push(parseCommandBoundaryCoverageExpectation(value)); + } else if (option === "--expect-screenshot-metric") { + target.push(parseScreenshotMetricExpectation(value)); } else if (option === "--expect-runtime-plugin-gap") { target.push(parseRuntimePluginGapExpectation(value)); } else if (option === "--expect-runtime-plugin-projection") { @@ -1237,6 +1279,20 @@ function parseArguments(args) { return { exportDirectory: exportPath, expectations }; } +function parseScreenshotMetricExpectation(value) { + const parts = value.split(":"); + if (parts.length !== 4 || parts.some(part => !part)) { + console.error(`Expected screenshot metric in form, got: ${value}`); + process.exit(2); + } + return { + runtimeId: parts[0], + fixtureId: parts[1], + metricKind: parts[2], + status: parts[3], + }; +} + function parseWorldLoweringGapExpectation(value) { const parts = value.split(":"); if (parts.length !== 5 || parts.some(part => !part)) { diff --git a/tools/parity/run-parity.mjs b/tools/parity/run-parity.mjs index f00ee54..9e8e995 100644 --- a/tools/parity/run-parity.mjs +++ b/tools/parity/run-parity.mjs @@ -97,6 +97,9 @@ async function evaluateFixture(fixture) { const authorityOwners = [...new Set(authorityWitnesses.map(witness => witness.owner))].filter(Boolean).sort(); const receiptRefs = [...new Set(authorityWitnesses.map(witness => witness.receiptRef))].filter(Boolean).sort(); const witnessRefs = [...new Set(authorityWitnesses.map(witness => witness.witnessRef))].filter(Boolean).sort(); + const textNodeCount = nodes.filter(isTextLikeNode).length; + const nonEmptyTextNodeCount = nodes.filter(hasVisibleText).length; + const layoutBoxCount = nodes.filter(hasLayoutBox).length; const metadataResult = await evaluateFixtureMetadata(fixture); const checks = []; @@ -250,6 +253,9 @@ async function evaluateFixture(fixture) { receiptRefs, witnessRefs, authorityWitnesses, + textNodeCount, + nonEmptyTextNodeCount, + layoutBoxCount, metadataPath: fixture.metadataPath || "", metadata: metadataResult.metadata, metadataErrors: metadataResult.errors, @@ -1815,6 +1821,29 @@ function firstNonEmptyString(...values) { return ""; } +function hasVisibleText(node) { + const props = objectProps(node.props); + return isTextLikeNode(node) && Boolean(firstNonEmptyString( + node.text, + node.value, + props.text, + props.value, + props.label, + props.title, + props.body, + )); +} + +function isTextLikeNode(node) { + const kind = typeof node.kind === "string" ? node.kind : ""; + return kind === "text" || kind.startsWith("text."); +} + +function hasLayoutBox(node) { + const layout = objectProps(node.layout); + return ["x", "y", "width", "height"].some(key => layout[key] !== undefined); +} + function objectProps(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : {}; } @@ -1925,6 +1954,7 @@ function buildConformanceExport(report) { interactiveWorldSurfaces: collectInteractiveWorldSurfaces(report), worldSurfaceLoweringCoverage: collectWorldSurfaceLoweringCoverage(report), commandBoundaryCoverage: collectCommandBoundaryCoverage(report), + screenshotComparisonMetrics: collectScreenshotComparisonMetrics(report), worldSurfaceLoweringGaps: collectWorldSurfaceLoweringGaps(report), splitTargetBlockers: collectSplitTargetBlockers(report), splitHandoffMoveCoverage: collectSplitHandoffMoveCoverage(report), @@ -2417,6 +2447,132 @@ function collectProviderPluginRequirementCoverage(report) { return coverage; } +function collectScreenshotComparisonMetrics(report) { + const fixtureById = new Map((report.fixtures || []).map(fixture => [fixture.id, fixture])); + const metrics = []; + for (const runtime of report.runtimes || []) { + const fixtureIds = [...new Set([ + ...(runtime.requiredFixtures || []), + ...(runtime.pluginFixtures || []), + ])].sort(); + for (const fixtureId of fixtureIds) { + const fixture = fixtureById.get(fixtureId); + if (!fixture) continue; + metrics.push(buildStructureMetric(runtime, fixture)); + metrics.push(buildColorTokenMetric(runtime, fixture)); + metrics.push(buildBoundingBoxMetric(runtime, fixture)); + metrics.push(buildTextPresenceMetric(runtime, fixture)); + } + } + return metrics.sort((left, right) => + `${left.runtimeId}:${left.fixtureId}:${left.metricKind}`.localeCompare(`${right.runtimeId}:${right.fixtureId}:${right.metricKind}`)); +} + +function buildStructureMetric(runtime, fixture) { + const checks = (fixture.checks || []).filter(check => + check.id.startsWith("component:") + || check.id.startsWith("minimum:") + || check.id.startsWith("controlPart:") + || check.id.startsWith("skin:") + || check.id.startsWith("embedded:")); + return buildScreenshotMetric(runtime, fixture, { + metricKind: "structure", + status: checks.length ? checks.every(check => check.pass) ? "pass" : "fail" : "not-required", + score: ratio(checks.filter(check => check.pass).length, checks.length), + expected: checks.map(check => check.id).sort(), + actual: { + kindCounts: fixture.kindCounts || {}, + embeddedDocuments: fixture.embeddedDocuments || [], + controlParts: fixture.controlParts || [], + controlSkins: fixture.controlSkins || [], + }, + detail: "Compares authored surface structure and slot/control anatomy, not byte-identical pixels.", + }); +} + +function buildColorTokenMetric(runtime, fixture) { + const checks = (fixture.checks || []).filter(check => check.id.startsWith("styleToken:")); + return buildScreenshotMetric(runtime, fixture, { + metricKind: "color-tokens", + status: checks.length ? checks.every(check => check.pass) ? "pass" : "fail" : "not-required", + score: ratio(checks.filter(check => check.pass).length, checks.length), + expected: checks.map(check => check.id.replace("styleToken:", "")).sort(), + actual: fixture.styleTokens || [], + detail: "Compares declared CultUI style token presence before runtime-specific color sampling exists.", + }); +} + +function buildBoundingBoxMetric(runtime, fixture) { + const captureStatus = runtime.capture?.status || ""; + const runtimeHasCaptureBody = ["chrome-headless", "ssh-png", "adb-png", "golden", "ssh-golden"].includes(captureStatus); + let status = "pass"; + let score = 1; + let detail = "Authored layout boxes are present in the surface evidence."; + if (!fixture.layoutBoxCount) { + status = runtimeHasCaptureBody ? "pending-runtime-probe" : "pending-capture"; + score = 0; + detail = runtimeHasCaptureBody + ? "Runtime capture exists, but no exported layout/bounding-box probe is attached yet." + : "Bounding-box comparison is blocked until this runtime publishes capture or layout probe artifacts."; + } + return buildScreenshotMetric(runtime, fixture, { + metricKind: "bounding-boxes", + status, + score, + expected: { + source: runtimeHasCaptureBody ? "runtime layout/capture probe" : "runtime capture artifact", + }, + actual: { + layoutBoxCount: fixture.layoutBoxCount || 0, + captureStatus, + }, + detail, + }); +} + +function buildTextPresenceMetric(runtime, fixture) { + const expectedText = (fixture.textNodeCount || 0) > 0; + const status = expectedText + ? fixture.nonEmptyTextNodeCount > 0 ? "pass" : "fail" + : "not-required"; + return buildScreenshotMetric(runtime, fixture, { + metricKind: "text-presence", + status, + score: expectedText ? ratio(fixture.nonEmptyTextNodeCount || 0, fixture.textNodeCount || 1) : 1, + expected: { + textNodes: fixture.textNodeCount || 0, + }, + actual: { + nonEmptyTextNodes: fixture.nonEmptyTextNodeCount || 0, + }, + detail: "Compares authored text presence before runtime OCR/text-layer extraction exists.", + }); +} + +function buildScreenshotMetric(runtime, fixture, metric) { + return { + runtimeId: runtime.id, + runtimeStatus: runtime.status || "", + runtimeOwnerRepo: resolveRuntimeProjectionOwnerRepo(runtime), + splitTarget: runtime.splitTarget || "", + fixtureId: fixture.id, + fixturePack: fixture.pack || "", + fixtureOwnerRepo: fixture.ownerRepo || "", + captureStatus: runtime.capture?.status || "", + metricKind: metric.metricKind, + status: metric.status, + score: Number(metric.score.toFixed(3)), + expected: metric.expected, + actual: metric.actual, + evidenceLayer: "surface-structure", + detail: metric.detail, + }; +} + +function ratio(numerator, denominator) { + return denominator ? Math.max(0, Math.min(1, numerator / denominator)) : 1; +} + function collectSplitTargetBlockers(report) { return (report.splitTargets || []).flatMap(target => target.blockerRecords || []); }