From ecf7bc73fbf2310d809352848e56836f87e2502f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 9 Jul 2026 01:57:14 +0100 Subject: [PATCH] Export conformance capability matrix --- docs/eve-migration-roadmap.md | 7 ++ docs/parity-testing-harness.md | 6 ++ ...cult.eve.conformance_export.v1.schema.json | 9 +++ scripts/run-conformance-consumer-smoke.ps1 | 2 +- tools/conformance/consume-export.mjs | 54 ++++++++++++++ tools/parity/run-parity.mjs | 73 +++++++++++++++++++ 6 files changed, 150 insertions(+), 1 deletion(-) diff --git a/docs/eve-migration-roadmap.md b/docs/eve-migration-roadmap.md index 3df7407..53204f8 100644 --- a/docs/eve-migration-roadmap.md +++ b/docs/eve-migration-roadmap.md @@ -280,6 +280,13 @@ Exit criteria: Recently cut: +- Conformance export now carries a first-class + `gamecult.eve.capability_matrix.v1` projection. The matrix summarizes pack + health, plugin ABI/capability claims, provider surfaces/commands/receipts, + runtime feature/capture/command coverage, split readiness, and handoff counts + from the exported evidence. The generic consumer smoke asserts it with + `--expect-capability-matrix`, so owner repos do not need to scrape the + Markdown report or rebuild Eve's summary logic to understand coverage. - Conformance export now copies handoff documents into `handoffs/*.json` rather than exporting Eve-local source paths only. The export carries handoff export paths for EveConformance, Sai/Norn plugins, Aetheria, and EveUnity; the diff --git a/docs/parity-testing-harness.md b/docs/parity-testing-harness.md index 1af0bd9..3e16c5e 100644 --- a/docs/parity-testing-harness.md +++ b/docs/parity-testing-harness.md @@ -90,6 +90,12 @@ Consumer smokes assert those exported copies, not just Eve-local source paths, so owner repos can inspect plugin, provider, runtime, and conformance handoff maps from the exported pack itself. +The conformance export also carries a `gamecult.eve.capability_matrix.v1` +projection. It summarizes pack health, plugin ABI/capability claims, provider +surfaces/commands/receipts, runtime feature/capture/command coverage, split +readiness, and handoff counts. This is derived evidence for consumers, not a +new source of truth. + The conformance export includes runtime `capabilityManifestPath` and `capabilityManifestErrors` fields so external consumers can distinguish a generic projection runtime from a runtime that can project plugin-declared diff --git a/schemas/gamecult.eve.conformance_export.v1.schema.json b/schemas/gamecult.eve.conformance_export.v1.schema.json index 83a95d5..6882cf2 100644 --- a/schemas/gamecult.eve.conformance_export.v1.schema.json +++ b/schemas/gamecult.eve.conformance_export.v1.schema.json @@ -57,6 +57,15 @@ "additionalProperties": true } }, + "capabilityMatrix": { + "type": "object", + "properties": { + "schema": { + "const": "gamecult.eve.capability_matrix.v1" + } + }, + "additionalProperties": true + }, "plugins": { "type": "array" }, diff --git a/scripts/run-conformance-consumer-smoke.ps1 b/scripts/run-conformance-consumer-smoke.ps1 index 467f122..ed01356 100644 --- a/scripts/run-conformance-consumer-smoke.ps1 +++ b/scripts/run-conformance-consumer-smoke.ps1 @@ -30,7 +30,7 @@ Copy-Item -LiteralPath $sourceExport -Destination $consumerExport -Recurse Push-Location $projectRoot try { - node .\tools\conformance\consume-export.mjs $consumerExport --expect-conformance-handoff + node .\tools\conformance\consume-export.mjs $consumerExport --expect-capability-matrix --expect-conformance-handoff if ($LASTEXITCODE -ne 0) { throw "Conformance consumer smoke failed with exit code $LASTEXITCODE" } diff --git a/tools/conformance/consume-export.mjs b/tools/conformance/consume-export.mjs index 44aa6a5..a3e8d70 100644 --- a/tools/conformance/consume-export.mjs +++ b/tools/conformance/consume-export.mjs @@ -16,6 +16,7 @@ if (!exportDirectory) { " --expect-runtime ", " --expect-scenario ", " --expect-split-target ", + " --expect-capability-matrix", " --expect-conformance-handoff", " --expect-plugin-operation ", " --expect-plugin-capability ", @@ -113,6 +114,9 @@ function validateIndex(index, directory, expectations, errors) { if (expectations.conformanceHandoff) { validateExportedHandoff(index.conformanceHandoffExportPath, directory, "conformanceHandoff", errors); } + if (expectations.capabilityMatrix) { + validateCapabilityMatrix(index.capabilityMatrix, packs, plugins, providers, runtimes, splitTargets, errors); + } for (const expectedFixture of expectations.fixtures) { if (!fixtureIds.has(expectedFixture)) errors.push(`fixture:${expectedFixture}:missing`); @@ -319,6 +323,51 @@ function validateExportedHandoff(exportPath, directory, label, errors) { } } +function validateCapabilityMatrix(matrix, packs, plugins, providers, runtimes, splitTargets, errors) { + if (!matrix) { + errors.push("capabilityMatrix:missing"); + return; + } + if (matrix.schema !== "gamecult.eve.capability_matrix.v1") { + errors.push(`capabilityMatrix:schema:expected gamecult.eve.capability_matrix.v1 got ${matrix.schema || ""}`); + } + if ((matrix.packs || []).length !== packs.length) { + errors.push(`capabilityMatrix:packs:expected ${packs.length} got ${(matrix.packs || []).length}`); + } + if ((matrix.plugins || []).length !== plugins.length) { + errors.push(`capabilityMatrix:plugins:expected ${plugins.length} got ${(matrix.plugins || []).length}`); + } + if ((matrix.providers || []).length !== providers.length) { + errors.push(`capabilityMatrix:providers:expected ${providers.length} got ${(matrix.providers || []).length}`); + } + if ((matrix.runtimes || []).length !== runtimes.length) { + errors.push(`capabilityMatrix:runtimes:expected ${runtimes.length} got ${(matrix.runtimes || []).length}`); + } + if ((matrix.splitTargets || []).length !== splitTargets.length) { + errors.push(`capabilityMatrix:splitTargets:expected ${splitTargets.length} got ${(matrix.splitTargets || []).length}`); + } + for (const packId of ["core", "plugin", "provider", "runtime"]) { + if (!(matrix.packs || []).some(pack => pack.packId === packId)) { + errors.push(`capabilityMatrix:packs:${packId}:missing`); + } + } + for (const plugin of plugins) { + if (!(matrix.plugins || []).some(candidate => candidate.pluginId === plugin.pluginId)) { + errors.push(`capabilityMatrix:plugins:${plugin.pluginId}:missing`); + } + } + for (const provider of providers) { + if (!(matrix.providers || []).some(candidate => candidate.providerId === provider.providerId)) { + errors.push(`capabilityMatrix:providers:${provider.providerId}:missing`); + } + } + for (const runtime of runtimes) { + if (!(matrix.runtimes || []).some(candidate => candidate.runtimeId === runtime.runtimeId)) { + errors.push(`capabilityMatrix:runtimes:${runtime.runtimeId}:missing`); + } + } +} + function parseArguments(args) { const exportPath = args[0] ? path.resolve(args[0]) : ""; const expectations = { @@ -344,6 +393,7 @@ function parseArguments(args) { splitTargetStatuses: [], splitTargetBlockers: [], splitTargetProofs: [], + capabilityMatrix: false, conformanceHandoff: false, }; const optionTargets = new Map([ @@ -373,6 +423,10 @@ function parseArguments(args) { for (let index = 1; index < args.length; index += 1) { const option = args[index]; + if (option === "--expect-capability-matrix") { + expectations.capabilityMatrix = true; + continue; + } if (option === "--expect-conformance-handoff") { expectations.conformanceHandoff = true; continue; diff --git a/tools/parity/run-parity.mjs b/tools/parity/run-parity.mjs index 0e7e9d4..d917bdb 100644 --- a/tools/parity/run-parity.mjs +++ b/tools/parity/run-parity.mjs @@ -1355,6 +1355,7 @@ function buildConformanceExport(report) { conformanceHandoffPath: report.repoStrategy.conformanceHandoffPath || "", conformanceHandoffExportPath: makeHandoffExportPath("conformance", "EveConformance", report.repoStrategy.conformanceHandoffPath || ""), packs, + capabilityMatrix: buildCapabilityMatrix(report), plugins: (report.plugins || []).map(plugin => ({ pluginId: plugin.pluginId, status: plugin.status, @@ -1404,6 +1405,77 @@ function buildConformanceExport(report) { }; } +function buildCapabilityMatrix(report) { + const packs = (report.conformancePacks || []).map(pack => { + const fixtures = (report.fixtures || []).filter(fixture => fixture.pack === pack.id); + return { + packId: pack.id, + ownerRepo: pack.ownerRepo || "", + fixtureCount: fixtures.length, + passedFixtures: fixtures.filter(fixture => fixture.status === "pass").length, + failedFixtures: fixtures.filter(fixture => fixture.status !== "pass").length, + exitCriteria: pack.exitCriteria || "", + }; + }); + + const plugins = (report.plugins || []).map(plugin => ({ + pluginId: plugin.pluginId, + ownerRepo: plugin.ownerRepo, + status: plugin.status, + capabilities: plugin.capabilities || [], + abiOperations: plugin.abiOperations || [], + handoffExportPath: makeHandoffExportPath("plugin", plugin.pluginId, plugin.handoffPath), + })); + + const providers = (report.providers || []).map(provider => ({ + providerId: provider.providerId, + ownerRepo: provider.ownerRepo, + status: provider.status, + surfaces: provider.surfaceIds || [], + commands: provider.commandIds || [], + receiptStates: provider.scenarioReceiptStates || [], + handoffExportPath: makeHandoffExportPath("provider", provider.providerId, provider.handoffPath), + })); + + const runtimes = (report.runtimes || []).map(runtime => ({ + runtimeId: runtime.id, + ownerRepo: runtime.ownerRepo, + splitTarget: runtime.splitTarget, + status: runtime.status, + supportedFeatures: runtime.supportedFeatures || [], + supportedPlugins: runtime.supportedPlugins || [], + unsupportedPlugins: runtime.unsupportedPlugins || [], + commandTransportSchema: runtime.commandTransportSmoke?.schema || "", + captureStatus: runtime.capture?.status || "", + splitHandoffExportPath: makeHandoffExportPath("runtime", runtime.id, runtime.splitHandoffPath || ""), + })); + + const splitTargets = (report.splitTargets || []).map(target => ({ + id: target.id, + ownerRepo: target.ownerRepo, + status: target.status, + blockerCount: (target.blockers || []).length, + passedProofs: (target.proofs || []).filter(proof => proof.status === "passed").length, + })); + + return { + schema: "gamecult.eve.capability_matrix.v1", + generatedAt: report.generatedAt, + summary: report.summary || {}, + handoffs: { + conformance: report.repoStrategy.conformanceHandoffPath ? 1 : 0, + plugins: plugins.filter(plugin => plugin.handoffExportPath).length, + providers: providers.filter(provider => provider.handoffExportPath).length, + runtimes: runtimes.filter(runtime => runtime.splitHandoffExportPath).length, + }, + packs, + plugins, + providers, + runtimes, + splitTargets, + }; +} + async function writeConformanceExport(conformanceExport, directory) { await mkdir(path.join(directory, "packs"), { recursive: true }); await mkdir(path.join(directory, "handoffs"), { recursive: true }); @@ -1463,6 +1535,7 @@ function renderConformanceExportMarkdown(conformanceExport) { "", `Boundary rule: ${conformanceExport.boundaryRule || "not declared"}`, `Conformance handoff: ${conformanceExport.conformanceHandoffExportPath || conformanceExport.conformanceHandoffPath || "not declared"}`, + `Capability matrix: ${conformanceExport.capabilityMatrix?.schema || "not declared"}`, "", "## Packs", "",