Export conformance capability matrix
This commit is contained in:
parent
d158425ace
commit
ecf7bc73fb
6 changed files with 150 additions and 1 deletions
|
|
@ -280,6 +280,13 @@ Exit criteria:
|
|||
|
||||
Recently cut:
|
||||
|
||||
- Conformance export now carries a first-class
|
||||
`gamecult.eve.capability_matrix.v1` projection. The matrix summarizes pack
|
||||
health, plugin ABI/capability claims, provider surfaces/commands/receipts,
|
||||
runtime feature/capture/command coverage, split readiness, and handoff counts
|
||||
from the exported evidence. The generic consumer smoke asserts it with
|
||||
`--expect-capability-matrix`, so owner repos do not need to scrape the
|
||||
Markdown report or rebuild Eve's summary logic to understand coverage.
|
||||
- Conformance export now copies handoff documents into `handoffs/*.json` rather
|
||||
than exporting Eve-local source paths only. The export carries handoff export
|
||||
paths for EveConformance, Sai/Norn plugins, Aetheria, and EveUnity; the
|
||||
|
|
|
|||
|
|
@ -90,6 +90,12 @@ Consumer smokes assert those exported copies, not just Eve-local source paths,
|
|||
so owner repos can inspect plugin, provider, runtime, and conformance handoff
|
||||
maps from the exported pack itself.
|
||||
|
||||
The conformance export also carries a `gamecult.eve.capability_matrix.v1`
|
||||
projection. It summarizes pack health, plugin ABI/capability claims, provider
|
||||
surfaces/commands/receipts, runtime feature/capture/command coverage, split
|
||||
readiness, and handoff counts. This is derived evidence for consumers, not a
|
||||
new source of truth.
|
||||
|
||||
The conformance export includes runtime `capabilityManifestPath` and
|
||||
`capabilityManifestErrors` fields so external consumers can distinguish a
|
||||
generic projection runtime from a runtime that can project plugin-declared
|
||||
|
|
|
|||
|
|
@ -57,6 +57,15 @@
|
|||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"capabilityMatrix": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"schema": {
|
||||
"const": "gamecult.eve.capability_matrix.v1"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"plugins": {
|
||||
"type": "array"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ Copy-Item -LiteralPath $sourceExport -Destination $consumerExport -Recurse
|
|||
|
||||
Push-Location $projectRoot
|
||||
try {
|
||||
node .\tools\conformance\consume-export.mjs $consumerExport --expect-conformance-handoff
|
||||
node .\tools\conformance\consume-export.mjs $consumerExport --expect-capability-matrix --expect-conformance-handoff
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Conformance consumer smoke failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ if (!exportDirectory) {
|
|||
" --expect-runtime <id>",
|
||||
" --expect-scenario <id>",
|
||||
" --expect-split-target <id>",
|
||||
" --expect-capability-matrix",
|
||||
" --expect-conformance-handoff",
|
||||
" --expect-plugin-operation <pluginId:operation>",
|
||||
" --expect-plugin-capability <pluginId:capability>",
|
||||
|
|
@ -113,6 +114,9 @@ function validateIndex(index, directory, expectations, errors) {
|
|||
if (expectations.conformanceHandoff) {
|
||||
validateExportedHandoff(index.conformanceHandoffExportPath, directory, "conformanceHandoff", errors);
|
||||
}
|
||||
if (expectations.capabilityMatrix) {
|
||||
validateCapabilityMatrix(index.capabilityMatrix, packs, plugins, providers, runtimes, splitTargets, errors);
|
||||
}
|
||||
|
||||
for (const expectedFixture of expectations.fixtures) {
|
||||
if (!fixtureIds.has(expectedFixture)) errors.push(`fixture:${expectedFixture}:missing`);
|
||||
|
|
@ -319,6 +323,51 @@ function validateExportedHandoff(exportPath, directory, label, errors) {
|
|||
}
|
||||
}
|
||||
|
||||
function validateCapabilityMatrix(matrix, packs, plugins, providers, runtimes, splitTargets, errors) {
|
||||
if (!matrix) {
|
||||
errors.push("capabilityMatrix:missing");
|
||||
return;
|
||||
}
|
||||
if (matrix.schema !== "gamecult.eve.capability_matrix.v1") {
|
||||
errors.push(`capabilityMatrix:schema:expected gamecult.eve.capability_matrix.v1 got ${matrix.schema || ""}`);
|
||||
}
|
||||
if ((matrix.packs || []).length !== packs.length) {
|
||||
errors.push(`capabilityMatrix:packs:expected ${packs.length} got ${(matrix.packs || []).length}`);
|
||||
}
|
||||
if ((matrix.plugins || []).length !== plugins.length) {
|
||||
errors.push(`capabilityMatrix:plugins:expected ${plugins.length} got ${(matrix.plugins || []).length}`);
|
||||
}
|
||||
if ((matrix.providers || []).length !== providers.length) {
|
||||
errors.push(`capabilityMatrix:providers:expected ${providers.length} got ${(matrix.providers || []).length}`);
|
||||
}
|
||||
if ((matrix.runtimes || []).length !== runtimes.length) {
|
||||
errors.push(`capabilityMatrix:runtimes:expected ${runtimes.length} got ${(matrix.runtimes || []).length}`);
|
||||
}
|
||||
if ((matrix.splitTargets || []).length !== splitTargets.length) {
|
||||
errors.push(`capabilityMatrix:splitTargets:expected ${splitTargets.length} got ${(matrix.splitTargets || []).length}`);
|
||||
}
|
||||
for (const packId of ["core", "plugin", "provider", "runtime"]) {
|
||||
if (!(matrix.packs || []).some(pack => pack.packId === packId)) {
|
||||
errors.push(`capabilityMatrix:packs:${packId}:missing`);
|
||||
}
|
||||
}
|
||||
for (const plugin of plugins) {
|
||||
if (!(matrix.plugins || []).some(candidate => candidate.pluginId === plugin.pluginId)) {
|
||||
errors.push(`capabilityMatrix:plugins:${plugin.pluginId}:missing`);
|
||||
}
|
||||
}
|
||||
for (const provider of providers) {
|
||||
if (!(matrix.providers || []).some(candidate => candidate.providerId === provider.providerId)) {
|
||||
errors.push(`capabilityMatrix:providers:${provider.providerId}:missing`);
|
||||
}
|
||||
}
|
||||
for (const runtime of runtimes) {
|
||||
if (!(matrix.runtimes || []).some(candidate => candidate.runtimeId === runtime.runtimeId)) {
|
||||
errors.push(`capabilityMatrix:runtimes:${runtime.runtimeId}:missing`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseArguments(args) {
|
||||
const exportPath = args[0] ? path.resolve(args[0]) : "";
|
||||
const expectations = {
|
||||
|
|
@ -344,6 +393,7 @@ function parseArguments(args) {
|
|||
splitTargetStatuses: [],
|
||||
splitTargetBlockers: [],
|
||||
splitTargetProofs: [],
|
||||
capabilityMatrix: false,
|
||||
conformanceHandoff: false,
|
||||
};
|
||||
const optionTargets = new Map([
|
||||
|
|
@ -373,6 +423,10 @@ function parseArguments(args) {
|
|||
|
||||
for (let index = 1; index < args.length; index += 1) {
|
||||
const option = args[index];
|
||||
if (option === "--expect-capability-matrix") {
|
||||
expectations.capabilityMatrix = true;
|
||||
continue;
|
||||
}
|
||||
if (option === "--expect-conformance-handoff") {
|
||||
expectations.conformanceHandoff = true;
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -1355,6 +1355,7 @@ function buildConformanceExport(report) {
|
|||
conformanceHandoffPath: report.repoStrategy.conformanceHandoffPath || "",
|
||||
conformanceHandoffExportPath: makeHandoffExportPath("conformance", "EveConformance", report.repoStrategy.conformanceHandoffPath || ""),
|
||||
packs,
|
||||
capabilityMatrix: buildCapabilityMatrix(report),
|
||||
plugins: (report.plugins || []).map(plugin => ({
|
||||
pluginId: plugin.pluginId,
|
||||
status: plugin.status,
|
||||
|
|
@ -1404,6 +1405,77 @@ function buildConformanceExport(report) {
|
|||
};
|
||||
}
|
||||
|
||||
function buildCapabilityMatrix(report) {
|
||||
const packs = (report.conformancePacks || []).map(pack => {
|
||||
const fixtures = (report.fixtures || []).filter(fixture => fixture.pack === pack.id);
|
||||
return {
|
||||
packId: pack.id,
|
||||
ownerRepo: pack.ownerRepo || "",
|
||||
fixtureCount: fixtures.length,
|
||||
passedFixtures: fixtures.filter(fixture => fixture.status === "pass").length,
|
||||
failedFixtures: fixtures.filter(fixture => fixture.status !== "pass").length,
|
||||
exitCriteria: pack.exitCriteria || "",
|
||||
};
|
||||
});
|
||||
|
||||
const plugins = (report.plugins || []).map(plugin => ({
|
||||
pluginId: plugin.pluginId,
|
||||
ownerRepo: plugin.ownerRepo,
|
||||
status: plugin.status,
|
||||
capabilities: plugin.capabilities || [],
|
||||
abiOperations: plugin.abiOperations || [],
|
||||
handoffExportPath: makeHandoffExportPath("plugin", plugin.pluginId, plugin.handoffPath),
|
||||
}));
|
||||
|
||||
const providers = (report.providers || []).map(provider => ({
|
||||
providerId: provider.providerId,
|
||||
ownerRepo: provider.ownerRepo,
|
||||
status: provider.status,
|
||||
surfaces: provider.surfaceIds || [],
|
||||
commands: provider.commandIds || [],
|
||||
receiptStates: provider.scenarioReceiptStates || [],
|
||||
handoffExportPath: makeHandoffExportPath("provider", provider.providerId, provider.handoffPath),
|
||||
}));
|
||||
|
||||
const runtimes = (report.runtimes || []).map(runtime => ({
|
||||
runtimeId: runtime.id,
|
||||
ownerRepo: runtime.ownerRepo,
|
||||
splitTarget: runtime.splitTarget,
|
||||
status: runtime.status,
|
||||
supportedFeatures: runtime.supportedFeatures || [],
|
||||
supportedPlugins: runtime.supportedPlugins || [],
|
||||
unsupportedPlugins: runtime.unsupportedPlugins || [],
|
||||
commandTransportSchema: runtime.commandTransportSmoke?.schema || "",
|
||||
captureStatus: runtime.capture?.status || "",
|
||||
splitHandoffExportPath: makeHandoffExportPath("runtime", runtime.id, runtime.splitHandoffPath || ""),
|
||||
}));
|
||||
|
||||
const splitTargets = (report.splitTargets || []).map(target => ({
|
||||
id: target.id,
|
||||
ownerRepo: target.ownerRepo,
|
||||
status: target.status,
|
||||
blockerCount: (target.blockers || []).length,
|
||||
passedProofs: (target.proofs || []).filter(proof => proof.status === "passed").length,
|
||||
}));
|
||||
|
||||
return {
|
||||
schema: "gamecult.eve.capability_matrix.v1",
|
||||
generatedAt: report.generatedAt,
|
||||
summary: report.summary || {},
|
||||
handoffs: {
|
||||
conformance: report.repoStrategy.conformanceHandoffPath ? 1 : 0,
|
||||
plugins: plugins.filter(plugin => plugin.handoffExportPath).length,
|
||||
providers: providers.filter(provider => provider.handoffExportPath).length,
|
||||
runtimes: runtimes.filter(runtime => runtime.splitHandoffExportPath).length,
|
||||
},
|
||||
packs,
|
||||
plugins,
|
||||
providers,
|
||||
runtimes,
|
||||
splitTargets,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeConformanceExport(conformanceExport, directory) {
|
||||
await mkdir(path.join(directory, "packs"), { recursive: true });
|
||||
await mkdir(path.join(directory, "handoffs"), { recursive: true });
|
||||
|
|
@ -1463,6 +1535,7 @@ function renderConformanceExportMarkdown(conformanceExport) {
|
|||
"",
|
||||
`Boundary rule: ${conformanceExport.boundaryRule || "not declared"}`,
|
||||
`Conformance handoff: ${conformanceExport.conformanceHandoffExportPath || conformanceExport.conformanceHandoffPath || "not declared"}`,
|
||||
`Capability matrix: ${conformanceExport.capabilityMatrix?.schema || "not declared"}`,
|
||||
"",
|
||||
"## Packs",
|
||||
"",
|
||||
|
|
|
|||
Loading…
Reference in a new issue