Semver check: behavioural tests for the measured public API
Real assemblies built in the test and compared through the CLI: removal without ### Breaking, removal under too small a bump (post-1.0 and 0.y.z), additions alone, a declared break with nothing measured, a removed GameCult.* assembly, tool failure failing closed, inherited-member removals, an already-published version, the baseline coming from the tag's blob, and a first release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens
parent
716ef9341c
commit
8f2eab7651
1 changed files with 200 additions and 1 deletions
|
|
@ -1,6 +1,6 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
|
@ -12,6 +12,7 @@ import {
|
|||
hasBreakingSection,
|
||||
nextVersions,
|
||||
parseVersion,
|
||||
readApiCompatRun,
|
||||
} from "./check-changelog-semver.mjs";
|
||||
|
||||
const checkerPath = fileURLToPath(new URL("./check-changelog-semver.mjs", import.meta.url));
|
||||
|
|
@ -255,3 +256,201 @@ test("CLI: does not re-litigate a version already published as a tag", () => {
|
|||
assert.match(output, /already published/);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Measured public API. The fixtures are real assemblies, built here with
|
||||
// the SDK and compared by the pinned ApiCompat tool through the CLI, so every
|
||||
// rule is observed where the release scripts observe it. ---
|
||||
|
||||
const fixtureRoot = mkdtempSync(join(tmpdir(), "cultlib-semver-fixtures-"));
|
||||
process.on("exit", () => rmSync(fixtureRoot, { recursive: true, force: true }));
|
||||
const builtFixtures = new Map();
|
||||
|
||||
// Builds `source` into <assemblyName>.dll (netstandard2.1, like the shipped
|
||||
// assemblies) once per distinct (name, source) and returns the dll's path.
|
||||
function buildAssembly(assemblyName, source) {
|
||||
const key = `${assemblyName}\n${source}`;
|
||||
if (!builtFixtures.has(key)) {
|
||||
const dir = join(fixtureRoot, `fx${builtFixtures.size}`);
|
||||
mkdirSync(dir);
|
||||
writeFileSync(
|
||||
join(dir, "Fixture.csproj"),
|
||||
`<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><TargetFramework>netstandard2.1</TargetFramework>` +
|
||||
`<AssemblyName>${assemblyName}</AssemblyName><Deterministic>true</Deterministic></PropertyGroup></Project>`,
|
||||
);
|
||||
writeFileSync(join(dir, "Source.cs"), source);
|
||||
execFileSync("dotnet", ["build", "-c", "Release", "-o", "out", "--nologo", "-v", "q"], { cwd: dir, stdio: "pipe" });
|
||||
builtFixtures.set(key, join(dir, "out", `${assemblyName}.dll`));
|
||||
}
|
||||
return builtFixtures.get(key);
|
||||
}
|
||||
|
||||
const PLUGINS = "unity/widget/Runtime/Plugins";
|
||||
|
||||
// A repo whose tag widget-v<previous> tracks `baseline` ({file name: dll path})
|
||||
// under PLUGINS, and whose working tree then holds `afterTag` instead.
|
||||
function withReleasedBaseline({ baseline, afterTag = baseline, previous = "1.0.0" }, fn) {
|
||||
withTempGitRepo((dir) => {
|
||||
mkdirSync(join(dir, PLUGINS), { recursive: true });
|
||||
for (const [name, path] of Object.entries(baseline)) copyFileSync(path, join(dir, PLUGINS, name));
|
||||
execFileSync("git", ["add", "."], { cwd: dir });
|
||||
execFileSync("git", ["commit", "-q", "-m", "release"], { cwd: dir });
|
||||
execFileSync("git", ["tag", `widget-v${previous}`], { cwd: dir });
|
||||
if (afterTag !== baseline) {
|
||||
for (const name of Object.keys(baseline)) rmSync(join(dir, PLUGINS, name));
|
||||
for (const [name, path] of Object.entries(afterTag)) copyFileSync(path, join(dir, PLUGINS, name));
|
||||
execFileSync("git", ["add", "-A", "."], { cwd: dir });
|
||||
execFileSync("git", ["commit", "-q", "-m", "after the release"], { cwd: dir });
|
||||
}
|
||||
fn(dir);
|
||||
});
|
||||
}
|
||||
|
||||
function measuredArgs(dir, version, builtPaths, { breaking = false } = {}) {
|
||||
const changelogPath = join(dir, "CHANGELOG.md");
|
||||
writeFileSync(changelogPath, changelogFor(version, { breaking }));
|
||||
return [
|
||||
"--package", "widget", "--changelog", changelogPath, "--version", version, "--tag-prefix", "widget",
|
||||
"--api-baseline-path", PLUGINS, ...builtPaths.flatMap((path) => ["--api-built", path]),
|
||||
];
|
||||
}
|
||||
|
||||
const WITH_GONE = "namespace N { public class C { public void Keep() {} public void Gone() {} } }";
|
||||
const WITHOUT_GONE = "namespace N { public class C { public void Keep() {} } }";
|
||||
const WITH_ADDED = "namespace N { public class C { public void Keep() {} public void Gone() {} public void Added() {} } }";
|
||||
|
||||
const oldWidget = () => buildAssembly("GameCult.Widget", WITH_GONE);
|
||||
const trimmedWidget = () => buildAssembly("GameCult.Widget", WITHOUT_GONE);
|
||||
|
||||
test("measured: a removed public member without ### Breaking is refused and named", () => {
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": oldWidget() } }, (dir) => {
|
||||
const output = runCheckerExpectFailure(dir, measuredArgs(dir, "1.1.0", [trimmedWidget()]));
|
||||
assert.match(output, /Gone/);
|
||||
assert.match(output, /declare them under "### Breaking"/);
|
||||
assert.match(output, /breaks 1 public API member/);
|
||||
});
|
||||
});
|
||||
|
||||
test("measured: a removed member with the heading needs a major bump after 1.0 and a minor bump before it", () => {
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": oldWidget() } }, (dir) => {
|
||||
const tooSmall = runCheckerExpectFailure(dir, measuredArgs(dir, "1.1.0", [trimmedWidget()], { breaking: true }));
|
||||
assert.match(tooSmall, /needs a major bump/);
|
||||
const ok = runChecker(dir, measuredArgs(dir, "2.0.0", [trimmedWidget()], { breaking: true }));
|
||||
assert.match(ok, /2\.0\.0 is a major bump/);
|
||||
});
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": oldWidget() }, previous: "0.4.0" }, (dir) => {
|
||||
const patch = runCheckerExpectFailure(dir, measuredArgs(dir, "0.4.1", [trimmedWidget()], { breaking: true }));
|
||||
assert.match(patch, /at least a minor bump/);
|
||||
const ok = runChecker(dir, measuredArgs(dir, "0.5.0", [trimmedWidget()], { breaking: true }));
|
||||
assert.match(ok, /0\.5\.0 is a minor bump/);
|
||||
});
|
||||
});
|
||||
|
||||
test("measured: additions alone pass under a patch bump with no heading", () => {
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": oldWidget() } }, (dir) => {
|
||||
const output = runChecker(dir, measuredArgs(dir, "1.0.1", [buildAssembly("GameCult.Widget", WITH_ADDED)]));
|
||||
assert.match(output, /1\.0\.1 is a patch bump over 1\.0\.0, no breaking section/);
|
||||
});
|
||||
});
|
||||
|
||||
test("measured: a declared break with nothing measured still passes", () => {
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": oldWidget() } }, (dir) => {
|
||||
const output = runChecker(dir, measuredArgs(dir, "2.0.0", [oldWidget()], { breaking: true }));
|
||||
assert.match(output, /2\.0\.0 is a major bump over 1\.0\.0 and has a "### Breaking" section, which agree/);
|
||||
});
|
||||
});
|
||||
|
||||
test("measured: a removed GameCult.* assembly is a break, a new assembly and a removed third-party one are not", () => {
|
||||
const other = buildAssembly("GameCult.Other", "namespace O { public class D {} }");
|
||||
const thirdParty = buildAssembly("Vendor.Lib", "namespace V { public class E {} }");
|
||||
const baseline = { "GameCult.Widget.dll": oldWidget(), "GameCult.Other.dll": other, "Vendor.Lib.dll": thirdParty };
|
||||
withReleasedBaseline({ baseline }, (dir) => {
|
||||
const refused = runCheckerExpectFailure(dir, measuredArgs(dir, "1.1.0", [oldWidget()]));
|
||||
assert.match(refused, /GameCult\.Other\.dll exists at widget-v1\.0\.0 but is no longer built/);
|
||||
assert.doesNotMatch(refused, /Vendor\.Lib/);
|
||||
assert.match(refused, /breaks 1 public API member/);
|
||||
const brandNew = buildAssembly("GameCult.Brand", "namespace B { public class F {} }");
|
||||
const ok = runChecker(dir, measuredArgs(dir, "1.0.1", [oldWidget(), other, brandNew]));
|
||||
assert.match(ok, /1\.0\.1 is a patch bump/);
|
||||
});
|
||||
});
|
||||
|
||||
test("measured: a tool failure with no diagnostic fails closed", () => {
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": oldWidget() } }, (dir) => {
|
||||
const notAnAssembly = join(dir, "GameCult.Widget.dll");
|
||||
writeFileSync(notAnAssembly, "not an assembly");
|
||||
const output = runCheckerExpectFailure(dir, measuredArgs(dir, "1.0.1", [notAnAssembly]));
|
||||
assert.match(output, /could not be measured/);
|
||||
assert.match(output, /without measuring/);
|
||||
});
|
||||
});
|
||||
|
||||
test("readApiCompatRun: diagnostics are breaks, an empty failure is a failure, an empty success is clean", () => {
|
||||
const breaks = readApiCompatRun({ status: 1, output: "API compatibility errors between a (left) and b (right):\nCP0002: Member 'void C.M()' exists on a but not on b\n" });
|
||||
assert.deepEqual(breaks, { breaks: ["CP0002: Member 'void C.M()' exists on a but not on b"] });
|
||||
assert.match(readApiCompatRun({ status: 1, output: "Unhandled exception: boom" }).failure, /exited 1 without measuring: Unhandled exception: boom/);
|
||||
assert.deepEqual(readApiCompatRun({ status: 0, output: "" }), { breaks: [] });
|
||||
assert.deepEqual(readApiCompatRun({ status: 0, output: "Could not resolve reference 'netstandard.dll'\n" }), { breaks: [] });
|
||||
});
|
||||
|
||||
test("measured: removed inherited members are caught although netstandard cannot be resolved", () => {
|
||||
// The shipped assemblies are netstandard2.1 and ApiCompat cannot resolve netstandard.dll
|
||||
// for them; both ways an inherited member can disappear are still reported.
|
||||
const viaNetstandardBase = buildAssembly("GameCult.Widget", "namespace N { public class C : System.Collections.Generic.List<int> { } }");
|
||||
const baseDropped = buildAssembly("GameCult.Widget", "namespace N { public class C { } }");
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": viaNetstandardBase } }, (dir) => {
|
||||
const output = runCheckerExpectFailure(dir, measuredArgs(dir, "1.1.0", [baseDropped]));
|
||||
assert.match(output, /does not inherit from base type 'System.Collections.Generic.List<int>'/);
|
||||
});
|
||||
const inheritsInh = buildAssembly("GameCult.Widget", "namespace N { public class B { public void Inh() {} } public class C : B { } }");
|
||||
const baseLostInh = buildAssembly("GameCult.Widget", "namespace N { public class B { } public class C : B { } }");
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": inheritsInh } }, (dir) => {
|
||||
const output = runCheckerExpectFailure(dir, measuredArgs(dir, "1.1.0", [baseLostInh]));
|
||||
assert.match(output, /N.B.Inh()/);
|
||||
});
|
||||
});
|
||||
|
||||
test("measured: an already-published version is not measured", () => {
|
||||
withReleasedBaseline({ baseline: { "GameCult.Widget.dll": oldWidget() } }, (dir) => {
|
||||
const junk = join(dir, "junk.dll");
|
||||
writeFileSync(junk, "junk");
|
||||
const output = runChecker(dir, measuredArgs(dir, "1.0.0", [junk], { breaking: true }));
|
||||
assert.match(output, /already published/);
|
||||
});
|
||||
});
|
||||
|
||||
test("measured: the baseline is the tag's blob, not the working tree's DLL", () => {
|
||||
withReleasedBaseline(
|
||||
{ baseline: { "GameCult.Widget.dll": oldWidget() }, afterTag: { "GameCult.Widget.dll": trimmedWidget() } },
|
||||
(dir) => {
|
||||
const output = runCheckerExpectFailure(dir, measuredArgs(dir, "1.1.0", [trimmedWidget()]));
|
||||
assert.match(output, /Gone/);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("measured: a first release measures nothing", () => {
|
||||
withTempGitRepo((dir) => {
|
||||
execFileSync("git", ["commit", "--allow-empty", "-q", "-m", "init"], { cwd: dir });
|
||||
const junk = join(dir, "junk.dll");
|
||||
writeFileSync(junk, "junk");
|
||||
const output = runChecker(dir, measuredArgs(dir, "1.0.0", [junk]));
|
||||
assert.match(output, /first release/);
|
||||
});
|
||||
});
|
||||
|
||||
test("evaluateRelease: measured breaks count as breaking for the lane, declared or not", () => {
|
||||
const evaluate = (version, previousVersion, breaking, measuredBreaks) =>
|
||||
evaluateRelease({ packageName: "p", changelogText: changelogFor(version, { breaking }), version, previousVersion, measuredBreaks });
|
||||
assert.equal(evaluate("1.1.0", "1.0.0", false, ["CP0002: x"]).ok, false);
|
||||
assert.equal(evaluate("1.1.0", "1.0.0", true, ["CP0002: x"]).ok, false);
|
||||
assert.equal(evaluate("2.0.0", "1.0.0", true, ["CP0002: x"]).ok, true);
|
||||
assert.equal(evaluate("0.2.0", "0.1.0", true, ["CP0002: x"]).ok, true);
|
||||
assert.equal(evaluate("0.1.1", "0.1.0", true, ["CP0002: x"]).ok, false);
|
||||
assert.equal(evaluate("1.0.1", "1.0.0", false, []).ok, true);
|
||||
const many = Array.from({ length: 7 }, (_, i) => `CP0002: m${i}`);
|
||||
const reason = evaluate("1.1.0", "1.0.0", false, many).reason;
|
||||
assert.match(reason, /breaks 7 public API/);
|
||||
assert.match(reason, /m4/);
|
||||
assert.doesNotMatch(reason, /m5/);
|
||||
assert.match(reason, /; \.\.\./);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue