From a3ccaa0290098ddebe583129ad9cfc875a2d51fa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Sun, 14 Jun 2026 13:56:56 +0100 Subject: [PATCH] Gate Python C# parity claims --- packages/cultcache-py/PARITY.md | 5 +- .../src/cultcache_py/compare_csharp.py | 50 +++++++++++++++++-- packages/cultcache-py/tests/test_cultcache.py | 32 +++++++++++- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/packages/cultcache-py/PARITY.md b/packages/cultcache-py/PARITY.md index 868500b2..75162801 100644 --- a/packages/cultcache-py/PARITY.md +++ b/packages/cultcache-py/PARITY.md @@ -169,7 +169,8 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti. - Python/C# public cache baseline: `cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get benchmarks with the same operation names and reports median Python-to-C# ratios - from three samples by default. + from three samples by default. It also emits per-metric threshold checks and an + overall parity status so performance claims have an explicit evidence gate. ## Local Python Gates @@ -194,7 +195,7 @@ node --test packages\cultnet-ts\dist-test\test\interop\cultnet-interop.test.js - Python does not claim to match the C# runtime for raw throughput. It has a measured local baseline plus a shared public `CultCache` upsert/get comparison - harness, not a performance victory certificate. + harness with threshold status, not a performance victory certificate. - Python does not own LiteNetLib security/session behavior; it speaks the schema-v0 interop lane used by the package harness. - Python does not implement the full C# daemon/server stack. Its current role is diff --git a/packages/cultcache-py/src/cultcache_py/compare_csharp.py b/packages/cultcache-py/src/cultcache_py/compare_csharp.py index de1e9b01..8a9e4c5b 100644 --- a/packages/cultcache-py/src/cultcache_py/compare_csharp.py +++ b/packages/cultcache-py/src/cultcache_py/compare_csharp.py @@ -12,12 +12,14 @@ from .benchmark import run_benchmark DEFAULT_SAMPLE_COUNT = 3 +DEFAULT_PARITY_THRESHOLD = 0.90 def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp") parser.add_argument("--records", type=int, default=5000) parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLE_COUNT) + parser.add_argument("--parity-threshold", type=float, default=DEFAULT_PARITY_THRESHOLD) parser.add_argument("--repo-root", type=Path, default=None) parser.add_argument("--json", action="store_true", dest="emit_json") args = parser.parse_args(argv) @@ -25,8 +27,15 @@ def main(argv: list[str] | None = None) -> int: raise ValueError("--records must be greater than zero") if args.samples <= 0: raise ValueError("--samples must be greater than zero") + if args.parity_threshold <= 0: + raise ValueError("--parity-threshold must be greater than zero") - result = compare_with_csharp(records=args.records, samples=args.samples, repo_root=args.repo_root) + result = compare_with_csharp( + records=args.records, + samples=args.samples, + repo_root=args.repo_root, + parity_threshold=args.parity_threshold, + ) if args.emit_json: print(json.dumps(result, indent=2, sort_keys=True)) else: @@ -39,10 +48,18 @@ def main(argv: list[str] | None = None) -> int: print(runtime) for metric in benchmark["metrics"]: print(f"- {metric['name']}: {metric['opsPerSecond']:.0f} ops/s ({metric['elapsedMs']:.2f} ms)") + print(f"parityThreshold: {result['parityThreshold']:.2f}") + print(f"parityStatus: {result['parityStatus']}") return 0 if result["csharpStatus"] == "ok" else 1 -def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, repo_root: Path | None = None) -> dict[str, Any]: +def compare_with_csharp( + *, + records: int, + samples: int = DEFAULT_SAMPLE_COUNT, + repo_root: Path | None = None, + parity_threshold: float = DEFAULT_PARITY_THRESHOLD, +) -> dict[str, Any]: resolved_root = _resolve_repo_root(repo_root) python_samples = [run_benchmark(records) for _ in range(samples)] python_result = _median_result(python_samples) @@ -55,7 +72,11 @@ def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, re break csharp_samples.append(csharp_result) csharp_result = _median_result(csharp_samples) if csharp_samples else None - comparison = _compare_common_metrics(python_result, csharp_result) if csharp_result else [] + comparison = _compare_common_metrics( + python_result, + csharp_result, + parity_threshold=parity_threshold, + ) if csharp_result else [] result: dict[str, Any] = { "records": records, "samples": samples, @@ -64,6 +85,8 @@ def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, re "pythonSamples": python_samples, "csharpStatus": status, "comparison": comparison, + "parityThreshold": parity_threshold, + "parityStatus": _parity_status(comparison, status), } if csharp_result is not None: result["csharp"] = csharp_result @@ -107,22 +130,39 @@ def _run_csharp_benchmark(repo_root: Path, records: int) -> tuple[dict[str, Any] return None, "invalid-json", "C# benchmark produced no stdout." -def _compare_common_metrics(python_result: dict[str, Any], csharp_result: dict[str, Any]) -> list[dict[str, Any]]: +def _compare_common_metrics( + python_result: dict[str, Any], + csharp_result: dict[str, Any], + *, + parity_threshold: float = DEFAULT_PARITY_THRESHOLD, +) -> list[dict[str, Any]]: python_metrics = {metric["name"]: metric for metric in python_result["metrics"]} csharp_metrics = {metric["name"]: metric for metric in csharp_result["metrics"]} comparison = [] for name in sorted(set(python_metrics).intersection(csharp_metrics)): python_ops = float(python_metrics[name]["opsPerSecond"]) csharp_ops = float(csharp_metrics[name]["opsPerSecond"]) + ratio = python_ops / csharp_ops if csharp_ops > 0 else None comparison.append({ "name": name, "pythonOpsPerSecond": python_ops, "csharpOpsPerSecond": csharp_ops, - "pythonToCsharpRatio": python_ops / csharp_ops if csharp_ops > 0 else None, + "pythonToCsharpRatio": ratio, + "meetsParityThreshold": ratio is not None and ratio >= parity_threshold, }) return comparison +def _parity_status(comparison: list[dict[str, Any]], csharp_status: str) -> str: + if csharp_status != "ok": + return "unknown" + if not comparison: + return "unknown" + if all(item.get("meetsParityThreshold") is True for item in comparison): + return "meets-threshold" + return "below-threshold" + + def _median_result(samples: list[dict[str, Any]]) -> dict[str, Any]: if not samples: raise ValueError("Cannot summarize zero benchmark samples") diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 1975bc95..9ade7e9c 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -20,7 +20,13 @@ from cultcache_py import ( define_document_type, ) from cultcache_py.benchmark import run_benchmark -from cultcache_py.compare_csharp import DEFAULT_SAMPLE_COUNT, _median_result +from cultcache_py.compare_csharp import ( + DEFAULT_PARITY_THRESHOLD, + DEFAULT_SAMPLE_COUNT, + _compare_common_metrics, + _median_result, + _parity_status, +) from cultcache_py.interop import read_note, write_note from cultcache_py.verify import verify from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response @@ -309,6 +315,7 @@ class CultCacheTests(unittest.TestCase): def test_compare_csharp_median_result_summarizes_samples(self) -> None: self.assertEqual(DEFAULT_SAMPLE_COUNT, 3) + self.assertEqual(DEFAULT_PARITY_THRESHOLD, 0.90) samples = [ { "runtime": "python", @@ -344,6 +351,29 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(summarized["metrics"][0]["opsPerSecond"], 200.0) self.assertEqual(summarized["metrics"][1]["opsPerSecond"], 75.0) + def test_compare_csharp_marks_threshold_status_per_metric(self) -> None: + python_result = { + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 2.0, "opsPerSecond": 50.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 95.0}, + ], + } + csharp_result = { + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0}, + ], + } + + comparison = _compare_common_metrics(python_result, csharp_result, parity_threshold=0.90) + by_name = {item["name"]: item for item in comparison} + + self.assertFalse(by_name["cache_get"]["meetsParityThreshold"]) + self.assertTrue(by_name["cache_upsert"]["meetsParityThreshold"]) + self.assertEqual(_parity_status(comparison, "ok"), "below-threshold") + self.assertEqual(_parity_status([by_name["cache_upsert"]], "ok"), "meets-threshold") + self.assertEqual(_parity_status(comparison, "failed"), "unknown") + def test_cache_put_without_backing_store_keeps_in_memory_value(self) -> None: document = define_database_entry_type( "bench.memory_item",