Gate Python C# parity claims

This commit is contained in:
Meta Vens 2026-06-14 13:56:56 +01:00
commit a3ccaa0290
3 changed files with 80 additions and 9 deletions

View file

@ -169,7 +169,8 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti.
- Python/C# public cache baseline:
`cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get
benchmarks with the same operation names and reports median Python-to-C# ratios
from three samples by default.
from three samples by default. It also emits per-metric threshold checks and an
overall parity status so performance claims have an explicit evidence gate.
## Local Python Gates
@ -194,7 +195,7 @@ node --test packages\cultnet-ts\dist-test\test\interop\cultnet-interop.test.js
- Python does not claim to match the C# runtime for raw throughput. It has a
measured local baseline plus a shared public `CultCache` upsert/get comparison
harness, not a performance victory certificate.
harness with threshold status, not a performance victory certificate.
- Python does not own LiteNetLib security/session behavior; it speaks the
schema-v0 interop lane used by the package harness.
- Python does not implement the full C# daemon/server stack. Its current role is

View file

@ -12,12 +12,14 @@ from .benchmark import run_benchmark
DEFAULT_SAMPLE_COUNT = 3
DEFAULT_PARITY_THRESHOLD = 0.90
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp")
parser.add_argument("--records", type=int, default=5000)
parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLE_COUNT)
parser.add_argument("--parity-threshold", type=float, default=DEFAULT_PARITY_THRESHOLD)
parser.add_argument("--repo-root", type=Path, default=None)
parser.add_argument("--json", action="store_true", dest="emit_json")
args = parser.parse_args(argv)
@ -25,8 +27,15 @@ def main(argv: list[str] | None = None) -> int:
raise ValueError("--records must be greater than zero")
if args.samples <= 0:
raise ValueError("--samples must be greater than zero")
if args.parity_threshold <= 0:
raise ValueError("--parity-threshold must be greater than zero")
result = compare_with_csharp(records=args.records, samples=args.samples, repo_root=args.repo_root)
result = compare_with_csharp(
records=args.records,
samples=args.samples,
repo_root=args.repo_root,
parity_threshold=args.parity_threshold,
)
if args.emit_json:
print(json.dumps(result, indent=2, sort_keys=True))
else:
@ -39,10 +48,18 @@ def main(argv: list[str] | None = None) -> int:
print(runtime)
for metric in benchmark["metrics"]:
print(f"- {metric['name']}: {metric['opsPerSecond']:.0f} ops/s ({metric['elapsedMs']:.2f} ms)")
print(f"parityThreshold: {result['parityThreshold']:.2f}")
print(f"parityStatus: {result['parityStatus']}")
return 0 if result["csharpStatus"] == "ok" else 1
def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, repo_root: Path | None = None) -> dict[str, Any]:
def compare_with_csharp(
*,
records: int,
samples: int = DEFAULT_SAMPLE_COUNT,
repo_root: Path | None = None,
parity_threshold: float = DEFAULT_PARITY_THRESHOLD,
) -> dict[str, Any]:
resolved_root = _resolve_repo_root(repo_root)
python_samples = [run_benchmark(records) for _ in range(samples)]
python_result = _median_result(python_samples)
@ -55,7 +72,11 @@ def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, re
break
csharp_samples.append(csharp_result)
csharp_result = _median_result(csharp_samples) if csharp_samples else None
comparison = _compare_common_metrics(python_result, csharp_result) if csharp_result else []
comparison = _compare_common_metrics(
python_result,
csharp_result,
parity_threshold=parity_threshold,
) if csharp_result else []
result: dict[str, Any] = {
"records": records,
"samples": samples,
@ -64,6 +85,8 @@ def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, re
"pythonSamples": python_samples,
"csharpStatus": status,
"comparison": comparison,
"parityThreshold": parity_threshold,
"parityStatus": _parity_status(comparison, status),
}
if csharp_result is not None:
result["csharp"] = csharp_result
@ -107,22 +130,39 @@ def _run_csharp_benchmark(repo_root: Path, records: int) -> tuple[dict[str, Any]
return None, "invalid-json", "C# benchmark produced no stdout."
def _compare_common_metrics(python_result: dict[str, Any], csharp_result: dict[str, Any]) -> list[dict[str, Any]]:
def _compare_common_metrics(
python_result: dict[str, Any],
csharp_result: dict[str, Any],
*,
parity_threshold: float = DEFAULT_PARITY_THRESHOLD,
) -> list[dict[str, Any]]:
python_metrics = {metric["name"]: metric for metric in python_result["metrics"]}
csharp_metrics = {metric["name"]: metric for metric in csharp_result["metrics"]}
comparison = []
for name in sorted(set(python_metrics).intersection(csharp_metrics)):
python_ops = float(python_metrics[name]["opsPerSecond"])
csharp_ops = float(csharp_metrics[name]["opsPerSecond"])
ratio = python_ops / csharp_ops if csharp_ops > 0 else None
comparison.append({
"name": name,
"pythonOpsPerSecond": python_ops,
"csharpOpsPerSecond": csharp_ops,
"pythonToCsharpRatio": python_ops / csharp_ops if csharp_ops > 0 else None,
"pythonToCsharpRatio": ratio,
"meetsParityThreshold": ratio is not None and ratio >= parity_threshold,
})
return comparison
def _parity_status(comparison: list[dict[str, Any]], csharp_status: str) -> str:
if csharp_status != "ok":
return "unknown"
if not comparison:
return "unknown"
if all(item.get("meetsParityThreshold") is True for item in comparison):
return "meets-threshold"
return "below-threshold"
def _median_result(samples: list[dict[str, Any]]) -> dict[str, Any]:
if not samples:
raise ValueError("Cannot summarize zero benchmark samples")

View file

@ -20,7 +20,13 @@ from cultcache_py import (
define_document_type,
)
from cultcache_py.benchmark import run_benchmark
from cultcache_py.compare_csharp import DEFAULT_SAMPLE_COUNT, _median_result
from cultcache_py.compare_csharp import (
DEFAULT_PARITY_THRESHOLD,
DEFAULT_SAMPLE_COUNT,
_compare_common_metrics,
_median_result,
_parity_status,
)
from cultcache_py.interop import read_note, write_note
from cultcache_py.verify import verify
from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response
@ -309,6 +315,7 @@ class CultCacheTests(unittest.TestCase):
def test_compare_csharp_median_result_summarizes_samples(self) -> None:
self.assertEqual(DEFAULT_SAMPLE_COUNT, 3)
self.assertEqual(DEFAULT_PARITY_THRESHOLD, 0.90)
samples = [
{
"runtime": "python",
@ -344,6 +351,29 @@ class CultCacheTests(unittest.TestCase):
self.assertEqual(summarized["metrics"][0]["opsPerSecond"], 200.0)
self.assertEqual(summarized["metrics"][1]["opsPerSecond"], 75.0)
def test_compare_csharp_marks_threshold_status_per_metric(self) -> None:
python_result = {
"metrics": [
{"name": "cache_get", "operations": 3, "elapsedMs": 2.0, "opsPerSecond": 50.0},
{"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 95.0},
],
}
csharp_result = {
"metrics": [
{"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0},
{"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0},
],
}
comparison = _compare_common_metrics(python_result, csharp_result, parity_threshold=0.90)
by_name = {item["name"]: item for item in comparison}
self.assertFalse(by_name["cache_get"]["meetsParityThreshold"])
self.assertTrue(by_name["cache_upsert"]["meetsParityThreshold"])
self.assertEqual(_parity_status(comparison, "ok"), "below-threshold")
self.assertEqual(_parity_status([by_name["cache_upsert"]], "ok"), "meets-threshold")
self.assertEqual(_parity_status(comparison, "failed"), "unknown")
def test_cache_put_without_backing_store_keeps_in_memory_value(self) -> None:
document = define_database_entry_type(
"bench.memory_item",