diff --git a/packages/cultcache-py/PARITY.md b/packages/cultcache-py/PARITY.md index 71c65461..0a4d2cef 100644 --- a/packages/cultcache-py/PARITY.md +++ b/packages/cultcache-py/PARITY.md @@ -127,9 +127,8 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti. Python-owned hot paths. - Python/C# public cache baseline: `cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get - benchmarks with the same operation names and reports Python-to-C# ratios. Use - `--samples` when a performance claim needs median evidence instead of a single - noisy run. + benchmarks with the same operation names and reports median Python-to-C# ratios + from three samples by default. ## Local Python Gates diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index f40f5dcf..50342081 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -350,6 +350,7 @@ It reports slot-indexed `DatabaseEntry` encode/decode throughput, framed CultNet MessagePack parse throughput, raw snapshot application into a registered cache, and public `CultCache` upsert/get throughput. The C# compare command also runs `packages/cultcache-py/tools/GameCult.Caching.Benchmark` and -reports Python-to-C# ratios for the shared public cache operations. Treat those -numbers as a local baseline, not a claim that Python matches the C# reference in -every workload. +reports median Python-to-C# ratios for the shared public cache operations. It +uses three samples by default; pass `--samples` when you need a different +evidence shape. Treat those numbers as a local baseline, not a claim that +Python matches the C# reference in every workload. diff --git a/packages/cultcache-py/src/cultcache_py/compare_csharp.py b/packages/cultcache-py/src/cultcache_py/compare_csharp.py index ab9a5671..de1e9b01 100644 --- a/packages/cultcache-py/src/cultcache_py/compare_csharp.py +++ b/packages/cultcache-py/src/cultcache_py/compare_csharp.py @@ -11,10 +11,13 @@ from typing import Any from .benchmark import run_benchmark +DEFAULT_SAMPLE_COUNT = 3 + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp") parser.add_argument("--records", type=int, default=5000) - parser.add_argument("--samples", type=int, default=1) + parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLE_COUNT) parser.add_argument("--repo-root", type=Path, default=None) parser.add_argument("--json", action="store_true", dest="emit_json") args = parser.parse_args(argv) @@ -39,7 +42,7 @@ def main(argv: list[str] | None = None) -> int: return 0 if result["csharpStatus"] == "ok" else 1 -def compare_with_csharp(*, records: int, samples: int = 1, repo_root: Path | None = None) -> dict[str, Any]: +def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, repo_root: Path | None = None) -> dict[str, Any]: resolved_root = _resolve_repo_root(repo_root) python_samples = [run_benchmark(records) for _ in range(samples)] python_result = _median_result(python_samples) diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 9628cc44..774ad0ce 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -18,7 +18,7 @@ from cultcache_py import ( define_document_type, ) from cultcache_py.benchmark import run_benchmark -from cultcache_py.compare_csharp import _median_result +from cultcache_py.compare_csharp import DEFAULT_SAMPLE_COUNT, _median_result from cultcache_py.interop import read_note, write_note from cultcache_py.verify import verify from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response @@ -282,6 +282,7 @@ class CultCacheTests(unittest.TestCase): self.assertTrue(all(metric["opsPerSecond"] > 0 for metric in result["metrics"])) def test_compare_csharp_median_result_summarizes_samples(self) -> None: + self.assertEqual(DEFAULT_SAMPLE_COUNT, 3) samples = [ { "runtime": "python",