diff --git a/packages/cultcache-py/PARITY.md b/packages/cultcache-py/PARITY.md index d135e5ef..31d3dd38 100644 --- a/packages/cultcache-py/PARITY.md +++ b/packages/cultcache-py/PARITY.md @@ -52,7 +52,9 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti. benchmark sanity for Python-owned hot paths. - Python/C# public cache baseline: `cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get - benchmarks with the same operation names and reports Python-to-C# ratios. + benchmarks with the same operation names and reports Python-to-C# ratios. Use + `--samples` when a performance claim needs median evidence instead of a single + noisy run. ## Local Python Gates @@ -61,7 +63,7 @@ $env:PYTHONPATH="$PWD\packages\cultcache-py\src" python -m unittest discover -s packages\cultcache-py\tests python -m cultcache_py.verify --json python -m cultcache_py.benchmark --records 1000 --json -python -m cultcache_py.compare_csharp --records 1000 --json +python -m cultcache_py.compare_csharp --records 1000 --samples 3 --json python -m pip wheel --no-deps -w $env:TEMP packages\cultcache-py ``` diff --git a/packages/cultcache-py/src/cultcache_py/compare_csharp.py b/packages/cultcache-py/src/cultcache_py/compare_csharp.py index 59dd09b6..ab9a5671 100644 --- a/packages/cultcache-py/src/cultcache_py/compare_csharp.py +++ b/packages/cultcache-py/src/cultcache_py/compare_csharp.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse import json +import statistics import subprocess import sys from pathlib import Path @@ -13,13 +14,16 @@ from .benchmark import run_benchmark def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp") parser.add_argument("--records", type=int, default=5000) + parser.add_argument("--samples", type=int, default=1) parser.add_argument("--repo-root", type=Path, default=None) parser.add_argument("--json", action="store_true", dest="emit_json") args = parser.parse_args(argv) if args.records <= 0: raise ValueError("--records must be greater than zero") + if args.samples <= 0: + raise ValueError("--samples must be greater than zero") - result = compare_with_csharp(records=args.records, repo_root=args.repo_root) + result = compare_with_csharp(records=args.records, samples=args.samples, repo_root=args.repo_root) if args.emit_json: print(json.dumps(result, indent=2, sort_keys=True)) else: @@ -35,20 +39,32 @@ def main(argv: list[str] | None = None) -> int: return 0 if result["csharpStatus"] == "ok" else 1 -def compare_with_csharp(*, records: int, repo_root: Path | None = None) -> dict[str, Any]: +def compare_with_csharp(*, records: int, samples: int = 1, repo_root: Path | None = None) -> dict[str, Any]: resolved_root = _resolve_repo_root(repo_root) - python_result = run_benchmark(records) - csharp_result, status, error = _run_csharp_benchmark(resolved_root, records) + python_samples = [run_benchmark(records) for _ in range(samples)] + python_result = _median_result(python_samples) + csharp_samples: list[dict[str, Any]] = [] + status = "ok" + error = None + for _ in range(samples): + csharp_result, status, error = _run_csharp_benchmark(resolved_root, records) + if csharp_result is None: + break + csharp_samples.append(csharp_result) + csharp_result = _median_result(csharp_samples) if csharp_samples else None comparison = _compare_common_metrics(python_result, csharp_result) if csharp_result else [] result: dict[str, Any] = { "records": records, + "samples": samples, "repoRoot": str(resolved_root), "python": python_result, + "pythonSamples": python_samples, "csharpStatus": status, "comparison": comparison, } if csharp_result is not None: result["csharp"] = csharp_result + result["csharpSamples"] = csharp_samples if error: result["csharpError"] = error return result @@ -104,5 +120,33 @@ def _compare_common_metrics(python_result: dict[str, Any], csharp_result: dict[s return comparison +def _median_result(samples: list[dict[str, Any]]) -> dict[str, Any]: + if not samples: + raise ValueError("Cannot summarize zero benchmark samples") + if len(samples) == 1: + return samples[0] + metric_names = [metric["name"] for metric in samples[0]["metrics"]] + metrics = [] + for name in metric_names: + matching = [_metric_by_name(sample, name) for sample in samples] + metrics.append({ + "name": name, + "operations": int(statistics.median(metric["operations"] for metric in matching)), + "elapsedMs": statistics.median(float(metric["elapsedMs"]) for metric in matching), + "opsPerSecond": statistics.median(float(metric["opsPerSecond"]) for metric in matching), + }) + result = dict(samples[0]) + result["metrics"] = metrics + result["sampleCount"] = len(samples) + return result + + +def _metric_by_name(sample: dict[str, Any], name: str) -> dict[str, Any]: + for metric in sample["metrics"]: + if metric["name"] == name: + return metric + raise KeyError(f"Benchmark sample is missing metric {name!r}") + + if __name__ == "__main__": raise SystemExit(main(sys.argv[1:])) diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 98c85d51..dfa904d1 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -17,6 +17,7 @@ from cultcache_py import ( define_document_type, ) from cultcache_py.benchmark import run_benchmark +from cultcache_py.compare_csharp import _median_result from cultcache_py.interop import read_note, write_note from cultcache_py.verify import verify from cultnet_py import ( @@ -263,6 +264,42 @@ class CultCacheTests(unittest.TestCase): }) self.assertTrue(all(metric["opsPerSecond"] > 0 for metric in result["metrics"])) + def test_compare_csharp_median_result_summarizes_samples(self) -> None: + samples = [ + { + "runtime": "python", + "records": 3, + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 30.0, "opsPerSecond": 100.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 60.0, "opsPerSecond": 50.0}, + ], + }, + { + "runtime": "python", + "records": 3, + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 10.0, "opsPerSecond": 300.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 20.0, "opsPerSecond": 150.0}, + ], + }, + { + "runtime": "python", + "records": 3, + "metrics": [ + {"name": "cache_get", "operations": 3, "elapsedMs": 20.0, "opsPerSecond": 200.0}, + {"name": "cache_upsert", "operations": 3, "elapsedMs": 40.0, "opsPerSecond": 75.0}, + ], + }, + ] + + summarized = _median_result(samples) + + self.assertEqual(summarized["sampleCount"], 3) + self.assertEqual(summarized["metrics"][0]["name"], "cache_get") + self.assertEqual(summarized["metrics"][0]["elapsedMs"], 20.0) + self.assertEqual(summarized["metrics"][0]["opsPerSecond"], 200.0) + self.assertEqual(summarized["metrics"][1]["opsPerSecond"], 75.0) + def test_cache_put_without_backing_store_keeps_in_memory_value(self) -> None: document = define_database_entry_type( "bench.memory_item",