Use median samples for Python C# comparison

This commit is contained in:
Meta Vens 2026-06-14 05:53:46 +01:00
commit 0827713076
4 changed files with 13 additions and 9 deletions

View file

@ -127,9 +127,8 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti.
Python-owned hot paths.
- Python/C# public cache baseline:
`cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get
benchmarks with the same operation names and reports Python-to-C# ratios. Use
`--samples` when a performance claim needs median evidence instead of a single
noisy run.
benchmarks with the same operation names and reports median Python-to-C# ratios
from three samples by default.
## Local Python Gates

View file

@ -350,6 +350,7 @@ It reports slot-indexed `DatabaseEntry` encode/decode throughput, framed
CultNet MessagePack parse throughput, raw snapshot application into a
registered cache, and public `CultCache` upsert/get throughput. The C# compare
command also runs `packages/cultcache-py/tools/GameCult.Caching.Benchmark` and
reports Python-to-C# ratios for the shared public cache operations. Treat those
numbers as a local baseline, not a claim that Python matches the C# reference in
every workload.
reports median Python-to-C# ratios for the shared public cache operations. It
uses three samples by default; pass `--samples` when you need a different
evidence shape. Treat those numbers as a local baseline, not a claim that
Python matches the C# reference in every workload.

View file

@ -11,10 +11,13 @@ from typing import Any
from .benchmark import run_benchmark
DEFAULT_SAMPLE_COUNT = 3
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp")
parser.add_argument("--records", type=int, default=5000)
parser.add_argument("--samples", type=int, default=1)
parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLE_COUNT)
parser.add_argument("--repo-root", type=Path, default=None)
parser.add_argument("--json", action="store_true", dest="emit_json")
args = parser.parse_args(argv)
@ -39,7 +42,7 @@ def main(argv: list[str] | None = None) -> int:
return 0 if result["csharpStatus"] == "ok" else 1
def compare_with_csharp(*, records: int, samples: int = 1, repo_root: Path | None = None) -> dict[str, Any]:
def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, repo_root: Path | None = None) -> dict[str, Any]:
resolved_root = _resolve_repo_root(repo_root)
python_samples = [run_benchmark(records) for _ in range(samples)]
python_result = _median_result(python_samples)

View file

@ -18,7 +18,7 @@ from cultcache_py import (
define_document_type,
)
from cultcache_py.benchmark import run_benchmark
from cultcache_py.compare_csharp import _median_result
from cultcache_py.compare_csharp import DEFAULT_SAMPLE_COUNT, _median_result
from cultcache_py.interop import read_note, write_note
from cultcache_py.verify import verify
from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response
@ -282,6 +282,7 @@ class CultCacheTests(unittest.TestCase):
self.assertTrue(all(metric["opsPerSecond"] > 0 for metric in result["metrics"]))
def test_compare_csharp_median_result_summarizes_samples(self) -> None:
self.assertEqual(DEFAULT_SAMPLE_COUNT, 3)
samples = [
{
"runtime": "python",