Use median samples for Python C# comparison
This commit is contained in:
parent
23adf5c182
commit
0827713076
4 changed files with 13 additions and 9 deletions
|
|
@ -127,9 +127,8 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti.
|
|||
Python-owned hot paths.
|
||||
- Python/C# public cache baseline:
|
||||
`cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get
|
||||
benchmarks with the same operation names and reports Python-to-C# ratios. Use
|
||||
`--samples` when a performance claim needs median evidence instead of a single
|
||||
noisy run.
|
||||
benchmarks with the same operation names and reports median Python-to-C# ratios
|
||||
from three samples by default.
|
||||
|
||||
## Local Python Gates
|
||||
|
||||
|
|
|
|||
|
|
@ -350,6 +350,7 @@ It reports slot-indexed `DatabaseEntry` encode/decode throughput, framed
|
|||
CultNet MessagePack parse throughput, raw snapshot application into a
|
||||
registered cache, and public `CultCache` upsert/get throughput. The C# compare
|
||||
command also runs `packages/cultcache-py/tools/GameCult.Caching.Benchmark` and
|
||||
reports Python-to-C# ratios for the shared public cache operations. Treat those
|
||||
numbers as a local baseline, not a claim that Python matches the C# reference in
|
||||
every workload.
|
||||
reports median Python-to-C# ratios for the shared public cache operations. It
|
||||
uses three samples by default; pass `--samples` when you need a different
|
||||
evidence shape. Treat those numbers as a local baseline, not a claim that
|
||||
Python matches the C# reference in every workload.
|
||||
|
|
|
|||
|
|
@ -11,10 +11,13 @@ from typing import Any
|
|||
from .benchmark import run_benchmark
|
||||
|
||||
|
||||
DEFAULT_SAMPLE_COUNT = 3
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp")
|
||||
parser.add_argument("--records", type=int, default=5000)
|
||||
parser.add_argument("--samples", type=int, default=1)
|
||||
parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLE_COUNT)
|
||||
parser.add_argument("--repo-root", type=Path, default=None)
|
||||
parser.add_argument("--json", action="store_true", dest="emit_json")
|
||||
args = parser.parse_args(argv)
|
||||
|
|
@ -39,7 +42,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
return 0 if result["csharpStatus"] == "ok" else 1
|
||||
|
||||
|
||||
def compare_with_csharp(*, records: int, samples: int = 1, repo_root: Path | None = None) -> dict[str, Any]:
|
||||
def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, repo_root: Path | None = None) -> dict[str, Any]:
|
||||
resolved_root = _resolve_repo_root(repo_root)
|
||||
python_samples = [run_benchmark(records) for _ in range(samples)]
|
||||
python_result = _median_result(python_samples)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from cultcache_py import (
|
|||
define_document_type,
|
||||
)
|
||||
from cultcache_py.benchmark import run_benchmark
|
||||
from cultcache_py.compare_csharp import _median_result
|
||||
from cultcache_py.compare_csharp import DEFAULT_SAMPLE_COUNT, _median_result
|
||||
from cultcache_py.interop import read_note, write_note
|
||||
from cultcache_py.verify import verify
|
||||
from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response
|
||||
|
|
@ -282,6 +282,7 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertTrue(all(metric["opsPerSecond"] > 0 for metric in result["metrics"]))
|
||||
|
||||
def test_compare_csharp_median_result_summarizes_samples(self) -> None:
|
||||
self.assertEqual(DEFAULT_SAMPLE_COUNT, 3)
|
||||
samples = [
|
||||
{
|
||||
"runtime": "python",
|
||||
|
|
|
|||
Loading…
Reference in a new issue