Gate Python C# parity claims
This commit is contained in:
parent
031d879695
commit
a3ccaa0290
3 changed files with 80 additions and 9 deletions
|
|
@ -169,7 +169,8 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti.
|
|||
- Python/C# public cache baseline:
|
||||
`cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get
|
||||
benchmarks with the same operation names and reports median Python-to-C# ratios
|
||||
from three samples by default.
|
||||
from three samples by default. It also emits per-metric threshold checks and an
|
||||
overall parity status so performance claims have an explicit evidence gate.
|
||||
|
||||
## Local Python Gates
|
||||
|
||||
|
|
@ -194,7 +195,7 @@ node --test packages\cultnet-ts\dist-test\test\interop\cultnet-interop.test.js
|
|||
|
||||
- Python does not claim to match the C# runtime for raw throughput. It has a
|
||||
measured local baseline plus a shared public `CultCache` upsert/get comparison
|
||||
harness, not a performance victory certificate.
|
||||
harness with threshold status, not a performance victory certificate.
|
||||
- Python does not own LiteNetLib security/session behavior; it speaks the
|
||||
schema-v0 interop lane used by the package harness.
|
||||
- Python does not implement the full C# daemon/server stack. Its current role is
|
||||
|
|
|
|||
|
|
@ -12,12 +12,14 @@ from .benchmark import run_benchmark
|
|||
|
||||
|
||||
DEFAULT_SAMPLE_COUNT = 3
|
||||
DEFAULT_PARITY_THRESHOLD = 0.90
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp")
|
||||
parser.add_argument("--records", type=int, default=5000)
|
||||
parser.add_argument("--samples", type=int, default=DEFAULT_SAMPLE_COUNT)
|
||||
parser.add_argument("--parity-threshold", type=float, default=DEFAULT_PARITY_THRESHOLD)
|
||||
parser.add_argument("--repo-root", type=Path, default=None)
|
||||
parser.add_argument("--json", action="store_true", dest="emit_json")
|
||||
args = parser.parse_args(argv)
|
||||
|
|
@ -25,8 +27,15 @@ def main(argv: list[str] | None = None) -> int:
|
|||
raise ValueError("--records must be greater than zero")
|
||||
if args.samples <= 0:
|
||||
raise ValueError("--samples must be greater than zero")
|
||||
if args.parity_threshold <= 0:
|
||||
raise ValueError("--parity-threshold must be greater than zero")
|
||||
|
||||
result = compare_with_csharp(records=args.records, samples=args.samples, repo_root=args.repo_root)
|
||||
result = compare_with_csharp(
|
||||
records=args.records,
|
||||
samples=args.samples,
|
||||
repo_root=args.repo_root,
|
||||
parity_threshold=args.parity_threshold,
|
||||
)
|
||||
if args.emit_json:
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
else:
|
||||
|
|
@ -39,10 +48,18 @@ def main(argv: list[str] | None = None) -> int:
|
|||
print(runtime)
|
||||
for metric in benchmark["metrics"]:
|
||||
print(f"- {metric['name']}: {metric['opsPerSecond']:.0f} ops/s ({metric['elapsedMs']:.2f} ms)")
|
||||
print(f"parityThreshold: {result['parityThreshold']:.2f}")
|
||||
print(f"parityStatus: {result['parityStatus']}")
|
||||
return 0 if result["csharpStatus"] == "ok" else 1
|
||||
|
||||
|
||||
def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, repo_root: Path | None = None) -> dict[str, Any]:
|
||||
def compare_with_csharp(
|
||||
*,
|
||||
records: int,
|
||||
samples: int = DEFAULT_SAMPLE_COUNT,
|
||||
repo_root: Path | None = None,
|
||||
parity_threshold: float = DEFAULT_PARITY_THRESHOLD,
|
||||
) -> dict[str, Any]:
|
||||
resolved_root = _resolve_repo_root(repo_root)
|
||||
python_samples = [run_benchmark(records) for _ in range(samples)]
|
||||
python_result = _median_result(python_samples)
|
||||
|
|
@ -55,7 +72,11 @@ def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, re
|
|||
break
|
||||
csharp_samples.append(csharp_result)
|
||||
csharp_result = _median_result(csharp_samples) if csharp_samples else None
|
||||
comparison = _compare_common_metrics(python_result, csharp_result) if csharp_result else []
|
||||
comparison = _compare_common_metrics(
|
||||
python_result,
|
||||
csharp_result,
|
||||
parity_threshold=parity_threshold,
|
||||
) if csharp_result else []
|
||||
result: dict[str, Any] = {
|
||||
"records": records,
|
||||
"samples": samples,
|
||||
|
|
@ -64,6 +85,8 @@ def compare_with_csharp(*, records: int, samples: int = DEFAULT_SAMPLE_COUNT, re
|
|||
"pythonSamples": python_samples,
|
||||
"csharpStatus": status,
|
||||
"comparison": comparison,
|
||||
"parityThreshold": parity_threshold,
|
||||
"parityStatus": _parity_status(comparison, status),
|
||||
}
|
||||
if csharp_result is not None:
|
||||
result["csharp"] = csharp_result
|
||||
|
|
@ -107,22 +130,39 @@ def _run_csharp_benchmark(repo_root: Path, records: int) -> tuple[dict[str, Any]
|
|||
return None, "invalid-json", "C# benchmark produced no stdout."
|
||||
|
||||
|
||||
def _compare_common_metrics(python_result: dict[str, Any], csharp_result: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
def _compare_common_metrics(
|
||||
python_result: dict[str, Any],
|
||||
csharp_result: dict[str, Any],
|
||||
*,
|
||||
parity_threshold: float = DEFAULT_PARITY_THRESHOLD,
|
||||
) -> list[dict[str, Any]]:
|
||||
python_metrics = {metric["name"]: metric for metric in python_result["metrics"]}
|
||||
csharp_metrics = {metric["name"]: metric for metric in csharp_result["metrics"]}
|
||||
comparison = []
|
||||
for name in sorted(set(python_metrics).intersection(csharp_metrics)):
|
||||
python_ops = float(python_metrics[name]["opsPerSecond"])
|
||||
csharp_ops = float(csharp_metrics[name]["opsPerSecond"])
|
||||
ratio = python_ops / csharp_ops if csharp_ops > 0 else None
|
||||
comparison.append({
|
||||
"name": name,
|
||||
"pythonOpsPerSecond": python_ops,
|
||||
"csharpOpsPerSecond": csharp_ops,
|
||||
"pythonToCsharpRatio": python_ops / csharp_ops if csharp_ops > 0 else None,
|
||||
"pythonToCsharpRatio": ratio,
|
||||
"meetsParityThreshold": ratio is not None and ratio >= parity_threshold,
|
||||
})
|
||||
return comparison
|
||||
|
||||
|
||||
def _parity_status(comparison: list[dict[str, Any]], csharp_status: str) -> str:
|
||||
if csharp_status != "ok":
|
||||
return "unknown"
|
||||
if not comparison:
|
||||
return "unknown"
|
||||
if all(item.get("meetsParityThreshold") is True for item in comparison):
|
||||
return "meets-threshold"
|
||||
return "below-threshold"
|
||||
|
||||
|
||||
def _median_result(samples: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
if not samples:
|
||||
raise ValueError("Cannot summarize zero benchmark samples")
|
||||
|
|
|
|||
|
|
@ -20,7 +20,13 @@ from cultcache_py import (
|
|||
define_document_type,
|
||||
)
|
||||
from cultcache_py.benchmark import run_benchmark
|
||||
from cultcache_py.compare_csharp import DEFAULT_SAMPLE_COUNT, _median_result
|
||||
from cultcache_py.compare_csharp import (
|
||||
DEFAULT_PARITY_THRESHOLD,
|
||||
DEFAULT_SAMPLE_COUNT,
|
||||
_compare_common_metrics,
|
||||
_median_result,
|
||||
_parity_status,
|
||||
)
|
||||
from cultcache_py.interop import read_note, write_note
|
||||
from cultcache_py.verify import verify
|
||||
from cultnet_py.interop_peer import append_shard_log_put, build_state, raw_snapshot_response
|
||||
|
|
@ -309,6 +315,7 @@ class CultCacheTests(unittest.TestCase):
|
|||
|
||||
def test_compare_csharp_median_result_summarizes_samples(self) -> None:
|
||||
self.assertEqual(DEFAULT_SAMPLE_COUNT, 3)
|
||||
self.assertEqual(DEFAULT_PARITY_THRESHOLD, 0.90)
|
||||
samples = [
|
||||
{
|
||||
"runtime": "python",
|
||||
|
|
@ -344,6 +351,29 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual(summarized["metrics"][0]["opsPerSecond"], 200.0)
|
||||
self.assertEqual(summarized["metrics"][1]["opsPerSecond"], 75.0)
|
||||
|
||||
def test_compare_csharp_marks_threshold_status_per_metric(self) -> None:
|
||||
python_result = {
|
||||
"metrics": [
|
||||
{"name": "cache_get", "operations": 3, "elapsedMs": 2.0, "opsPerSecond": 50.0},
|
||||
{"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 95.0},
|
||||
],
|
||||
}
|
||||
csharp_result = {
|
||||
"metrics": [
|
||||
{"name": "cache_get", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0},
|
||||
{"name": "cache_upsert", "operations": 3, "elapsedMs": 1.0, "opsPerSecond": 100.0},
|
||||
],
|
||||
}
|
||||
|
||||
comparison = _compare_common_metrics(python_result, csharp_result, parity_threshold=0.90)
|
||||
by_name = {item["name"]: item for item in comparison}
|
||||
|
||||
self.assertFalse(by_name["cache_get"]["meetsParityThreshold"])
|
||||
self.assertTrue(by_name["cache_upsert"]["meetsParityThreshold"])
|
||||
self.assertEqual(_parity_status(comparison, "ok"), "below-threshold")
|
||||
self.assertEqual(_parity_status([by_name["cache_upsert"]], "ok"), "meets-threshold")
|
||||
self.assertEqual(_parity_status(comparison, "failed"), "unknown")
|
||||
|
||||
def test_cache_put_without_backing_store_keeps_in_memory_value(self) -> None:
|
||||
document = define_database_entry_type(
|
||||
"bench.memory_item",
|
||||
|
|
|
|||
Loading…
Reference in a new issue