diff --git a/packages/cultcache-py/PARITY.md b/packages/cultcache-py/PARITY.md index 42155a13..4d2ec5ca 100644 --- a/packages/cultcache-py/PARITY.md +++ b/packages/cultcache-py/PARITY.md @@ -28,6 +28,9 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti. - Python package health: `cultcache-py-verify --json` checks public exports, `py.typed` markers, and benchmark sanity for Python-owned hot paths. +- Python/C# public cache baseline: + `cultcache_py.compare_csharp` runs Python and C# `CultCache` upsert/get + benchmarks with the same operation names and reports Python-to-C# ratios. ## Local Python Gates @@ -36,6 +39,7 @@ $env:PYTHONPATH="$PWD\packages\cultcache-py\src" python -m unittest discover -s packages\cultcache-py\tests python -m cultcache_py.verify --json python -m cultcache_py.benchmark --records 1000 --json +python -m cultcache_py.compare_csharp --records 1000 --json python -m pip wheel --no-deps -w $env:TEMP packages\cultcache-py ``` @@ -49,8 +53,9 @@ node --test packages\cultnet-ts\dist-test\test\interop\cultnet-interop.test.js ## Still Not Claimed -- Python does not match the C# runtime for raw throughput. It has a measured - baseline, not a performance victory certificate. +- Python does not claim to match the C# runtime for raw throughput. It has a + measured local baseline plus a shared public `CultCache` upsert/get comparison + harness, not a performance victory certificate. - Python does not own LiteNetLib security/session behavior; it speaks the schema-v0 interop lane used by the package harness. - Python does not implement the full C# daemon/server stack. Its current role is diff --git a/packages/cultcache-py/README.md b/packages/cultcache-py/README.md index b512c31b..4bdd911c 100644 --- a/packages/cultcache-py/README.md +++ b/packages/cultcache-py/README.md @@ -10,6 +10,7 @@ It is intentionally small. It is not an ORM, a database, or distributed consensu ## Current Shape - document types are registered explicitly +- a registered cache can be used in memory without attaching a backing store - `SingleFileMessagePackBackingStore` writes `cultcache.store.v1` snapshots: `[format, schemaCatalog, records]` - records store raw MessagePack payload bytes under schema-catalog identity @@ -29,7 +30,7 @@ It is intentionally small. It is not an ORM, a database, or distributed consensu ```python from dataclasses import dataclass, asdict -from cultcache_py import CultCache, JsonLinesBackingStore, define_document_type +from cultcache_py import CultCache, define_document_type @dataclass class Settings: @@ -46,15 +47,17 @@ settings_doc = define_document_type( cache = ( CultCache.builder() .register_document_type(settings_doc) - .add_generic_store(JsonLinesBackingStore("state.cultcache.jsonl")) .build() ) -cache.pull_all_backing_stores() cache.put_global(settings_doc, Settings(theme="ash", retries=3)) settings = cache.get_required_global(settings_doc) ``` +Add `JsonLinesBackingStore` or `SingleFileMessagePackBackingStore` when the +cache should persist state; local in-memory cache reads and writes do not need a +store. + ## Public Surface - `define_document_type(...)` @@ -308,14 +311,19 @@ Current receipts: ## Performance Baseline -The package ships a lightweight benchmark for Python-owned hot paths: +The package ships a lightweight benchmark for Python-owned hot paths. In a +CultLib source checkout, it can also run the C# reference benchmark beside it: ```powershell python -m cultcache_py.benchmark --records 1000 --json +python -m cultcache_py.compare_csharp --records 1000 --json python -m cultcache_py.verify --json ``` It reports slot-indexed `DatabaseEntry` encode/decode throughput, framed -CultNet MessagePack parse throughput, and raw snapshot application into a -registered cache. Treat it as a local baseline, not a claim that Python matches -the C# reference in every workload. +CultNet MessagePack parse throughput, raw snapshot application into a +registered cache, and public `CultCache` upsert/get throughput. The C# compare +command also runs `packages/cultcache-py/tools/GameCult.Caching.Benchmark` and +reports Python-to-C# ratios for the shared public cache operations. Treat those +numbers as a local baseline, not a claim that Python matches the C# reference in +every workload. diff --git a/packages/cultcache-py/pyproject.toml b/packages/cultcache-py/pyproject.toml index 9dc53769..979a01e3 100644 --- a/packages/cultcache-py/pyproject.toml +++ b/packages/cultcache-py/pyproject.toml @@ -25,6 +25,7 @@ msgpack = ["msgpack>=1.0"] cultcache-py-interop = "cultcache_py.interop:main" cultnet-py-interop = "cultnet_py.interop_peer:main" cultcache-py-benchmark = "cultcache_py.benchmark:main" +cultcache-py-compare-csharp = "cultcache_py.compare_csharp:main" cultcache-py-verify = "cultcache_py.verify:main" [tool.setuptools.packages.find] diff --git a/packages/cultcache-py/src/cultcache_py/benchmark.py b/packages/cultcache-py/src/cultcache_py/benchmark.py index 12d18b93..c5447c25 100644 --- a/packages/cultcache-py/src/cultcache_py/benchmark.py +++ b/packages/cultcache-py/src/cultcache_py/benchmark.py @@ -88,14 +88,27 @@ def run_benchmark(records: int) -> dict[str, Any]: cache.register_document_type(document) cache.add_generic_store(InMemoryBackingStore()) _, apply_metric = measure("raw_snapshot_apply", records, lambda: apply_raw_snapshot(cache, [document], snapshot)) + upsert_cache = CultCache() + upsert_cache.register_document_type(document) + _, upsert_metric = measure("cache_upsert", records, lambda: [ + upsert_cache.put(document, f"item:{index}", values[index]) + for index in range(records) + ]) + _, get_metric = measure("cache_get", records, lambda: [ + upsert_cache.get(document, f"item:{index}") + for index in range(records) + ]) return { + "runtime": "python", "records": records, "metrics": [ encode_metric, decode_metric, frame_metric, apply_metric, + upsert_metric, + get_metric, ], "wireMessageCount": len(wire_messages), } diff --git a/packages/cultcache-py/src/cultcache_py/cache.py b/packages/cultcache-py/src/cultcache_py/cache.py index 8e17b32e..465d34c5 100644 --- a/packages/cultcache-py/src/cultcache_py/cache.py +++ b/packages/cultcache-py/src/cultcache_py/cache.py @@ -178,8 +178,6 @@ class CultCache: catalog_entry=catalog_entry, ) stores = self._stores_for_type(document.type) - if not stores: - raise CultCacheError(f"No backing store registered for {document.type}") for store in stores: store.push(envelope) self._state.values[(document.type, key)] = value diff --git a/packages/cultcache-py/src/cultcache_py/compare_csharp.py b/packages/cultcache-py/src/cultcache_py/compare_csharp.py new file mode 100644 index 00000000..59dd09b6 --- /dev/null +++ b/packages/cultcache-py/src/cultcache_py/compare_csharp.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +from .benchmark import run_benchmark + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="cultcache-py-compare-csharp") + parser.add_argument("--records", type=int, default=5000) + parser.add_argument("--repo-root", type=Path, default=None) + parser.add_argument("--json", action="store_true", dest="emit_json") + args = parser.parse_args(argv) + if args.records <= 0: + raise ValueError("--records must be greater than zero") + + result = compare_with_csharp(records=args.records, repo_root=args.repo_root) + if args.emit_json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + print(f"records: {result['records']}") + print(f"csharpStatus: {result['csharpStatus']}") + for runtime in ("python", "csharp"): + benchmark = result.get(runtime) + if not benchmark: + continue + print(runtime) + for metric in benchmark["metrics"]: + print(f"- {metric['name']}: {metric['opsPerSecond']:.0f} ops/s ({metric['elapsedMs']:.2f} ms)") + return 0 if result["csharpStatus"] == "ok" else 1 + + +def compare_with_csharp(*, records: int, repo_root: Path | None = None) -> dict[str, Any]: + resolved_root = _resolve_repo_root(repo_root) + python_result = run_benchmark(records) + csharp_result, status, error = _run_csharp_benchmark(resolved_root, records) + comparison = _compare_common_metrics(python_result, csharp_result) if csharp_result else [] + result: dict[str, Any] = { + "records": records, + "repoRoot": str(resolved_root), + "python": python_result, + "csharpStatus": status, + "comparison": comparison, + } + if csharp_result is not None: + result["csharp"] = csharp_result + if error: + result["csharpError"] = error + return result + + +def _resolve_repo_root(repo_root: Path | None) -> Path: + if repo_root is not None: + return repo_root.resolve() + current = Path(__file__).resolve() + for parent in current.parents: + if (parent / "CultLib.sln").is_file(): + return parent + raise FileNotFoundError("Could not locate CultLib.sln; pass --repo-root.") + + +def _run_csharp_benchmark(repo_root: Path, records: int) -> tuple[dict[str, Any] | None, str, str | None]: + project = repo_root / "packages" / "cultcache-py" / "tools" / "GameCult.Caching.Benchmark" / "GameCult.Caching.Benchmark.csproj" + if not project.is_file(): + return None, "missing", f"C# benchmark project not found: {project}" + + completed = subprocess.run( + ["dotnet", "run", "--project", str(project), "-c", "Release", "--", "--records", str(records), "--json"], + cwd=repo_root, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if completed.returncode != 0: + return None, "failed", completed.stderr.strip() or completed.stdout.strip() + try: + payload = next(line for line in reversed(completed.stdout.splitlines()) if line.strip()) + return json.loads(payload), "ok", None + except json.JSONDecodeError as exc: + return None, "invalid-json", f"{exc}: {completed.stdout}" + except StopIteration: + return None, "invalid-json", "C# benchmark produced no stdout." + + +def _compare_common_metrics(python_result: dict[str, Any], csharp_result: dict[str, Any]) -> list[dict[str, Any]]: + python_metrics = {metric["name"]: metric for metric in python_result["metrics"]} + csharp_metrics = {metric["name"]: metric for metric in csharp_result["metrics"]} + comparison = [] + for name in sorted(set(python_metrics).intersection(csharp_metrics)): + python_ops = float(python_metrics[name]["opsPerSecond"]) + csharp_ops = float(csharp_metrics[name]["opsPerSecond"]) + comparison.append({ + "name": name, + "pythonOpsPerSecond": python_ops, + "csharpOpsPerSecond": csharp_ops, + "pythonToCsharpRatio": python_ops / csharp_ops if csharp_ops > 0 else None, + }) + return comparison + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/packages/cultcache-py/src/cultcache_py/verify.py b/packages/cultcache-py/src/cultcache_py/verify.py index f748a1ff..c2d78cf3 100644 --- a/packages/cultcache-py/src/cultcache_py/verify.py +++ b/packages/cultcache-py/src/cultcache_py/verify.py @@ -100,6 +100,8 @@ def _check_benchmark(records: int) -> dict[str, Any]: "database_entry_decode", "cultnet_frame_parse", "raw_snapshot_apply", + "cache_upsert", + "cache_get", ] metric_map = {metric["name"]: metric for metric in metrics["metrics"]} missing = [ diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 22753d95..c9eabb2f 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -211,9 +211,27 @@ class CultCacheTests(unittest.TestCase): "database_entry_decode", "cultnet_frame_parse", "raw_snapshot_apply", + "cache_upsert", + "cache_get", }) self.assertTrue(all(metric["opsPerSecond"] > 0 for metric in result["metrics"])) + def test_cache_put_without_backing_store_keeps_in_memory_value(self) -> None: + document = define_database_entry_type( + "bench.memory_item", + [ + ("name", 0), + ("value", 1, 0), + ], + ) + cache = CultCache() + cache.register_document_type(document) + + cache.put(document, "item:one", {"name": "one", "value": 1}) + + self.assertEqual(cache.get(document, "item:one"), {"name": "one", "value": 1}) + self.assertEqual(cache.get_required_envelope(document, "item:one").key, "item:one") + def test_verify_reports_python_runtime_surface_health(self) -> None: result = verify(records=4) diff --git a/packages/cultcache-py/tools/GameCult.Caching.Benchmark/GameCult.Caching.Benchmark.csproj b/packages/cultcache-py/tools/GameCult.Caching.Benchmark/GameCult.Caching.Benchmark.csproj new file mode 100644 index 00000000..9856b397 --- /dev/null +++ b/packages/cultcache-py/tools/GameCult.Caching.Benchmark/GameCult.Caching.Benchmark.csproj @@ -0,0 +1,12 @@ + + + Exe + net10.0 + enable + enable + + + + + + diff --git a/packages/cultcache-py/tools/GameCult.Caching.Benchmark/Program.cs b/packages/cultcache-py/tools/GameCult.Caching.Benchmark/Program.cs new file mode 100644 index 00000000..698b07ec --- /dev/null +++ b/packages/cultcache-py/tools/GameCult.Caching.Benchmark/Program.cs @@ -0,0 +1,104 @@ +using System.Diagnostics; +using System.Text.Json; +using GameCult.Caching; + +var records = 5000; +var emitJson = false; +for (var index = 0; index < args.Length; index++) +{ + if (args[index] == "--records" && index + 1 < args.Length) + { + records = int.Parse(args[++index]); + } + else if (args[index] == "--json") + { + emitJson = true; + } +} + +if (records <= 0) +{ + throw new ArgumentOutOfRangeException(nameof(records), "--records must be greater than zero."); +} + +var result = Benchmark.Run(records); +var options = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; +if (emitJson) +{ + Console.WriteLine(JsonSerializer.Serialize(result, options)); +} +else +{ + Console.WriteLine($"records: {result.Records}"); + foreach (var metric in result.Metrics) + { + Console.WriteLine($"{metric.Name}: {metric.OpsPerSecond:F0} ops/s ({metric.ElapsedMs:F2} ms)"); + } +} + +internal static class Benchmark +{ + public static BenchmarkResult Run(int records) + { + _ = CultDocumentRegistry.Shared.GetRequired(); + var values = Enumerable.Range(0, records) + .Select(index => new BenchItem + { + Name = $"item-{index}", + Category = $"cat-{index % 8}", + Value = index + }) + .ToArray(); + + var cache = new CultCache(); + var keys = Enumerable.Range(0, records) + .Select(index => new CultRecordKey($"item:{index}")) + .ToArray(); + + var upsertMetric = Measure("cache_upsert", records, () => + { + for (var index = 0; index < records; index++) + { + cache.UpsertAsync(values[index], new CultRecordHandle(keys[index])) + .GetAwaiter() + .GetResult(); + } + }); + var getMetric = Measure("cache_get", records, () => + { + for (var index = 0; index < records; index++) + { + _ = cache.Get(keys[index]); + } + }); + + return new BenchmarkResult("csharp", records, new[] { upsertMetric, getMetric }); + } + + private static BenchmarkMetric Measure(string name, int operations, Action action) + { + var started = Stopwatch.StartNew(); + action(); + started.Stop(); + return new BenchmarkMetric( + name, + operations, + started.Elapsed.TotalMilliseconds, + operations / started.Elapsed.TotalSeconds); + } +} + +internal sealed record BenchmarkResult(string Runtime, int Records, IReadOnlyList Metrics); + +internal sealed record BenchmarkMetric(string Name, int Operations, double ElapsedMs, double OpsPerSecond); + +[CultDocument("bench.item", "bench.item.v1")] +internal sealed class BenchItem +{ + [CultName] + public string Name { get; set; } = string.Empty; + + public string Category { get; set; } = string.Empty; + + public int Value { get; set; } +}