diff --git a/packages/cultcache-py/PARITY.md b/packages/cultcache-py/PARITY.md index ef67875a..251e815d 100644 --- a/packages/cultcache-py/PARITY.md +++ b/packages/cultcache-py/PARITY.md @@ -21,6 +21,11 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti. also ingest and filter remote schema catalog responses through typed `CultNetSchemaCatalog` and `CultNetSchemaDescriptor` helpers, including `CultNetRawClient.sync_schema_catalog(...)`. +- Python CultNet shard catalog: + Provides typed `CultNetShardCatalog` and `CultNetShardDescriptor` helpers for + ingesting, filtering, and emitting shard topology responses, plus + `CultNetRawClient.sync_shard_catalog(...)` for peer topology discovery over + the same framed CultNet pipe. - Python CultMesh discovery client: Fetches Verse catalogs and peer exchange responses from the live Python endpoint over CultNet frames. diff --git a/packages/cultcache-py/src/cultcache_py/verify.py b/packages/cultcache-py/src/cultcache_py/verify.py index 371f1124..edab7fb0 100644 --- a/packages/cultcache-py/src/cultcache_py/verify.py +++ b/packages/cultcache-py/src/cultcache_py/verify.py @@ -34,6 +34,8 @@ EXPECTED_EXPORTS: dict[str, tuple[str, ...]] = { "CultNetRawClient", "CultNetSchemaCatalog", "CultNetSchemaDescriptor", + "CultNetShardCatalog", + "CultNetShardDescriptor", "CultNetSimulationConsensus", "CultNetSimulationConsensusOptions", "CultNetSimulationObservationHub", diff --git a/packages/cultcache-py/src/cultnet_py/__init__.py b/packages/cultcache-py/src/cultnet_py/__init__.py index be16cc72..0d81d51f 100644 --- a/packages/cultcache-py/src/cultnet_py/__init__.py +++ b/packages/cultcache-py/src/cultnet_py/__init__.py @@ -34,6 +34,7 @@ from .schema_catalog import ( wire_message_schema_catalog, wire_message_schema_json, ) +from .shard_catalog import CultNetShardCatalog, CultNetShardDescriptor from .simulation import ( CultNetSimulationConsensus, CultNetSimulationConsensusOptions, @@ -46,6 +47,8 @@ __all__ = [ "CultNetRawClient", "CultNetSchemaCatalog", "CultNetSchemaDescriptor", + "CultNetShardCatalog", + "CultNetShardDescriptor", "CultNetAppliedRecord", "CultNetClientAuthorityScope", "CultNetSimulationConsensus", diff --git a/packages/cultcache-py/src/cultnet_py/client.py b/packages/cultcache-py/src/cultnet_py/client.py index 633e22f0..769baf8f 100644 --- a/packages/cultcache-py/src/cultnet_py/client.py +++ b/packages/cultcache-py/src/cultnet_py/client.py @@ -17,6 +17,7 @@ from .messages import ( snapshot_request, ) from .schema_catalog import CultNetSchemaCatalog, CultNetSchemaDescriptor +from .shard_catalog import CultNetShardCatalog, CultNetShardDescriptor @dataclass(frozen=True) @@ -123,6 +124,35 @@ class CultNetRawClient: expected_schema_version="cultnet.shard_catalog_response.v0", ) + def fetch_shard_descriptors( + self, + *, + message_id: str = "cultnet-python-shard-catalog", + schema_ids: list[str] | None = None, + record_keys: list[str] | None = None, + ) -> list[CultNetShardDescriptor]: + response = self.fetch_shard_catalog( + message_id=message_id, + schema_ids=schema_ids, + record_keys=record_keys, + ) + return CultNetShardCatalog().apply_response(response) + + def sync_shard_catalog( + self, + catalog: CultNetShardCatalog, + *, + message_id: str = "cultnet-python-shard-catalog", + schema_ids: list[str] | None = None, + record_keys: list[str] | None = None, + ) -> list[CultNetShardDescriptor]: + response = self.fetch_shard_catalog( + message_id=message_id, + schema_ids=schema_ids, + record_keys=record_keys, + ) + return catalog.apply_response(response) + def fetch_shard_log( self, *, diff --git a/packages/cultcache-py/src/cultnet_py/shard_catalog.py b/packages/cultcache-py/src/cultnet_py/shard_catalog.py new file mode 100644 index 00000000..951ad5b7 --- /dev/null +++ b/packages/cultcache-py/src/cultnet_py/shard_catalog.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class CultNetShardDescriptor: + shard_id: str + owner_runtime_id: str + epoch: int = 0 + is_primary: bool = False + schema_ids: tuple[str, ...] = () + key_prefix: str | None = None + primary_endpoints: tuple[str, ...] = () + replica_endpoints: tuple[str, ...] = () + read_replica_endpoints: tuple[str, ...] = () + region: str | None = None + authority_lease_id: str | None = None + + @classmethod + def from_wire(cls, value: dict[str, Any]) -> "CultNetShardDescriptor": + shard_id = str(value.get("shardId") or "") + owner_runtime_id = str(value.get("ownerRuntimeId") or "") + if not shard_id: + raise ValueError("shard descriptor shardId must be non-empty") + if not owner_runtime_id: + raise ValueError("shard descriptor ownerRuntimeId must be non-empty") + return cls( + shard_id=shard_id, + owner_runtime_id=owner_runtime_id, + epoch=int(value.get("epoch") or 0), + is_primary=value.get("isPrimary") is True, + schema_ids=_string_tuple(value.get("schemaIds")), + key_prefix=_optional_string(value.get("keyPrefix")), + primary_endpoints=_string_tuple(value.get("primaryEndpoints")), + replica_endpoints=_string_tuple(value.get("replicaEndpoints")), + read_replica_endpoints=_string_tuple(value.get("readReplicaEndpoints")), + region=_optional_string(value.get("region")), + authority_lease_id=_optional_string(value.get("authorityLeaseId")), + ) + + def to_wire(self) -> dict[str, Any]: + wire: dict[str, Any] = { + "shardId": self.shard_id, + "ownerRuntimeId": self.owner_runtime_id, + "epoch": self.epoch, + "isPrimary": self.is_primary, + "schemaIds": list(self.schema_ids), + "primaryEndpoints": list(self.primary_endpoints), + "replicaEndpoints": list(self.replica_endpoints), + "readReplicaEndpoints": list(self.read_replica_endpoints), + } + if self.key_prefix is not None: + wire["keyPrefix"] = self.key_prefix + if self.region is not None: + wire["region"] = self.region + if self.authority_lease_id is not None: + wire["authorityLeaseId"] = self.authority_lease_id + return wire + + def serves(self, *, schema_id: str | None = None, record_key: str | None = None) -> bool: + if schema_id is not None and self.schema_ids and schema_id not in self.schema_ids: + return False + if record_key is not None and self.key_prefix and not record_key.startswith(self.key_prefix): + return False + return True + + +@dataclass +class CultNetShardCatalog: + _shards: dict[str, CultNetShardDescriptor] = field(default_factory=dict) + + def upsert(self, descriptor: CultNetShardDescriptor | dict[str, Any]) -> CultNetShardDescriptor: + value = descriptor if isinstance(descriptor, CultNetShardDescriptor) else CultNetShardDescriptor.from_wire(descriptor) + self._shards[value.shard_id] = value + return value + + def get(self, shard_id: str) -> CultNetShardDescriptor | None: + return self._shards.get(shard_id) + + def list( + self, + *, + schema_ids: list[str] | None = None, + record_keys: list[str] | None = None, + ) -> list[CultNetShardDescriptor]: + requested_schema_ids = set(schema_ids or []) + requested_record_keys = list(record_keys or []) + return [ + shard + for shard in sorted(self._shards.values(), key=lambda item: item.shard_id) + if _shard_matches(shard, requested_schema_ids, requested_record_keys) + ] + + def apply_response(self, response: dict[str, Any]) -> list[CultNetShardDescriptor]: + if response.get("schemaVersion") != "cultnet.shard_catalog_response.v0": + raise ValueError(f"Expected cultnet.shard_catalog_response.v0, received {response.get('schemaVersion')!r}") + applied = [] + for shard in response.get("shards") or []: + if not isinstance(shard, dict): + continue + applied.append(self.upsert(shard)) + return applied + + def create_response( + self, + *, + message_id: str = "", + schema_ids: list[str] | None = None, + record_keys: list[str] | None = None, + ) -> dict[str, Any]: + return { + "schemaVersion": "cultnet.shard_catalog_response.v0", + "messageId": message_id, + "shards": [shard.to_wire() for shard in self.list(schema_ids=schema_ids, record_keys=record_keys)], + } + + +def _shard_matches( + shard: CultNetShardDescriptor, + requested_schema_ids: set[str], + requested_record_keys: list[str], +) -> bool: + if requested_schema_ids and not any(schema_id in shard.schema_ids for schema_id in requested_schema_ids): + return False + if requested_record_keys and not any(shard.serves(record_key=record_key) for record_key in requested_record_keys): + return False + return True + + +def _string_tuple(value: Any) -> tuple[str, ...]: + return tuple(str(item) for item in value or []) + + +def _optional_string(value: Any) -> str | None: + if value is None: + return None + text = str(value) + return text if text else None diff --git a/packages/cultcache-py/tests/test_cultcache.py b/packages/cultcache-py/tests/test_cultcache.py index 2c1335dd..eba8cc7f 100644 --- a/packages/cultcache-py/tests/test_cultcache.py +++ b/packages/cultcache-py/tests/test_cultcache.py @@ -26,6 +26,8 @@ from cultnet_py import ( CultNetRawClient, CultNetSchemaCatalog, CultNetSchemaDescriptor, + CultNetShardCatalog, + CultNetShardDescriptor, CultNetSimulationConsensusOptions, apply_raw_snapshot, apply_shard_log_response, @@ -426,7 +428,18 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(request["shardId"], "interop") response = {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": request["messageId"], "documents": []} elif request["schemaVersion"] == "cultnet.shard_catalog_request.v0": - response = {"schemaVersion": "cultnet.shard_catalog_response.v0", "messageId": request["messageId"], "shards": []} + response = { + "schemaVersion": "cultnet.shard_catalog_response.v0", + "messageId": request["messageId"], + "shards": [{ + "shardId": "interop", + "ownerRuntimeId": "socket-test", + "epoch": 1, + "schemaIds": request["schemaIds"], + "keyPrefix": "record-", + "primaryEndpoints": ["cultnet://127.0.0.1:1"], + }], + } elif request["schemaVersion"] == "cultnet.shard_log_request.v0": self.assertEqual(request["afterSequence"], 7) response = { @@ -453,7 +466,10 @@ class CultCacheTests(unittest.TestCase): client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0) self.assertEqual(client.fetch_schema_catalog(kinds=["wire_message"])["schemaVersion"], "cultnet.schema_catalog_response.v0") self.assertEqual(client.fetch_snapshot(shard_id="interop", shard_epoch=1)["schemaVersion"], "cultnet.snapshot_response_raw.v0") - self.assertEqual(client.fetch_shard_catalog(schema_ids=["schema-a"])["schemaVersion"], "cultnet.shard_catalog_response.v0") + shard_catalog = CultNetShardCatalog() + synced_shards = client.sync_shard_catalog(shard_catalog, schema_ids=["schema-a"]) + self.assertEqual(synced_shards[0].shard_id, "interop") + self.assertEqual(shard_catalog.get("interop"), synced_shards[0]) self.assertEqual(client.fetch_shard_log(shard_id="interop", shard_epoch=1, after_sequence=7)["schemaVersion"], "cultnet.shard_log_response.v0") thread.join(2.0) @@ -902,6 +918,47 @@ class CultCacheTests(unittest.TestCase): self.assertEqual(log["afterSequence"], 3) self.assertEqual(log["limit"], 2) + def test_cultnet_shard_catalog_applies_filters_and_responses(self) -> None: + catalog = CultNetShardCatalog() + primary = catalog.upsert(CultNetShardDescriptor( + shard_id="notes", + owner_runtime_id="python-runtime", + epoch=2, + is_primary=True, + schema_ids=("schema-note",), + key_prefix="note:", + primary_endpoints=("cultnet://127.0.0.1:3075",), + read_replica_endpoints=("cultnet://127.0.0.1:3075",), + region="local", + )) + catalog.upsert({ + "shardId": "facts", + "ownerRuntimeId": "python-runtime", + "epoch": 1, + "schemaIds": ["schema-fact"], + "keyPrefix": "fact:", + "primaryEndpoints": ["cultnet://127.0.0.1:3076"], + }) + + self.assertTrue(primary.serves(schema_id="schema-note", record_key="note:1")) + self.assertFalse(primary.serves(schema_id="schema-note", record_key="fact:1")) + self.assertEqual(catalog.list(schema_ids=["schema-note"]), [primary]) + self.assertEqual(catalog.list(record_keys=["note:1"]), [primary]) + + response = catalog.create_response( + message_id="shards", + schema_ids=["schema-note"], + record_keys=["note:1"], + ) + self.assertEqual(response["schemaVersion"], "cultnet.shard_catalog_response.v0") + self.assertEqual(response["shards"][0]["shardId"], "notes") + self.assertEqual(response["shards"][0]["ownerRuntimeId"], "python-runtime") + + remote = CultNetShardCatalog() + applied = remote.apply_response(response) + self.assertEqual(applied[0].shard_id, "notes") + self.assertEqual(remote.get("notes"), applied[0]) + def test_cultnet_simulation_observation_helper_matches_schema_v0_shape(self) -> None: claim_hash = compute_simulation_claim_hash("frame:42", "subject:player-1", "hit") self.assertEqual(len(claim_hash), 64) @@ -1708,6 +1765,8 @@ class CultCacheTests(unittest.TestCase): ) snapshot_response = raw_client.fetch_snapshot(schema_ids=["mesh.server_note.v1"]) shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.server_note.v1"]) + synced_shard_catalog = CultNetShardCatalog() + synced_shards = raw_client.sync_shard_catalog(synced_shard_catalog, schema_ids=["mesh.server_note.v1"]) shard_log = raw_client.fetch_shard_log(shard_id="primary", shard_epoch=1) discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0) fetched_verses = discovery_client.fetch_verses(transport_version="cultmesh.v0") @@ -1749,6 +1808,8 @@ class CultCacheTests(unittest.TestCase): self.assertIsNotNone(synced_schema_catalog.get("https://github.com/GameCult/cultnet-ts/contracts/cultnet.document-put-raw.schema.json")) self.assertEqual(snapshot_response["documents"][0]["recordKey"], "note:1") self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["mesh.server_note.v1"]) + self.assertEqual(synced_shards[0].shard_id, "primary") + self.assertEqual(synced_shard_catalog.get("primary"), synced_shards[0]) self.assertEqual(shard_log["entries"][0]["changeKind"], "added") self.assertEqual(fetched_verses[0].verse_id, "server-verse") self.assertEqual(fetched_peers[0].peer_id, "mesh-server")