Add Python CultNet shard catalog facade
This commit is contained in:
parent
c47b84560e
commit
c1afc6b77f
6 changed files with 243 additions and 2 deletions
|
|
@ -21,6 +21,11 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti.
|
|||
also ingest and filter remote schema catalog responses through typed
|
||||
`CultNetSchemaCatalog` and `CultNetSchemaDescriptor` helpers, including
|
||||
`CultNetRawClient.sync_schema_catalog(...)`.
|
||||
- Python CultNet shard catalog:
|
||||
Provides typed `CultNetShardCatalog` and `CultNetShardDescriptor` helpers for
|
||||
ingesting, filtering, and emitting shard topology responses, plus
|
||||
`CultNetRawClient.sync_shard_catalog(...)` for peer topology discovery over
|
||||
the same framed CultNet pipe.
|
||||
- Python CultMesh discovery client:
|
||||
Fetches Verse catalogs and peer exchange responses from the live Python
|
||||
endpoint over CultNet frames.
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ EXPECTED_EXPORTS: dict[str, tuple[str, ...]] = {
|
|||
"CultNetRawClient",
|
||||
"CultNetSchemaCatalog",
|
||||
"CultNetSchemaDescriptor",
|
||||
"CultNetShardCatalog",
|
||||
"CultNetShardDescriptor",
|
||||
"CultNetSimulationConsensus",
|
||||
"CultNetSimulationConsensusOptions",
|
||||
"CultNetSimulationObservationHub",
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from .schema_catalog import (
|
|||
wire_message_schema_catalog,
|
||||
wire_message_schema_json,
|
||||
)
|
||||
from .shard_catalog import CultNetShardCatalog, CultNetShardDescriptor
|
||||
from .simulation import (
|
||||
CultNetSimulationConsensus,
|
||||
CultNetSimulationConsensusOptions,
|
||||
|
|
@ -46,6 +47,8 @@ __all__ = [
|
|||
"CultNetRawClient",
|
||||
"CultNetSchemaCatalog",
|
||||
"CultNetSchemaDescriptor",
|
||||
"CultNetShardCatalog",
|
||||
"CultNetShardDescriptor",
|
||||
"CultNetAppliedRecord",
|
||||
"CultNetClientAuthorityScope",
|
||||
"CultNetSimulationConsensus",
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from .messages import (
|
|||
snapshot_request,
|
||||
)
|
||||
from .schema_catalog import CultNetSchemaCatalog, CultNetSchemaDescriptor
|
||||
from .shard_catalog import CultNetShardCatalog, CultNetShardDescriptor
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -123,6 +124,35 @@ class CultNetRawClient:
|
|||
expected_schema_version="cultnet.shard_catalog_response.v0",
|
||||
)
|
||||
|
||||
def fetch_shard_descriptors(
|
||||
self,
|
||||
*,
|
||||
message_id: str = "cultnet-python-shard-catalog",
|
||||
schema_ids: list[str] | None = None,
|
||||
record_keys: list[str] | None = None,
|
||||
) -> list[CultNetShardDescriptor]:
|
||||
response = self.fetch_shard_catalog(
|
||||
message_id=message_id,
|
||||
schema_ids=schema_ids,
|
||||
record_keys=record_keys,
|
||||
)
|
||||
return CultNetShardCatalog().apply_response(response)
|
||||
|
||||
def sync_shard_catalog(
|
||||
self,
|
||||
catalog: CultNetShardCatalog,
|
||||
*,
|
||||
message_id: str = "cultnet-python-shard-catalog",
|
||||
schema_ids: list[str] | None = None,
|
||||
record_keys: list[str] | None = None,
|
||||
) -> list[CultNetShardDescriptor]:
|
||||
response = self.fetch_shard_catalog(
|
||||
message_id=message_id,
|
||||
schema_ids=schema_ids,
|
||||
record_keys=record_keys,
|
||||
)
|
||||
return catalog.apply_response(response)
|
||||
|
||||
def fetch_shard_log(
|
||||
self,
|
||||
*,
|
||||
|
|
|
|||
140
packages/cultcache-py/src/cultnet_py/shard_catalog.py
Normal file
140
packages/cultcache-py/src/cultnet_py/shard_catalog.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CultNetShardDescriptor:
|
||||
shard_id: str
|
||||
owner_runtime_id: str
|
||||
epoch: int = 0
|
||||
is_primary: bool = False
|
||||
schema_ids: tuple[str, ...] = ()
|
||||
key_prefix: str | None = None
|
||||
primary_endpoints: tuple[str, ...] = ()
|
||||
replica_endpoints: tuple[str, ...] = ()
|
||||
read_replica_endpoints: tuple[str, ...] = ()
|
||||
region: str | None = None
|
||||
authority_lease_id: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_wire(cls, value: dict[str, Any]) -> "CultNetShardDescriptor":
|
||||
shard_id = str(value.get("shardId") or "")
|
||||
owner_runtime_id = str(value.get("ownerRuntimeId") or "")
|
||||
if not shard_id:
|
||||
raise ValueError("shard descriptor shardId must be non-empty")
|
||||
if not owner_runtime_id:
|
||||
raise ValueError("shard descriptor ownerRuntimeId must be non-empty")
|
||||
return cls(
|
||||
shard_id=shard_id,
|
||||
owner_runtime_id=owner_runtime_id,
|
||||
epoch=int(value.get("epoch") or 0),
|
||||
is_primary=value.get("isPrimary") is True,
|
||||
schema_ids=_string_tuple(value.get("schemaIds")),
|
||||
key_prefix=_optional_string(value.get("keyPrefix")),
|
||||
primary_endpoints=_string_tuple(value.get("primaryEndpoints")),
|
||||
replica_endpoints=_string_tuple(value.get("replicaEndpoints")),
|
||||
read_replica_endpoints=_string_tuple(value.get("readReplicaEndpoints")),
|
||||
region=_optional_string(value.get("region")),
|
||||
authority_lease_id=_optional_string(value.get("authorityLeaseId")),
|
||||
)
|
||||
|
||||
def to_wire(self) -> dict[str, Any]:
|
||||
wire: dict[str, Any] = {
|
||||
"shardId": self.shard_id,
|
||||
"ownerRuntimeId": self.owner_runtime_id,
|
||||
"epoch": self.epoch,
|
||||
"isPrimary": self.is_primary,
|
||||
"schemaIds": list(self.schema_ids),
|
||||
"primaryEndpoints": list(self.primary_endpoints),
|
||||
"replicaEndpoints": list(self.replica_endpoints),
|
||||
"readReplicaEndpoints": list(self.read_replica_endpoints),
|
||||
}
|
||||
if self.key_prefix is not None:
|
||||
wire["keyPrefix"] = self.key_prefix
|
||||
if self.region is not None:
|
||||
wire["region"] = self.region
|
||||
if self.authority_lease_id is not None:
|
||||
wire["authorityLeaseId"] = self.authority_lease_id
|
||||
return wire
|
||||
|
||||
def serves(self, *, schema_id: str | None = None, record_key: str | None = None) -> bool:
|
||||
if schema_id is not None and self.schema_ids and schema_id not in self.schema_ids:
|
||||
return False
|
||||
if record_key is not None and self.key_prefix and not record_key.startswith(self.key_prefix):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
class CultNetShardCatalog:
|
||||
_shards: dict[str, CultNetShardDescriptor] = field(default_factory=dict)
|
||||
|
||||
def upsert(self, descriptor: CultNetShardDescriptor | dict[str, Any]) -> CultNetShardDescriptor:
|
||||
value = descriptor if isinstance(descriptor, CultNetShardDescriptor) else CultNetShardDescriptor.from_wire(descriptor)
|
||||
self._shards[value.shard_id] = value
|
||||
return value
|
||||
|
||||
def get(self, shard_id: str) -> CultNetShardDescriptor | None:
|
||||
return self._shards.get(shard_id)
|
||||
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
schema_ids: list[str] | None = None,
|
||||
record_keys: list[str] | None = None,
|
||||
) -> list[CultNetShardDescriptor]:
|
||||
requested_schema_ids = set(schema_ids or [])
|
||||
requested_record_keys = list(record_keys or [])
|
||||
return [
|
||||
shard
|
||||
for shard in sorted(self._shards.values(), key=lambda item: item.shard_id)
|
||||
if _shard_matches(shard, requested_schema_ids, requested_record_keys)
|
||||
]
|
||||
|
||||
def apply_response(self, response: dict[str, Any]) -> list[CultNetShardDescriptor]:
|
||||
if response.get("schemaVersion") != "cultnet.shard_catalog_response.v0":
|
||||
raise ValueError(f"Expected cultnet.shard_catalog_response.v0, received {response.get('schemaVersion')!r}")
|
||||
applied = []
|
||||
for shard in response.get("shards") or []:
|
||||
if not isinstance(shard, dict):
|
||||
continue
|
||||
applied.append(self.upsert(shard))
|
||||
return applied
|
||||
|
||||
def create_response(
|
||||
self,
|
||||
*,
|
||||
message_id: str = "",
|
||||
schema_ids: list[str] | None = None,
|
||||
record_keys: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"schemaVersion": "cultnet.shard_catalog_response.v0",
|
||||
"messageId": message_id,
|
||||
"shards": [shard.to_wire() for shard in self.list(schema_ids=schema_ids, record_keys=record_keys)],
|
||||
}
|
||||
|
||||
|
||||
def _shard_matches(
|
||||
shard: CultNetShardDescriptor,
|
||||
requested_schema_ids: set[str],
|
||||
requested_record_keys: list[str],
|
||||
) -> bool:
|
||||
if requested_schema_ids and not any(schema_id in shard.schema_ids for schema_id in requested_schema_ids):
|
||||
return False
|
||||
if requested_record_keys and not any(shard.serves(record_key=record_key) for record_key in requested_record_keys):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _string_tuple(value: Any) -> tuple[str, ...]:
|
||||
return tuple(str(item) for item in value or [])
|
||||
|
||||
|
||||
def _optional_string(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value)
|
||||
return text if text else None
|
||||
|
|
@ -26,6 +26,8 @@ from cultnet_py import (
|
|||
CultNetRawClient,
|
||||
CultNetSchemaCatalog,
|
||||
CultNetSchemaDescriptor,
|
||||
CultNetShardCatalog,
|
||||
CultNetShardDescriptor,
|
||||
CultNetSimulationConsensusOptions,
|
||||
apply_raw_snapshot,
|
||||
apply_shard_log_response,
|
||||
|
|
@ -426,7 +428,18 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual(request["shardId"], "interop")
|
||||
response = {"schemaVersion": "cultnet.snapshot_response_raw.v0", "messageId": request["messageId"], "documents": []}
|
||||
elif request["schemaVersion"] == "cultnet.shard_catalog_request.v0":
|
||||
response = {"schemaVersion": "cultnet.shard_catalog_response.v0", "messageId": request["messageId"], "shards": []}
|
||||
response = {
|
||||
"schemaVersion": "cultnet.shard_catalog_response.v0",
|
||||
"messageId": request["messageId"],
|
||||
"shards": [{
|
||||
"shardId": "interop",
|
||||
"ownerRuntimeId": "socket-test",
|
||||
"epoch": 1,
|
||||
"schemaIds": request["schemaIds"],
|
||||
"keyPrefix": "record-",
|
||||
"primaryEndpoints": ["cultnet://127.0.0.1:1"],
|
||||
}],
|
||||
}
|
||||
elif request["schemaVersion"] == "cultnet.shard_log_request.v0":
|
||||
self.assertEqual(request["afterSequence"], 7)
|
||||
response = {
|
||||
|
|
@ -453,7 +466,10 @@ class CultCacheTests(unittest.TestCase):
|
|||
client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0)
|
||||
self.assertEqual(client.fetch_schema_catalog(kinds=["wire_message"])["schemaVersion"], "cultnet.schema_catalog_response.v0")
|
||||
self.assertEqual(client.fetch_snapshot(shard_id="interop", shard_epoch=1)["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
||||
self.assertEqual(client.fetch_shard_catalog(schema_ids=["schema-a"])["schemaVersion"], "cultnet.shard_catalog_response.v0")
|
||||
shard_catalog = CultNetShardCatalog()
|
||||
synced_shards = client.sync_shard_catalog(shard_catalog, schema_ids=["schema-a"])
|
||||
self.assertEqual(synced_shards[0].shard_id, "interop")
|
||||
self.assertEqual(shard_catalog.get("interop"), synced_shards[0])
|
||||
self.assertEqual(client.fetch_shard_log(shard_id="interop", shard_epoch=1, after_sequence=7)["schemaVersion"], "cultnet.shard_log_response.v0")
|
||||
|
||||
thread.join(2.0)
|
||||
|
|
@ -902,6 +918,47 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual(log["afterSequence"], 3)
|
||||
self.assertEqual(log["limit"], 2)
|
||||
|
||||
def test_cultnet_shard_catalog_applies_filters_and_responses(self) -> None:
|
||||
catalog = CultNetShardCatalog()
|
||||
primary = catalog.upsert(CultNetShardDescriptor(
|
||||
shard_id="notes",
|
||||
owner_runtime_id="python-runtime",
|
||||
epoch=2,
|
||||
is_primary=True,
|
||||
schema_ids=("schema-note",),
|
||||
key_prefix="note:",
|
||||
primary_endpoints=("cultnet://127.0.0.1:3075",),
|
||||
read_replica_endpoints=("cultnet://127.0.0.1:3075",),
|
||||
region="local",
|
||||
))
|
||||
catalog.upsert({
|
||||
"shardId": "facts",
|
||||
"ownerRuntimeId": "python-runtime",
|
||||
"epoch": 1,
|
||||
"schemaIds": ["schema-fact"],
|
||||
"keyPrefix": "fact:",
|
||||
"primaryEndpoints": ["cultnet://127.0.0.1:3076"],
|
||||
})
|
||||
|
||||
self.assertTrue(primary.serves(schema_id="schema-note", record_key="note:1"))
|
||||
self.assertFalse(primary.serves(schema_id="schema-note", record_key="fact:1"))
|
||||
self.assertEqual(catalog.list(schema_ids=["schema-note"]), [primary])
|
||||
self.assertEqual(catalog.list(record_keys=["note:1"]), [primary])
|
||||
|
||||
response = catalog.create_response(
|
||||
message_id="shards",
|
||||
schema_ids=["schema-note"],
|
||||
record_keys=["note:1"],
|
||||
)
|
||||
self.assertEqual(response["schemaVersion"], "cultnet.shard_catalog_response.v0")
|
||||
self.assertEqual(response["shards"][0]["shardId"], "notes")
|
||||
self.assertEqual(response["shards"][0]["ownerRuntimeId"], "python-runtime")
|
||||
|
||||
remote = CultNetShardCatalog()
|
||||
applied = remote.apply_response(response)
|
||||
self.assertEqual(applied[0].shard_id, "notes")
|
||||
self.assertEqual(remote.get("notes"), applied[0])
|
||||
|
||||
def test_cultnet_simulation_observation_helper_matches_schema_v0_shape(self) -> None:
|
||||
claim_hash = compute_simulation_claim_hash("frame:42", "subject:player-1", "hit")
|
||||
self.assertEqual(len(claim_hash), 64)
|
||||
|
|
@ -1708,6 +1765,8 @@ class CultCacheTests(unittest.TestCase):
|
|||
)
|
||||
snapshot_response = raw_client.fetch_snapshot(schema_ids=["mesh.server_note.v1"])
|
||||
shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.server_note.v1"])
|
||||
synced_shard_catalog = CultNetShardCatalog()
|
||||
synced_shards = raw_client.sync_shard_catalog(synced_shard_catalog, schema_ids=["mesh.server_note.v1"])
|
||||
shard_log = raw_client.fetch_shard_log(shard_id="primary", shard_epoch=1)
|
||||
discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
||||
fetched_verses = discovery_client.fetch_verses(transport_version="cultmesh.v0")
|
||||
|
|
@ -1749,6 +1808,8 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertIsNotNone(synced_schema_catalog.get("https://github.com/GameCult/cultnet-ts/contracts/cultnet.document-put-raw.schema.json"))
|
||||
self.assertEqual(snapshot_response["documents"][0]["recordKey"], "note:1")
|
||||
self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["mesh.server_note.v1"])
|
||||
self.assertEqual(synced_shards[0].shard_id, "primary")
|
||||
self.assertEqual(synced_shard_catalog.get("primary"), synced_shards[0])
|
||||
self.assertEqual(shard_log["entries"][0]["changeKind"], "added")
|
||||
self.assertEqual(fetched_verses[0].verse_id, "server-verse")
|
||||
self.assertEqual(fetched_peers[0].peer_id, "mesh-server")
|
||||
|
|
|
|||
Loading…
Reference in a new issue