Add Python CultMesh shard log responses
This commit is contained in:
parent
96fd34231a
commit
b8c0352300
3 changed files with 154 additions and 4 deletions
|
|
@ -39,6 +39,8 @@ CultCache, CultNet, and CultMesh. It is evidence, not confetti.
|
|||
old record before local or replicated application.
|
||||
It can project local envelopes into schema-v0 raw snapshot responses and apply
|
||||
those responses back through the shared CultNet replication path.
|
||||
Raw shard-scoped writes append a package-local mutation log that can be
|
||||
projected into schema-v0 shard-log responses for peer catch-up.
|
||||
- Python CultMesh committed simulation facts:
|
||||
The live interop harness sends Python a simulation observation, receives a
|
||||
consensus candidate, then requests the committed `gamecult.mesh.simulation_fact`
|
||||
|
|
@ -86,5 +88,6 @@ node --test packages\cultnet-ts\dist-test\test\interop\cultnet-interop.test.js
|
|||
package runtime, raw interop peer, local CultMesh session facade, and typed
|
||||
state participant.
|
||||
- Python does not claim the full C# shard mutation-log service surface. It can
|
||||
create/apply raw snapshots and consume shard-log responses, while the live
|
||||
interop peer owns its harness mutation log.
|
||||
create/apply raw snapshots, create/apply package-local raw shard-log
|
||||
responses, and consume remote shard-log responses, while the live interop peer
|
||||
owns its harness mutation log.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
|
@ -160,11 +161,29 @@ class CultMeshNode:
|
|||
limit=limit,
|
||||
)
|
||||
|
||||
def create_shard_log_response(
|
||||
self,
|
||||
*,
|
||||
shard_id: str,
|
||||
message_id: str = "",
|
||||
shard_epoch: int | None = None,
|
||||
after_sequence: int = 0,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return self.database.create_shard_log_response(
|
||||
shard_id=shard_id,
|
||||
message_id=message_id,
|
||||
shard_epoch=shard_epoch,
|
||||
after_sequence=after_sequence,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
class CultMeshDatabase:
|
||||
def __init__(self, node: CultMeshNode) -> None:
|
||||
self._node = node
|
||||
self._subscribers: list[tuple[DocumentDefinition[Any] | None, str | None, Callable[[CultMeshDatabaseChange], None]]] = []
|
||||
self._shard_logs: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
@property
|
||||
def cache(self) -> CultCache:
|
||||
|
|
@ -286,7 +305,7 @@ class CultMeshDatabase:
|
|||
)
|
||||
value = self.cache.put_envelope(document, envelope)
|
||||
self._publish_local_change(document, key, "added" if previous is None else "updated", value, previous)
|
||||
return document_put_raw(
|
||||
message = document_put_raw(
|
||||
message_id=message_id,
|
||||
key=key,
|
||||
schema_id=envelope.schema_id or catalog_entry.schema_id,
|
||||
|
|
@ -296,6 +315,8 @@ class CultMeshDatabase:
|
|||
shard_id=shard_id,
|
||||
shard_epoch=shard_epoch,
|
||||
)
|
||||
self._append_shard_log_put(message, "added" if previous is None else "updated")
|
||||
return message
|
||||
|
||||
def get(self, document: DocumentDefinition[Any], key: str) -> Any:
|
||||
return self.cache.get(document, key)
|
||||
|
|
@ -329,13 +350,15 @@ class CultMeshDatabase:
|
|||
) -> CultNetMessage:
|
||||
schema_id = document.catalog_entry().schema_id
|
||||
self.delete(document, key)
|
||||
return document_delete(
|
||||
message = document_delete(
|
||||
message_id=message_id,
|
||||
schema_id=schema_id,
|
||||
record_key=key,
|
||||
shard_id=shard_id,
|
||||
shard_epoch=shard_epoch,
|
||||
)
|
||||
self._append_shard_log_delete(message)
|
||||
return message
|
||||
|
||||
def snapshot(self) -> dict[str, dict[str, Any]]:
|
||||
return self.cache.snapshot()
|
||||
|
|
@ -417,6 +440,41 @@ class CultMeshDatabase:
|
|||
)
|
||||
return self.apply_shard_log_response(response)
|
||||
|
||||
def create_shard_log_response(
|
||||
self,
|
||||
*,
|
||||
shard_id: str,
|
||||
message_id: str = "",
|
||||
shard_epoch: int | None = None,
|
||||
after_sequence: int = 0,
|
||||
limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if not shard_id:
|
||||
raise ValueError("shard_id must be non-empty")
|
||||
if after_sequence < 0:
|
||||
raise ValueError("after_sequence must be non-negative")
|
||||
entries = [
|
||||
dict(entry)
|
||||
for entry in self._shard_logs.get(shard_id, [])
|
||||
if int(entry.get("sequence") or 0) > after_sequence
|
||||
]
|
||||
if limit is not None:
|
||||
if limit < 0:
|
||||
raise ValueError("limit must be non-negative")
|
||||
entries = entries[:limit]
|
||||
response: dict[str, Any] = {
|
||||
"schemaVersion": "cultnet.shard_log_response.v0",
|
||||
"messageId": message_id,
|
||||
"shardId": shard_id,
|
||||
"entries": entries,
|
||||
"resyncRequired": False,
|
||||
}
|
||||
if shard_epoch is not None:
|
||||
response["shardEpoch"] = shard_epoch
|
||||
else:
|
||||
response["shardEpoch"] = self._latest_shard_epoch(shard_id) or 0
|
||||
return response
|
||||
|
||||
def apply_snapshot_response(self, response: dict[str, Any]) -> list[CultNetAppliedRecord]:
|
||||
previous = self._previous_values_for_snapshot_response(response)
|
||||
applied = apply_cultnet_raw_snapshot(self.cache, self.documents, response)
|
||||
|
|
@ -477,6 +535,49 @@ class CultMeshDatabase:
|
|||
continue
|
||||
callback(change)
|
||||
|
||||
def _append_shard_log_put(self, message: CultNetMessage, change_kind: str) -> None:
|
||||
wire = message.to_wire()
|
||||
shard_id = wire.get("shardId")
|
||||
if not shard_id:
|
||||
return
|
||||
self._append_shard_log_entry(
|
||||
str(shard_id),
|
||||
{
|
||||
"changeKind": change_kind,
|
||||
"put": wire,
|
||||
},
|
||||
)
|
||||
|
||||
def _append_shard_log_delete(self, message: CultNetMessage) -> None:
|
||||
wire = message.to_wire()
|
||||
shard_id = wire.get("shardId")
|
||||
if not shard_id:
|
||||
return
|
||||
self._append_shard_log_entry(
|
||||
str(shard_id),
|
||||
{
|
||||
"changeKind": "removed",
|
||||
"delete": wire,
|
||||
},
|
||||
)
|
||||
|
||||
def _append_shard_log_entry(self, shard_id: str, entry: dict[str, Any]) -> None:
|
||||
log = self._shard_logs.setdefault(shard_id, [])
|
||||
entry["sequence"] = len(log) + 1
|
||||
entry["committedAt"] = datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
log.append(entry)
|
||||
|
||||
def _latest_shard_epoch(self, shard_id: str) -> int | None:
|
||||
log = self._shard_logs.get(shard_id, [])
|
||||
for entry in reversed(log):
|
||||
put = entry.get("put")
|
||||
if isinstance(put, dict) and isinstance(put.get("shardEpoch"), int):
|
||||
return put["shardEpoch"]
|
||||
delete = entry.get("delete")
|
||||
if isinstance(delete, dict) and isinstance(delete.get("shardEpoch"), int):
|
||||
return delete["shardEpoch"]
|
||||
return None
|
||||
|
||||
def _change_matches_extractor(
|
||||
self,
|
||||
change: CultMeshDatabaseChange,
|
||||
|
|
|
|||
|
|
@ -1013,6 +1013,52 @@ class CultCacheTests(unittest.TestCase):
|
|||
self.assertEqual(target.database.get_required(document, "note:1"), {"body": "include"})
|
||||
self.assertIsNone(target.database.get(document, "note:2"))
|
||||
|
||||
def test_cultmesh_database_creates_shard_log_response_from_raw_mutations(self) -> None:
|
||||
document = define_database_entry_type(
|
||||
"mesh.log_note",
|
||||
[("body", 0)],
|
||||
schema_id="mesh.log_note.v1",
|
||||
)
|
||||
source = CultMesh.create_node(runtime_id="mesh-log-source")
|
||||
source.database.register_document(document)
|
||||
source.database.put_raw_message(document, "note:1", {"body": "first"}, shard_id="notes", shard_epoch=3)
|
||||
source.database.put_raw_message(document, "note:1", {"body": "second"}, shard_id="notes", shard_epoch=3)
|
||||
source.database.put_raw_message(document, "note:2", {"body": "other"}, shard_id="other", shard_epoch=1)
|
||||
source.database.delete_raw_message(document, "note:1", shard_id="notes", shard_epoch=3)
|
||||
|
||||
response = source.database.create_shard_log_response(
|
||||
message_id="log-1",
|
||||
shard_id="notes",
|
||||
shard_epoch=3,
|
||||
after_sequence=1,
|
||||
limit=2,
|
||||
)
|
||||
|
||||
self.assertEqual(response["schemaVersion"], "cultnet.shard_log_response.v0")
|
||||
self.assertEqual(response["messageId"], "log-1")
|
||||
self.assertEqual(response["shardId"], "notes")
|
||||
self.assertEqual(response["shardEpoch"], 3)
|
||||
self.assertFalse(response["resyncRequired"])
|
||||
self.assertEqual([entry["sequence"] for entry in response["entries"]], [2, 3])
|
||||
self.assertEqual([entry["changeKind"] for entry in response["entries"]], ["updated", "removed"])
|
||||
self.assertEqual(response["entries"][0]["put"]["document"]["recordKey"], "note:1")
|
||||
self.assertEqual(response["entries"][1]["delete"]["recordKey"], "note:1")
|
||||
empty = source.database.create_shard_log_response(shard_id="missing")
|
||||
self.assertEqual(empty["shardEpoch"], 0)
|
||||
self.assertEqual(empty["entries"], [])
|
||||
with self.assertRaises(ValueError):
|
||||
source.database.create_shard_log_response(shard_id="notes", after_sequence=-1)
|
||||
with self.assertRaises(ValueError):
|
||||
source.database.create_shard_log_response(shard_id="notes", limit=-1)
|
||||
|
||||
target = CultMesh.create_node(runtime_id="mesh-log-target")
|
||||
target.database.register_document(document)
|
||||
target.database.put(document, "note:1", {"body": "first"})
|
||||
applied = target.database.apply_shard_log_response(response)
|
||||
self.assertEqual([record.change_kind for record in applied], ["updated", "removed"])
|
||||
self.assertIsNone(target.database.get(document, "note:1"))
|
||||
self.assertIsNone(target.database.get(document, "note:2"))
|
||||
|
||||
def test_cultmesh_database_watchers_observe_name_and_index_changes(self) -> None:
|
||||
document = define_database_entry_type(
|
||||
"mesh.named_watch",
|
||||
|
|
|
|||
Loading…
Reference in a new issue