3b83cb6c08
Sixteen conflicted files resolved by the spec's rule: main owns payload ownership, bounds and the Clock seam; C2a owns persistence semantics and exact binding. MintStoredAt is internal static (previous, now); the cache and the store pass their cache's Clock. Package versions are main's. The cultmesh-ts payload-parse acceptance and its test are deleted under retire-schema-alias; peer snapshot reads take the schema's own id, else an id the definition lists as compatible. The held firstRecordAtKey fallback is removed: main's handles take definitions only, so nothing could reach it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5676 lines
264 KiB
Python
5676 lines
264 KiB
Python
from __future__ import annotations
|
|
|
|
import errno
|
|
import json
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
import msgpack
|
|
from cultcache_py import (
|
|
CultCache,
|
|
SingleFileMessagePackBackingStore,
|
|
define_database_entry_type,
|
|
define_document_type,
|
|
)
|
|
from cultcache_py.interop import INTEROP_SCHEMA_VERSION, interop_note_document
|
|
from cultmesh_py.verify import verify
|
|
from cultnet_py import (
|
|
compute_simulation_claim_hash,
|
|
CultNetClientAuthorityScope,
|
|
CultNetRawClient,
|
|
CultNetRawDocumentRecord,
|
|
CultNetRawSnapshotResponse,
|
|
CultNetSchemaCatalog,
|
|
CultNetShardCatalog,
|
|
CultNetShardDescriptor,
|
|
CultNetShardLogResponse,
|
|
CultNetFileShardReplicaCursorStore,
|
|
CultNetFileShardMutationLogStore,
|
|
CultNetInMemoryShardReplicaCursorStore,
|
|
CultNetSchemaWriteForwarder,
|
|
CultNetSchemaShardLogFetcher,
|
|
CultNetSchemaShardSnapshotFetcher,
|
|
CultNetPeerError,
|
|
CultNetShardReplicator,
|
|
CultNetShardReplicatorOptions,
|
|
CultNetShardMutationLogStore,
|
|
CultNetShardWriteForwarder,
|
|
CultNetSimulationConsensusOptions,
|
|
CultNetSimulationConsensusCandidate,
|
|
CultNetSimulationObservation,
|
|
CultNetSimulationObservationHub,
|
|
CultNetRudpPacketType,
|
|
CultNetRudpSendOptions,
|
|
CultNetRudpSession,
|
|
CultNetRudpSessionOptions,
|
|
CultNetRudpSocketMode,
|
|
CultNetRudpSocketTransportConnection,
|
|
CultNetRudpSocketTransportOptions,
|
|
apply_shard_log_response,
|
|
create_rudp_schema_transport,
|
|
decode_rudp_packet,
|
|
document_delete,
|
|
document_put_raw,
|
|
encode_rudp_packet,
|
|
hello,
|
|
simulation_observation,
|
|
snapshot_request,
|
|
)
|
|
from cultnet_py.cultmesh_contracts import VERSE_CATALOG_REQUEST
|
|
from cultmesh_py import create_node
|
|
from cultmesh_py import (
|
|
CultMesh,
|
|
CultMeshDatabase,
|
|
CultMeshDatabaseChange,
|
|
CultMeshGameSessionOptions,
|
|
CultMeshNodeOptions,
|
|
CultMeshReactiveDocumentOptions,
|
|
CultMeshPeerCard,
|
|
CultMeshPeerCatalog,
|
|
CultMeshSimulationFact,
|
|
CultMeshAuthorityLease,
|
|
CultMeshAuthorityLeaseCatalog,
|
|
CultMeshHmacAuthorityLeaseVerifier,
|
|
CultMeshDiscoveryClient,
|
|
CultMeshPeerExchangeClient,
|
|
CultMeshPeerHealthMonitor,
|
|
CultMeshSimulationObservationFanout,
|
|
CultMeshSnapshotFanout,
|
|
CultMeshStreamCatalog,
|
|
CultMeshStreamConsumerProfile,
|
|
CultMeshStreamDescriptor,
|
|
CultMeshStreamFrameHandle,
|
|
CultMeshVerseCatalog,
|
|
CultMeshVerseCompatibility,
|
|
CultMeshVerseDescriptor,
|
|
CultMeshVerseDiscoveryClient,
|
|
READY_SCHEMA_VERSION,
|
|
peer_exchange_request,
|
|
simulation_fact_document,
|
|
verse_catalog_request,
|
|
)
|
|
|
|
|
|
def bind_udp_socket() -> socket.socket:
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
sock.bind(("127.0.0.1", 0))
|
|
sock.settimeout(0.02)
|
|
return sock
|
|
|
|
|
|
def pump_rudp_handshake(
|
|
client: CultNetRudpSocketTransportConnection,
|
|
server: CultNetRudpSocketTransportConnection,
|
|
) -> None:
|
|
for _ in range(20):
|
|
server.receive_once()
|
|
client.receive_once()
|
|
server.receive_once()
|
|
if client.connected and server.connected:
|
|
return
|
|
time.sleep(0.005)
|
|
raise AssertionError("RUDP socket handshake did not complete")
|
|
|
|
|
|
def receive_rudp_frame(transport: CultNetRudpSocketTransportConnection):
|
|
for _ in range(20):
|
|
frame = transport.receive_once()
|
|
if frame is not None:
|
|
return frame
|
|
time.sleep(0.005)
|
|
raise AssertionError("RUDP socket frame was not delivered")
|
|
|
|
|
|
@dataclass
|
|
class Item:
|
|
name: str
|
|
category: str
|
|
value: int
|
|
|
|
|
|
class CultMeshTests(unittest.TestCase):
|
|
def _wait_until(self, predicate: Callable[[], bool], *, timeout_seconds: float = 2.0) -> None:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.01)
|
|
self.fail("Timed out waiting for condition")
|
|
|
|
def test_verify_reports_python_runtime_surface_health(self) -> None:
|
|
result = verify(records=4)
|
|
|
|
self.assertEqual(result["status"], "ok")
|
|
self.assertEqual(
|
|
{check["name"] for check in result["checks"]},
|
|
{
|
|
"public_exports",
|
|
"typed_markers",
|
|
"local_cultmesh_wire_smoke",
|
|
"cultmesh_capability_truth",
|
|
"benchmark_sanity",
|
|
},
|
|
)
|
|
by_name = {check["name"]: check for check in result["checks"]}
|
|
self.assertEqual(by_name["public_exports"]["missing"], {})
|
|
self.assertEqual(by_name["cultmesh_capability_truth"]["failures"], [])
|
|
|
|
def test_cultmesh_client_uses_rudp_schema_transport_endpoint(self) -> None:
|
|
import msgpack # type: ignore
|
|
|
|
server_socket = bind_udp_socket()
|
|
server_host, server_port = server_socket.getsockname()[:2]
|
|
connection_id = 0x43554C54
|
|
server = CultNetRudpSocketTransportConnection(
|
|
CultNetRudpSocketTransportOptions(
|
|
runtime_id="python-rudp-schema-server",
|
|
socket=server_socket,
|
|
mode=CultNetRudpSocketMode.SERVER,
|
|
connection_id=connection_id,
|
|
initial_sequence=100,
|
|
resend_delay_ms=25,
|
|
)
|
|
)
|
|
received_messages: list[dict[str, object]] = []
|
|
|
|
def serve_once() -> None:
|
|
deadline = time.monotonic() + 2.0
|
|
try:
|
|
while time.monotonic() < deadline:
|
|
frame = server.receive_once()
|
|
server.poll_resends()
|
|
if frame is None:
|
|
time.sleep(0.005)
|
|
continue
|
|
received = msgpack.unpackb(frame.payload, raw=False)
|
|
received_messages.append(received)
|
|
response = {
|
|
"schemaVersion": "cultnet.schema_catalog_response.v0",
|
|
"messageId": received["messageId"],
|
|
"schemas": [],
|
|
}
|
|
server.send("schema", msgpack.packb(response, use_bin_type=True))
|
|
return
|
|
raise AssertionError("RUDP schema server did not receive a request")
|
|
finally:
|
|
server.close()
|
|
|
|
server_thread = threading.Thread(target=serve_once, daemon=True)
|
|
server_thread.start()
|
|
client = CultMesh.create_client(
|
|
endpoint=f"rudp://{server_host}:{server_port}",
|
|
timeout_seconds=2.0,
|
|
connection_id=connection_id,
|
|
runtime_id="python-rudp-schema-client",
|
|
)
|
|
|
|
response = client.fetch_schema_catalog(message_id="rudp-transport-factory")
|
|
server_thread.join(timeout=2.0)
|
|
|
|
self.assertFalse(server_thread.is_alive())
|
|
self.assertEqual(response["schemaVersion"], "cultnet.schema_catalog_response.v0")
|
|
self.assertEqual(response["messageId"], "rudp-transport-factory")
|
|
self.assertEqual(received_messages[0]["schemaVersion"], "cultnet.schema_catalog_request.v0")
|
|
self.assertEqual(received_messages[0]["messageId"], "rudp-transport-factory")
|
|
|
|
def _rudp_client_bound_host(self, endpoint: str, bind_host: str | None = None) -> str:
|
|
client = CultMesh.create_rudp_client("python-cultmesh-bind-client", 0x10203060, endpoint, bind_host=bind_host)
|
|
try:
|
|
return client.socket.getsockname()[0]
|
|
finally:
|
|
client.socket.close()
|
|
|
|
# A client bound to loopback cannot send to another host on Windows, so a
|
|
# client for a remote endpoint binds the unspecified address of its family.
|
|
def test_cultmesh_rudp_client_for_a_remote_endpoint_binds_the_unspecified_address(self) -> None:
|
|
self.assertEqual(self._rudp_client_bound_host("rudp://10.77.0.1:17872"), "0.0.0.0")
|
|
self.assertEqual(self._rudp_client_bound_host("rudp://[2001:db8::1]:17872"), "::")
|
|
|
|
def test_cultmesh_rudp_client_for_a_loopback_endpoint_binds_loopback(self) -> None:
|
|
self.assertEqual(self._rudp_client_bound_host("rudp://127.0.0.1:17872"), "127.0.0.1")
|
|
self.assertEqual(self._rudp_client_bound_host("rudp://[::1]:17872"), "::1")
|
|
|
|
def test_cultmesh_rudp_client_binds_an_explicit_bind_host_whatever_the_endpoint(self) -> None:
|
|
self.assertEqual(self._rudp_client_bound_host("rudp://10.77.0.1:17872", "127.0.0.1"), "127.0.0.1")
|
|
self.assertEqual(self._rudp_client_bound_host("rudp://127.0.0.1:17872", "0.0.0.0"), "0.0.0.0")
|
|
|
|
def test_cultmesh_rudp_server_binds_loopback_unless_told_otherwise(self) -> None:
|
|
loopback = CultMesh.create_rudp_server("python-cultmesh-bind-server", 0x10203061)
|
|
anywhere = CultMesh.create_rudp_server("python-cultmesh-bind-server", 0x10203062, bind_host="0.0.0.0")
|
|
try:
|
|
self.assertEqual(loopback.socket.getsockname()[0], "127.0.0.1")
|
|
self.assertEqual(anywhere.socket.getsockname()[0], "0.0.0.0")
|
|
finally:
|
|
loopback.socket.close()
|
|
anywhere.socket.close()
|
|
|
|
def test_cultmesh_facade_creates_rudp_client_from_peer_endpoint(self) -> None:
|
|
connection_id = 0x10203042
|
|
server = CultMesh.create_rudp_server(
|
|
"python-cultmesh-rudp-server",
|
|
connection_id,
|
|
initial_sequence=100,
|
|
resend_delay_ms=25,
|
|
max_fragment_bytes=1024,
|
|
max_pending_reliable_packets=16,
|
|
)
|
|
server_host, server_port = server.socket.getsockname()[:2]
|
|
endpoint = CultMesh.parse_rudp_endpoint(f"rudp://{server_host}:{server_port}")
|
|
self.assertEqual(endpoint.host, server_host)
|
|
self.assertEqual(endpoint.port, server_port)
|
|
peer = CultMeshPeerCard(
|
|
peer_id="python-cultmesh-rudp-server",
|
|
verse_id="local",
|
|
endpoints=(endpoint.uri,),
|
|
roles=("schema",),
|
|
authority_lease_id="lease:python-cultmesh-rudp-server",
|
|
)
|
|
peers = CultMesh.create_peer_catalog()
|
|
leases = CultMesh.create_authority_lease_catalog()
|
|
peers.upsert(peer)
|
|
with self.assertRaisesRegex(ValueError, "No authorized RUDP peer"):
|
|
CultMesh.create_rudp_client_for_authorized_peer(
|
|
"python-cultmesh-rudp-client",
|
|
connection_id,
|
|
peers,
|
|
leases,
|
|
"local",
|
|
"schema",
|
|
resend_delay_ms=25,
|
|
max_fragment_bytes=1024,
|
|
max_pending_reliable_packets=16,
|
|
)
|
|
leases.upsert(
|
|
CultMeshAuthorityLease(
|
|
lease_id="lease:python-cultmesh-rudp-server",
|
|
verse_id="local",
|
|
peer_id="python-cultmesh-rudp-server",
|
|
roles=("schema",),
|
|
valid_from=datetime.now(UTC) - timedelta(seconds=1),
|
|
expires_at=datetime.now(UTC) + timedelta(seconds=30),
|
|
issuer_runtime_id="python-authority",
|
|
)
|
|
)
|
|
client = CultMesh.create_rudp_client_for_authorized_peer(
|
|
"python-cultmesh-rudp-client",
|
|
connection_id,
|
|
peers,
|
|
leases,
|
|
"local",
|
|
"schema",
|
|
resend_delay_ms=25,
|
|
max_fragment_bytes=1024,
|
|
max_pending_reliable_packets=16,
|
|
)
|
|
|
|
try:
|
|
client.connect(b"join")
|
|
pump_rudp_handshake(client, server)
|
|
self.assertTrue(client.connected)
|
|
self.assertTrue(server.connected)
|
|
|
|
client.send("schema", b"client-state")
|
|
server_frame = receive_rudp_frame(server)
|
|
self.assertEqual(server_frame.channel_id, "schema")
|
|
self.assertEqual(server_frame.payload, b"client-state")
|
|
self.assertEqual(server.profile["transports"][0]["protocol"], "rudp")
|
|
self.assertEqual(client.profile["transports"][0]["protocol"], "rudp")
|
|
finally:
|
|
client.close()
|
|
server.close()
|
|
|
|
def test_cultmesh_node_applies_foreign_schema_snapshot_as_local_update(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.runtime-policy",
|
|
[
|
|
("schema_version", 0),
|
|
("name", 1),
|
|
("value", 2),
|
|
],
|
|
schema_id="mesh.runtime-policy.current",
|
|
schema_name="mesh.runtime_policy",
|
|
schema_version="mesh.runtime_policy.v1",
|
|
)
|
|
node = create_node(runtime_id="python-node")
|
|
node.register_document(document)
|
|
node.put(document, "policy:1", {
|
|
"schema_version": "mesh.runtime_policy.v1",
|
|
"name": "policy",
|
|
"value": "old",
|
|
})
|
|
observed: list[CultMeshDatabaseChange] = []
|
|
unsubscribe = node.database.watch_record(document, "policy:1", observed.append)
|
|
try:
|
|
applied = node.database.apply_snapshot_response({
|
|
"schemaVersion": "cultnet.snapshot_response_raw.v0",
|
|
"messageId": "foreign-schema-snapshot",
|
|
"documents": [
|
|
{
|
|
"schemaId": "sha256:foreign-mesh-runtime-policy",
|
|
"recordKey": "policy:1",
|
|
"storedAt": "2026-06-25T00:00:00Z",
|
|
"payloadEncoding": "messagepack",
|
|
"payload": document.encode_payload({
|
|
"schema_version": "mesh.runtime_policy.v1",
|
|
"name": "policy",
|
|
"value": "new",
|
|
}),
|
|
},
|
|
],
|
|
})
|
|
finally:
|
|
unsubscribe()
|
|
|
|
local_schema_id = document.catalog_entry().schema_id
|
|
self.assertEqual(applied[0].schema_id, local_schema_id)
|
|
self.assertEqual(observed[0].schema_id, local_schema_id)
|
|
self.assertEqual(observed[0].change_kind, "updated")
|
|
self.assertEqual(observed[0].previous_value["value"], "old")
|
|
self.assertEqual(observed[0].value["value"], "new")
|
|
self.assertEqual(node.cache.get_required_envelope(document, "policy:1").schema_id, local_schema_id)
|
|
|
|
# A record arriving under a compatible id or the schema name is the same record: one id, the local one, so an existing
|
|
# record is updated with its previous value.
|
|
def test_cultmesh_node_reports_an_update_when_a_record_arrives_under_a_compatible_id_or_the_schema_name(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.compatible-policy",
|
|
[("schema_version", 0), ("name", 1), ("value", 2)],
|
|
schema_id="mesh.compatible-policy.current",
|
|
schema_name="mesh.compatible_policy",
|
|
schema_version="mesh.compatible_policy.v1",
|
|
compatible_schema_ids=["mesh.compatible-policy.current", "mesh.compatible-policy.older"],
|
|
)
|
|
for wire_schema_id in ("mesh.compatible-policy.older", "mesh.compatible_policy"):
|
|
node = create_node(runtime_id="python-node")
|
|
node.register_document(document)
|
|
node.put(document, "policy:1", {"schema_version": "mesh.compatible_policy.v1", "name": "policy", "value": "old"})
|
|
observed: list[CultMeshDatabaseChange] = []
|
|
unsubscribe = node.database.watch_record(document, "policy:1", observed.append)
|
|
try:
|
|
applied = node.database.apply_snapshot_response({
|
|
"schemaVersion": "cultnet.snapshot_response_raw.v0",
|
|
"messageId": "compatible-snapshot",
|
|
"documents": [{
|
|
"schemaId": wire_schema_id,
|
|
"recordKey": "policy:1",
|
|
"storedAt": "2026-06-25T00:00:00Z",
|
|
"payloadEncoding": "messagepack",
|
|
"payload": document.encode_payload({
|
|
"schema_version": "mesh.compatible_policy.v1", "name": "policy", "value": "new",
|
|
}),
|
|
}],
|
|
})
|
|
finally:
|
|
unsubscribe()
|
|
|
|
local_schema_id = document.catalog_entry().schema_id
|
|
self.assertEqual(applied[0].schema_id, local_schema_id, wire_schema_id)
|
|
self.assertEqual([change.change_kind for change in observed], ["updated"], wire_schema_id)
|
|
self.assertEqual(observed[0].previous_value["value"], "old", wire_schema_id)
|
|
self.assertEqual(observed[0].value["value"], "new", wire_schema_id)
|
|
|
|
def test_cultmesh_node_syncs_snapshot_and_shard_log_through_cultnet_client(self) -> None:
|
|
import msgpack # type: ignore
|
|
from cultnet_py import read_frame, write_frame
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.sync_item",
|
|
[
|
|
("name", 0),
|
|
("category", 1),
|
|
("value", 2, 0),
|
|
],
|
|
cls=Item,
|
|
)
|
|
node = create_node(runtime_id="python-node")
|
|
node.register_document(document)
|
|
schema_id = document.catalog_entry().schema_id
|
|
ready = threading.Event()
|
|
server_error: list[BaseException] = []
|
|
port_holder: list[int] = []
|
|
|
|
def serve_requests() -> None:
|
|
try:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
|
|
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
server.bind(("127.0.0.1", 0))
|
|
port_holder.append(server.getsockname()[1])
|
|
server.listen(2)
|
|
ready.set()
|
|
for _ in range(2):
|
|
connection, _ = server.accept()
|
|
with connection:
|
|
stream = connection.makefile("rwb")
|
|
request = msgpack.unpackb(read_frame(stream), raw=False)
|
|
if request["schemaVersion"] == "cultnet.snapshot_request.v0":
|
|
response = {
|
|
"schemaVersion": "cultnet.snapshot_response_raw.v0",
|
|
"messageId": request["messageId"],
|
|
"documents": [
|
|
{
|
|
"schemaId": schema_id,
|
|
"recordKey": "item:node",
|
|
"storedAt": "2026-06-13T00:00:00Z",
|
|
"payloadEncoding": "messagepack",
|
|
"payload": document.encode_payload(Item("node", "mesh", 1)),
|
|
}
|
|
],
|
|
}
|
|
elif request["schemaVersion"] == "cultnet.shard_log_request.v0":
|
|
response = {
|
|
"schemaVersion": "cultnet.shard_log_response.v0",
|
|
"messageId": request["messageId"],
|
|
"shardId": request["shardId"],
|
|
"shardEpoch": request["shardEpoch"],
|
|
"resyncRequired": False,
|
|
"entries": [
|
|
{
|
|
"sequence": 1,
|
|
"changeKind": "updated",
|
|
"put": {
|
|
"schemaVersion": "cultnet.document_put_raw.v0",
|
|
"messageId": "put-node",
|
|
"document": {
|
|
"schemaId": schema_id,
|
|
"recordKey": "item:node",
|
|
"storedAt": "2026-06-13T00:00:01Z",
|
|
"payloadEncoding": "messagepack",
|
|
"payload": document.encode_payload(Item("node", "mesh", 9)),
|
|
},
|
|
"shardId": request["shardId"],
|
|
"shardEpoch": request["shardEpoch"],
|
|
},
|
|
}
|
|
],
|
|
}
|
|
else:
|
|
raise AssertionError(f"unexpected request {request['schemaVersion']}")
|
|
write_frame(stream, msgpack.packb(response, use_bin_type=True))
|
|
stream.flush()
|
|
except BaseException as error:
|
|
server_error.append(error)
|
|
ready.set()
|
|
|
|
thread = threading.Thread(target=serve_requests, daemon=True)
|
|
thread.start()
|
|
self.assertTrue(ready.wait(2.0))
|
|
self.assertFalse(server_error)
|
|
|
|
client = CultNetRawClient("127.0.0.1", port_holder[0], timeout_seconds=2.0)
|
|
snapshot_changes = node.sync_snapshot(client, schema_ids=[schema_id])
|
|
log_changes = node.sync_shard_log(client, shard_id="mesh", shard_epoch=1)
|
|
|
|
thread.join(2.0)
|
|
self.assertFalse(server_error)
|
|
self.assertEqual(snapshot_changes[0].record_key, "item:node")
|
|
self.assertEqual(log_changes[0].change_kind, "updated")
|
|
self.assertEqual(node.get_required(document, "item:node").value, 9)
|
|
|
|
def test_cultmesh_node_sync_document_returns_requested_alias_type(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.sync_alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.sync_alias_note.v1",
|
|
schema_name="mesh.sync_alias_note",
|
|
schema_version="mesh.sync_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.sync_alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.sync_alias_note.v1",
|
|
schema_name="mesh.sync_alias_note",
|
|
schema_version="mesh.sync_alias_note.v1",
|
|
)
|
|
node = create_node(runtime_id="python-node")
|
|
node.register_document(document)
|
|
requests: list[dict[str, Any]] = []
|
|
|
|
class SnapshotClient:
|
|
def fetch_snapshot_response(
|
|
self,
|
|
*,
|
|
schema_ids: list[str] | None = None,
|
|
record_keys: list[str] | None = None,
|
|
shard_id: str | None = None,
|
|
shard_epoch: int | None = None,
|
|
) -> dict[str, Any]:
|
|
requests.append({
|
|
"schema_ids": schema_ids,
|
|
"record_keys": record_keys,
|
|
"shard_id": shard_id,
|
|
"shard_epoch": shard_epoch,
|
|
})
|
|
return {
|
|
"schemaVersion": "cultnet.snapshot_response_raw.v0",
|
|
"messageId": "sync-document",
|
|
"documents": [
|
|
{
|
|
"schemaId": document.catalog_entry().schema_id,
|
|
"recordKey": "note:remote",
|
|
"storedAt": "2026-06-27T00:00:00Z",
|
|
"payloadEncoding": "messagepack",
|
|
"payload": document.encode_payload(CanonicalNote("synced-alias")),
|
|
}
|
|
],
|
|
}
|
|
|
|
synced = node.sync_document(SnapshotClient(), alias, "note:remote")
|
|
facade_synced = CultMesh.sync_document(node, SnapshotClient(), alias, "note:remote")
|
|
|
|
self.assertIsInstance(synced, UiNote)
|
|
self.assertEqual(synced.body, "synced-alias")
|
|
self.assertIsInstance(facade_synced, UiNote)
|
|
self.assertEqual(facade_synced.body, "synced-alias")
|
|
self.assertEqual(node.get_required(document, "note:remote").body, "synced-alias")
|
|
self.assertIsInstance(node.get_required(alias, "note:remote"), UiNote)
|
|
self.assertEqual(requests, [
|
|
{
|
|
"schema_ids": ["mesh.sync_alias_note.v1"],
|
|
"record_keys": ["note:remote"],
|
|
"shard_id": None,
|
|
"shard_epoch": None,
|
|
},
|
|
{
|
|
"schema_ids": ["mesh.sync_alias_note.v1"],
|
|
"record_keys": ["note:remote"],
|
|
"shard_id": None,
|
|
"shard_epoch": None,
|
|
},
|
|
])
|
|
|
|
def test_cultmesh_node_sync_document_from_publication_returns_requested_alias_type(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.publication_alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.publication_alias_note.v1",
|
|
schema_name="mesh.publication_alias_note",
|
|
schema_version="mesh.publication_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.publication_alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.publication_alias_note.v1",
|
|
schema_name="mesh.publication_alias_note",
|
|
schema_version="mesh.publication_alias_note.v1",
|
|
)
|
|
node = create_node(runtime_id="python-publication-node")
|
|
node.register_document(document)
|
|
requests: list[dict[str, Any]] = []
|
|
|
|
class SnapshotClient:
|
|
def fetch_snapshot_response(
|
|
self,
|
|
*,
|
|
schema_ids: list[str] | None = None,
|
|
record_keys: list[str] | None = None,
|
|
shard_id: str | None = None,
|
|
shard_epoch: int | None = None,
|
|
) -> dict[str, Any]:
|
|
requests.append({
|
|
"schema_ids": schema_ids,
|
|
"record_keys": record_keys,
|
|
"shard_id": shard_id,
|
|
"shard_epoch": shard_epoch,
|
|
})
|
|
return {
|
|
"schemaVersion": "cultnet.snapshot_response_raw.v0",
|
|
"messageId": "publication-sync-document",
|
|
"documents": [
|
|
{
|
|
"schemaId": document.catalog_entry().schema_id,
|
|
"recordKey": "note:published-peer",
|
|
"storedAt": "2026-06-27T00:00:00Z",
|
|
"payloadEncoding": "messagepack",
|
|
"payload": document.encode_payload(CanonicalNote("peer-publication-alias")),
|
|
}
|
|
],
|
|
}
|
|
|
|
source = CultMesh.publication_source_from_peer_snapshot(
|
|
SnapshotClient(),
|
|
shard_id="notes",
|
|
shard_epoch=3,
|
|
)
|
|
|
|
synced = node.sync_document_from_publication(source, alias, "note:published-peer")
|
|
facade_synced = CultMesh.sync_document_from_publication(node, source, alias, "note:published-peer")
|
|
|
|
self.assertIsInstance(synced, UiNote)
|
|
self.assertEqual(synced.body, "peer-publication-alias")
|
|
self.assertIsInstance(facade_synced, UiNote)
|
|
self.assertEqual(facade_synced.body, "peer-publication-alias")
|
|
self.assertEqual(node.get_required(document, "note:published-peer").body, "peer-publication-alias")
|
|
self.assertEqual(requests, [
|
|
{
|
|
"schema_ids": ["mesh.publication_alias_note.v1"],
|
|
"record_keys": ["note:published-peer"],
|
|
"shard_id": "notes",
|
|
"shard_epoch": 3,
|
|
},
|
|
{
|
|
"schema_ids": ["mesh.publication_alias_note.v1"],
|
|
"record_keys": ["note:published-peer"],
|
|
"shard_id": "notes",
|
|
"shard_epoch": 3,
|
|
},
|
|
])
|
|
|
|
def test_cultmesh_node_sync_document_from_single_file_publication_uses_node_aliases(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.file_publication_alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.file_publication_alias_note.v1",
|
|
schema_name="mesh.file_publication_alias_note",
|
|
schema_version="mesh.file_publication_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.file_publication_alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.file_publication_alias_note.v1",
|
|
schema_name="mesh.file_publication_alias_note",
|
|
schema_version="mesh.file_publication_alias_note.v1",
|
|
)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = Path(tmp) / "publication.ccmp"
|
|
writer = CultCache()
|
|
writer.register_document_type(document)
|
|
writer.add_generic_store(SingleFileMessagePackBackingStore(path))
|
|
writer.put(document, "note:published-file", CanonicalNote("file-publication-alias"))
|
|
|
|
node = create_node(runtime_id="python-file-publication-node")
|
|
node.register_document(document)
|
|
source = CultMesh.publication_source_from_single_file(path)
|
|
|
|
synced = CultMesh.sync_document_from_publication(node, source, alias, "note:published-file")
|
|
|
|
self.assertIsInstance(synced, UiNote)
|
|
self.assertEqual(synced.body, "file-publication-alias")
|
|
self.assertEqual(node.get_required(document, "note:published-file").body, "file-publication-alias")
|
|
self.assertIsInstance(node.get_required(alias, "note:published-file"), UiNote)
|
|
|
|
def test_cultmesh_node_sync_documents_from_publication_uses_binding_sources_and_aliases(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.catalog_publication_alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.catalog_publication_alias_note.v1",
|
|
schema_name="mesh.catalog_publication_alias_note",
|
|
schema_version="mesh.catalog_publication_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.catalog_publication_alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.catalog_publication_alias_note.v1",
|
|
schema_name="mesh.catalog_publication_alias_note",
|
|
schema_version="mesh.catalog_publication_alias_note.v1",
|
|
)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
first_path = Path(tmp) / "first.ccmp"
|
|
second_path = Path(tmp) / "second.ccmp"
|
|
first = CultCache()
|
|
first.register_document_type(document)
|
|
first.add_generic_store(SingleFileMessagePackBackingStore(first_path))
|
|
first.put(document, "note:first", CanonicalNote("first catalog publication"))
|
|
second = CultCache()
|
|
second.register_document_type(document)
|
|
second.add_generic_store(SingleFileMessagePackBackingStore(second_path))
|
|
second.put(document, "note:second", CanonicalNote("second catalog publication"))
|
|
|
|
node = create_node(runtime_id="python-catalog-publication-node")
|
|
node.register_document(document)
|
|
source = CultMesh.publication_source_from_single_file(first_path)
|
|
bindings = [
|
|
CultMesh.publication_document(alias, "note:first"),
|
|
CultMesh.publication_document(
|
|
alias,
|
|
"note:second",
|
|
source=CultMesh.publication_source_from_single_file(second_path),
|
|
),
|
|
]
|
|
|
|
synced = node.sync_documents_from_publication(source, bindings)
|
|
facade_synced = CultMesh.sync_documents_from_publication(node, source, bindings)
|
|
|
|
self.assertEqual([value.body for value in synced], [
|
|
"first catalog publication",
|
|
"second catalog publication",
|
|
])
|
|
self.assertTrue(all(isinstance(value, UiNote) for value in synced))
|
|
self.assertEqual([value.body for value in facade_synced], [
|
|
"first catalog publication",
|
|
"second catalog publication",
|
|
])
|
|
self.assertEqual(node.get_required(document, "note:first").body, "first catalog publication")
|
|
self.assertEqual(node.get_required(document, "note:second").body, "second catalog publication")
|
|
self.assertIsInstance(node.get_required(alias, "note:second"), UiNote)
|
|
|
|
def test_cultmesh_node_emits_raw_put_and_delete_messages_for_local_writes(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.emit_item",
|
|
[
|
|
("name", 0),
|
|
("category", 1),
|
|
("value", 2, 0),
|
|
],
|
|
cls=Item,
|
|
)
|
|
node = create_node(runtime_id="python-node")
|
|
node.register_document(document)
|
|
|
|
put = node.put_raw_message(
|
|
document,
|
|
"item:emit",
|
|
Item("wand", "gear", 4),
|
|
message_id="put-emit",
|
|
shard_id="mesh",
|
|
shard_epoch=1,
|
|
).to_wire()
|
|
self.assertEqual(node.get_required(document, "item:emit").name, "wand")
|
|
self.assertEqual(put["schemaVersion"], "cultnet.document_put_raw.v0")
|
|
self.assertEqual(put["messageId"], "put-emit")
|
|
self.assertEqual(put["document"]["schemaId"], document.catalog_entry().schema_id)
|
|
self.assertEqual(put["document"]["recordKey"], "item:emit")
|
|
self.assertEqual(put["document"]["sourceRuntimeId"], "python-node")
|
|
self.assertEqual(put["shardId"], "mesh")
|
|
self.assertEqual(put["shardEpoch"], 1)
|
|
|
|
delete = node.delete_raw_message(
|
|
document,
|
|
"item:emit",
|
|
message_id="delete-emit",
|
|
shard_id="mesh",
|
|
shard_epoch=1,
|
|
).to_wire()
|
|
self.assertIsNone(node.get(document, "item:emit"))
|
|
self.assertEqual(delete["schemaVersion"], "cultnet.document_delete.v0")
|
|
self.assertEqual(delete["messageId"], "delete-emit")
|
|
self.assertEqual(delete["schemaId"], document.catalog_entry().schema_id)
|
|
self.assertEqual(delete["recordKey"], "item:emit")
|
|
self.assertEqual(delete["shardId"], "mesh")
|
|
self.assertEqual(delete["shardEpoch"], 1)
|
|
|
|
def test_cultmesh_node_resolves_same_schema_aliases_for_local_state(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.alias_note.v1",
|
|
schema_name="mesh.alias_note",
|
|
schema_version="mesh.alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.alias_note.v1",
|
|
schema_name="mesh.alias_note",
|
|
schema_version="mesh.alias_note.v1",
|
|
)
|
|
node = create_node(runtime_id="python-node")
|
|
node.register_document(document)
|
|
seen: list[CultMeshDatabaseChange] = []
|
|
all_seen: list[CultMeshDatabaseChange] = []
|
|
node.database.watch(all_seen.append, document=alias)
|
|
node.database.watch_record(alias, "note:alias", seen.append)
|
|
|
|
node.put(alias, "note:alias", UiNote("from-alias-put"))
|
|
self.assertEqual(node.get_required(document, "note:alias").body, "from-alias-put")
|
|
alias_value = node.get_required(alias, "note:alias")
|
|
self.assertIsInstance(alias_value, UiNote)
|
|
self.assertEqual(alias_value.body, "from-alias-put")
|
|
alias_collection = node.get_all(alias)
|
|
self.assertEqual(len(alias_collection), 1)
|
|
self.assertIsInstance(alias_collection[0], UiNote)
|
|
self.assertEqual(alias_collection[0].body, "from-alias-put")
|
|
self.assertEqual(seen[0].document.type, alias.type)
|
|
self.assertEqual(seen[0].schema_id, "mesh.alias_note.v1")
|
|
self.assertIsInstance(seen[0].value, UiNote)
|
|
self.assertEqual(seen[0].value.body, "from-alias-put")
|
|
self.assertEqual(all_seen[0].document.type, alias.type)
|
|
self.assertIsInstance(all_seen[0].value, UiNote)
|
|
self.assertEqual(all_seen[0].value.body, "from-alias-put")
|
|
|
|
node.put(document, "note:alias", CanonicalNote("from-canonical-put"))
|
|
self.assertEqual(seen[1].document.type, alias.type)
|
|
self.assertIsInstance(seen[1].value, UiNote)
|
|
self.assertIsInstance(seen[1].previous_value, UiNote)
|
|
self.assertEqual(seen[1].value.body, "from-canonical-put")
|
|
self.assertEqual(seen[1].previous_value.body, "from-alias-put")
|
|
|
|
put = node.put_raw_message(
|
|
alias,
|
|
"note:alias",
|
|
UiNote("from-alias-raw-put"),
|
|
message_id="alias-put",
|
|
).to_wire()
|
|
self.assertEqual(put["document"]["schemaId"], "mesh.alias_note.v1")
|
|
self.assertEqual(put["document"]["recordKey"], "note:alias")
|
|
self.assertEqual(node.get_required(alias, "note:alias").body, "from-alias-raw-put")
|
|
|
|
delete = node.delete_raw_message(alias, "note:alias", message_id="alias-delete").to_wire()
|
|
self.assertEqual(delete["schemaId"], "mesh.alias_note.v1")
|
|
self.assertIsNone(node.get(alias, "note:alias"))
|
|
self.assertEqual(node.get_all(alias), [])
|
|
|
|
def test_cultmesh_reactive_document_coalesces_explicit_alias_updates(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
revision: int
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
revision: int
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.reactive_alias_note",
|
|
[("body", 0), ("revision", 1)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.reactive_alias_note.v1",
|
|
schema_name="mesh.reactive_alias_note",
|
|
schema_version="mesh.reactive_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.reactive_alias_note.ui",
|
|
[("body", 0), ("revision", 1)],
|
|
cls=UiNote,
|
|
schema_id="mesh.reactive_alias_note.v1",
|
|
schema_name="mesh.reactive_alias_note",
|
|
schema_version="mesh.reactive_alias_note.v1",
|
|
)
|
|
node = create_node(runtime_id="python-reactive")
|
|
node.register_document(document)
|
|
node.put(document, "note:reactive", CanonicalNote("initial", 1))
|
|
seen: list[CultMeshDatabaseChange] = []
|
|
node.database.watch_record(alias, "note:reactive", seen.append)
|
|
|
|
reactive = node.authoritative_writer(alias, "note:reactive").reactive(
|
|
CultMeshReactiveDocumentOptions(flush_delay_seconds=0.02)
|
|
)
|
|
try:
|
|
self.assertIsInstance(reactive.snapshot, UiNote)
|
|
reactive.update(lambda note: setattr(note, "body", "first-local-edit"))
|
|
reactive.update(lambda note: setattr(note, "body", "second-local-edit"))
|
|
reactive.update(lambda note: setattr(note, "revision", 2))
|
|
|
|
self._wait_until(lambda: node.get_required(alias, "note:reactive").body == "second-local-edit")
|
|
|
|
self.assertFalse(reactive.is_dirty)
|
|
self.assertEqual(node.get_required(document, "note:reactive").revision, 2)
|
|
self.assertEqual(seen[-1].value.body, "second-local-edit")
|
|
finally:
|
|
reactive.dispose()
|
|
|
|
def test_cultmesh_reactive_document_tracks_canonical_reconciliation_delta(self) -> None:
|
|
@dataclass
|
|
class Note:
|
|
body: str
|
|
revision: int
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.reactive_reconcile_note",
|
|
[("body", 0), ("revision", 1)],
|
|
cls=Note,
|
|
schema_id="mesh.reactive_reconcile_note.v1",
|
|
schema_name="mesh.reactive_reconcile_note",
|
|
schema_version="mesh.reactive_reconcile_note.v1",
|
|
)
|
|
node = create_node(runtime_id="python-reactive-reconcile")
|
|
node.register_document(document)
|
|
node.put(document, "note:reconcile", Note("initial", 1))
|
|
|
|
reactive = CultMesh.authoritative_writer(
|
|
node,
|
|
document,
|
|
"note:reconcile",
|
|
).reactive(CultMeshReactiveDocumentOptions(flush_delay_seconds=60))
|
|
try:
|
|
reactive.update(lambda note: setattr(note, "body", "local-prediction"))
|
|
reactive.update(lambda note: setattr(note, "revision", 2))
|
|
|
|
node.put(document, "note:reconcile", Note("authoritative", 7))
|
|
|
|
self.assertEqual(reactive.snapshot.body, "local-prediction")
|
|
self.assertIsNotNone(reactive.reconciliation)
|
|
self.assertEqual(reactive.reconciliation.canonical.body, "authoritative")
|
|
self.assertEqual(reactive.reconciliation.predicted.body, "local-prediction")
|
|
self.assertEqual(reactive.reconciliation.delta["body"], "local-prediction")
|
|
self.assertEqual(reactive.reconciliation.delta["revision"], -5)
|
|
|
|
reactive.flush()
|
|
self.assertEqual(node.get_required(document, "note:reconcile").body, "local-prediction")
|
|
self.assertIsNone(reactive.reconciliation)
|
|
finally:
|
|
reactive.dispose()
|
|
|
|
def test_cultmesh_reactive_document_is_idle_until_explicit_nested_update(self) -> None:
|
|
document = define_document_type(
|
|
"mesh.nested_loadout",
|
|
encode=lambda value: value,
|
|
decode=lambda value: value,
|
|
)
|
|
node = create_node(runtime_id="python-reactive-nested")
|
|
node.register_document(document)
|
|
node.put(
|
|
document,
|
|
"ship:nested",
|
|
{"loadout": {"weapon": {"tuning": {"damage": 10}}}},
|
|
)
|
|
|
|
with patch("cultmesh_py.node.threading.Timer") as timer:
|
|
reactive = node.authoritative_writer(document, "ship:nested").reactive()
|
|
timer.assert_not_called()
|
|
|
|
try:
|
|
borrowed = reactive.snapshot
|
|
borrowed["loadout"]["weapon"]["tuning"]["damage"] = 99
|
|
self.assertEqual(
|
|
node.get_required(document, "ship:nested")["loadout"]["weapon"]["tuning"]["damage"],
|
|
10,
|
|
)
|
|
|
|
reactive.update(
|
|
lambda draft: draft["loadout"]["weapon"]["tuning"].__setitem__("damage", 12)
|
|
)
|
|
reactive.flush()
|
|
|
|
self.assertEqual(reactive.snapshot["loadout"]["weapon"]["tuning"]["damage"], 12)
|
|
self.assertEqual(
|
|
node.get_required(document, "ship:nested")["loadout"]["weapon"]["tuning"]["damage"],
|
|
12,
|
|
)
|
|
finally:
|
|
reactive.dispose()
|
|
|
|
def test_cultmesh_reactive_scheduling_scales_with_changed_documents_only(self) -> None:
|
|
document = define_document_type(
|
|
"mesh.reactive_scale_note",
|
|
encode=lambda value: value,
|
|
decode=lambda value: value,
|
|
)
|
|
|
|
for document_count in (1, 100, 1000):
|
|
node = create_node(runtime_id=f"python-reactive-scale-{document_count}")
|
|
node.register_document(document)
|
|
for index in range(document_count):
|
|
node.put(document, f"note:{index}", {"body": "initial", "revision": 1})
|
|
|
|
scheduled_timers: list[Any] = []
|
|
|
|
class FakeScheduledCall:
|
|
def __init__(self, delay_seconds: float, callback: Callable[[], None]) -> None:
|
|
self.delay_seconds = delay_seconds
|
|
self.callback = callback
|
|
self.cancelled = False
|
|
|
|
def cancel(self) -> None:
|
|
self.cancelled = True
|
|
|
|
class FakeScheduler:
|
|
def schedule(
|
|
self,
|
|
delay_seconds: float,
|
|
callback: Callable[[], None],
|
|
) -> FakeScheduledCall:
|
|
call = FakeScheduledCall(delay_seconds, callback)
|
|
scheduled_timers.append(call)
|
|
return call
|
|
|
|
reactive_documents: list[Any] = []
|
|
with patch("cultmesh_py.node._REACTIVE_SCHEDULER", FakeScheduler()):
|
|
try:
|
|
for index in range(document_count):
|
|
reactive_documents.append(
|
|
node.authoritative_writer(document, f"note:{index}").reactive()
|
|
)
|
|
|
|
self.assertEqual(
|
|
len(scheduled_timers),
|
|
0,
|
|
f"{document_count} idle documents must schedule no timers",
|
|
)
|
|
|
|
changed_document_count = max(1, document_count // 100)
|
|
for index in range(changed_document_count):
|
|
reactive_documents[index].update(
|
|
lambda draft, changed=index: draft.update(
|
|
body=f"changed-{changed}",
|
|
revision=2,
|
|
)
|
|
)
|
|
|
|
self.assertEqual(len(scheduled_timers), changed_document_count)
|
|
self.assertEqual(
|
|
sum(reactive.is_dirty for reactive in reactive_documents),
|
|
changed_document_count,
|
|
)
|
|
finally:
|
|
for reactive in reactive_documents:
|
|
reactive.dispose()
|
|
|
|
self.assertTrue(all(timer.cancelled for timer in scheduled_timers))
|
|
|
|
def test_cultmesh_reactive_documents_share_one_lazy_scheduler_thread(self) -> None:
|
|
document = define_document_type(
|
|
"mesh.reactive_shared_scheduler_note",
|
|
encode=lambda value: value,
|
|
decode=lambda value: value,
|
|
)
|
|
node = create_node(runtime_id="python-reactive-shared-scheduler")
|
|
node.register_document(document)
|
|
reactive_documents: list[Any] = []
|
|
try:
|
|
for index in range(100):
|
|
node.put(document, f"note:{index}", {"body": "initial", "revision": 1})
|
|
reactive_documents.append(
|
|
node.authoritative_writer(document, f"note:{index}").reactive(
|
|
CultMeshReactiveDocumentOptions(flush_delay_seconds=60.0)
|
|
)
|
|
)
|
|
|
|
before = sum(
|
|
thread.name == "cultmesh-reactive-scheduler"
|
|
for thread in threading.enumerate()
|
|
)
|
|
for index, reactive in enumerate(reactive_documents):
|
|
reactive.update(
|
|
lambda draft, changed=index: draft.update(
|
|
body=f"changed-{changed}",
|
|
revision=2,
|
|
)
|
|
)
|
|
after = sum(
|
|
thread.name == "cultmesh-reactive-scheduler"
|
|
for thread in threading.enumerate()
|
|
)
|
|
|
|
self.assertLessEqual(before, 1)
|
|
self.assertEqual(after, 1)
|
|
finally:
|
|
for reactive in reactive_documents:
|
|
reactive.dispose()
|
|
|
|
def test_cultmesh_observed_document_cannot_publish_borrowed_mutation(self) -> None:
|
|
document = define_document_type(
|
|
"mesh.observed_note",
|
|
encode=lambda value: value,
|
|
decode=lambda value: value,
|
|
)
|
|
node = create_node(runtime_id="python-observed")
|
|
node.register_document(document)
|
|
node.put(document, "note:observed", {"body": "initial"})
|
|
|
|
observed = node.observe_document(document, "note:observed")
|
|
try:
|
|
observed.current["body"] = "borrowed-local-edit"
|
|
self.assertEqual(node.get_required(document, "note:observed")["body"], "initial")
|
|
|
|
node.put(document, "note:observed", {"body": "canonical"})
|
|
self.assertEqual(observed.current["body"], "canonical")
|
|
finally:
|
|
observed.dispose()
|
|
|
|
def test_cultmesh_node_uses_cultcache_store(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
document = define_database_entry_type("mesh.note", [("body", 0)])
|
|
store_path = Path(tmp) / "mesh.cc"
|
|
node = create_node(store_path, runtime_id="mesh-test")
|
|
self.assertIsInstance(node.database, CultMeshDatabase)
|
|
node.database.register_document(document)
|
|
node.database.put(document, "note:1", {"body": "hello"})
|
|
self.assertEqual(node.get_required(document, "note:1")["body"], "hello")
|
|
|
|
reopened = create_node(store_path, runtime_id="mesh-test")
|
|
reopened.database.register_document(document)
|
|
reopened.database.pull()
|
|
self.assertEqual(reopened.database.get(document, "note:1")["body"], "hello")
|
|
|
|
def test_cultmesh_node_options_attach_default_durable_shard_log_store(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
document = define_database_entry_type(
|
|
"mesh.durable_option_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.durable_option_note.v1",
|
|
)
|
|
cache_path = Path(tmp) / "world.cc"
|
|
node = CultMesh.create_node(
|
|
cache_path,
|
|
runtime_id="durable-option-primary",
|
|
options=CultMeshNodeOptions(enable_durable_shard_logs=True),
|
|
)
|
|
node.database.register_document(document)
|
|
node.database.put_raw_message(document, "note:1", {"body": "stored"}, shard_id="notes", shard_epoch=5)
|
|
|
|
log_path = Path(tmp) / "world.cultmesh" / "shard-logs"
|
|
self.assertTrue(log_path.exists())
|
|
|
|
reopened = CultMesh.start_node(
|
|
cache_path,
|
|
runtime_id="durable-option-primary",
|
|
enable_durable_shard_logs=True,
|
|
)
|
|
reopened.database.register_document(document)
|
|
response = reopened.database.build_shard_log_response(shard_id="notes", shard_epoch=5)
|
|
|
|
self.assertEqual(reopened.database.shard_ids(), ["notes"])
|
|
self.assertEqual([entry.sequence for entry in response.entries], [1])
|
|
self.assertEqual(response.entries[0].raw_document.record_key, "note:1")
|
|
|
|
def test_cultmesh_database_watchers_observe_local_changes_and_unsubscribe(self) -> None:
|
|
document = define_database_entry_type("mesh.watch", [("body", 0)])
|
|
node = CultMesh.create_node(runtime_id="mesh-watch")
|
|
node.database.register_document(document)
|
|
changes: list[CultMeshDatabaseChange] = []
|
|
all_changes: list[CultMeshDatabaseChange] = []
|
|
|
|
unsubscribe_record = node.database.watch_record(document, "note:1", changes.append)
|
|
unsubscribe_all = node.database.watch(all_changes.append, document=document)
|
|
|
|
node.database.put(document, "note:1", {"body": "hello"})
|
|
node.database.put(document, "note:2", {"body": "skip"})
|
|
node.database.put(document, "note:1", {"body": "updated"})
|
|
node.database.delete(document, "note:1")
|
|
unsubscribe_record()
|
|
node.database.put(document, "note:1", {"body": "after-unsubscribe"})
|
|
unsubscribe_all()
|
|
|
|
self.assertEqual([change.change_kind for change in changes], ["added", "updated", "removed"])
|
|
self.assertEqual([change.record_key for change in changes], ["note:1", "note:1", "note:1"])
|
|
self.assertEqual(changes[1].previous_value, {"body": "hello"})
|
|
self.assertIsNone(changes[2].value)
|
|
self.assertEqual([change.record_key for change in all_changes], ["note:1", "note:2", "note:1", "note:1", "note:1"])
|
|
|
|
def test_cultmesh_database_register_document_is_idempotent(self) -> None:
|
|
document = define_database_entry_type("mesh.idempotent", [("body", 0)])
|
|
conflicting = define_database_entry_type("mesh.idempotent", [("title", 0)])
|
|
node = CultMesh.create_node(runtime_id="mesh-idempotent")
|
|
|
|
node.database.register_document(document)
|
|
node.database.register_document(document)
|
|
node.database.put(document, "note:1", {"body": "still registered once"})
|
|
|
|
self.assertEqual(node.database.get_required(document, "note:1")["body"], "still registered once")
|
|
self.assertEqual([registered.type for registered in node.documents], ["mesh.idempotent"])
|
|
with self.assertRaises(ValueError):
|
|
node.database.register_document(conflicting)
|
|
|
|
def test_cultmesh_database_global_document_facade_uses_singleton_key_and_watchers(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.global_settings",
|
|
[("theme", 0)],
|
|
global_document=True,
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-global")
|
|
node.database.register_document(document)
|
|
changes: list[CultMeshDatabaseChange] = []
|
|
|
|
node.database.watch_global(document, changes.append)
|
|
node.database.put_global(document, {"theme": "ash"})
|
|
node.put_global(document, {"theme": "ember"})
|
|
node.delete_global(document)
|
|
|
|
self.assertIsNone(node.database.get_global(document))
|
|
self.assertEqual([change.record_key for change in changes], [node.cache.GLOBAL_KEY] * 3)
|
|
self.assertEqual([change.change_kind for change in changes], ["added", "updated", "removed"])
|
|
self.assertEqual(changes[1].previous_value, {"theme": "ash"})
|
|
self.assertEqual(changes[2].previous_value, {"theme": "ember"})
|
|
with self.assertRaises(Exception):
|
|
node.database.put(document, "settings:wrong", {"theme": "bad"})
|
|
|
|
def test_cultmesh_database_creates_raw_snapshot_response_from_envelopes(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.snapshot_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.snapshot_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="mesh-snapshot-source")
|
|
source.database.register_document(document)
|
|
source.database.put(document, "note:1", {"body": "include"})
|
|
source.database.put(document, "note:2", {"body": "skip"})
|
|
|
|
typed_response = source.database.build_snapshot_response(
|
|
message_id="snapshot-1",
|
|
schema_ids=["mesh.snapshot_note.v1"],
|
|
record_keys=["note:1"],
|
|
shard_id="notes",
|
|
shard_epoch=4,
|
|
shard_log_sequence=7,
|
|
)
|
|
response = source.database.create_snapshot_response(
|
|
message_id="snapshot-1",
|
|
schema_ids=["mesh.snapshot_note.v1"],
|
|
record_keys=["note:1"],
|
|
shard_id="notes",
|
|
shard_epoch=4,
|
|
shard_log_sequence=7,
|
|
)
|
|
|
|
self.assertIsInstance(typed_response, CultNetRawSnapshotResponse)
|
|
self.assertEqual(typed_response.message_id, "snapshot-1")
|
|
self.assertEqual(typed_response.documents[0].record_key, "note:1")
|
|
self.assertEqual(typed_response.to_wire(), response)
|
|
self.assertEqual(source.build_snapshot_response(schema_ids=["mesh.snapshot_note.v1"]).documents[0].record_key, "note:1")
|
|
self.assertEqual(response["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
|
self.assertEqual(response["messageId"], "snapshot-1")
|
|
self.assertEqual(response["shardId"], "notes")
|
|
self.assertEqual(response["shardEpoch"], 4)
|
|
self.assertEqual(response["shardLogSequence"], 7)
|
|
self.assertEqual([record["recordKey"] for record in response["documents"]], ["note:1"])
|
|
self.assertEqual(response["documents"][0]["schemaId"], "mesh.snapshot_note.v1")
|
|
self.assertEqual(response["documents"][0]["payloadEncoding"], "messagepack")
|
|
|
|
target = CultMesh.create_node(runtime_id="mesh-snapshot-target")
|
|
target.database.register_document(document)
|
|
applied = target.database.apply_snapshot_response(response)
|
|
self.assertEqual([(record.schema_id, record.record_key) for record in applied], [("mesh.snapshot_note.v1", "note:1")])
|
|
self.assertEqual(target.database.get_required(document, "note:1"), {"body": "include"})
|
|
self.assertIsNone(target.database.get(document, "note:2"))
|
|
|
|
typed_target = CultMesh.create_node(runtime_id="mesh-snapshot-typed-target")
|
|
typed_target.database.register_document(document)
|
|
typed_applied = typed_target.database.apply_snapshot_response(typed_response)
|
|
self.assertEqual([(record.schema_id, record.record_key) for record in typed_applied], [("mesh.snapshot_note.v1", "note:1")])
|
|
self.assertEqual(typed_target.database.get_required(document, "note:1"), {"body": "include"})
|
|
|
|
def test_cultmesh_database_filters_snapshot_by_logged_shard_membership(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.sharded_snapshot_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.sharded_snapshot_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="mesh-sharded-snapshot-source")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:notes", {"body": "include"}, shard_id="notes", shard_epoch=2)
|
|
source.database.put_raw_message(document, "note:other", {"body": "skip"}, shard_id="other", shard_epoch=1)
|
|
|
|
response = source.database.build_snapshot_response(
|
|
schema_ids=["mesh.sharded_snapshot_note.v1"],
|
|
shard_id="notes",
|
|
shard_epoch=2,
|
|
)
|
|
|
|
self.assertEqual(response.shard_id, "notes")
|
|
self.assertEqual(response.shard_epoch, 2)
|
|
self.assertEqual([record.record_key for record in response.documents], ["note:notes"])
|
|
|
|
def test_cultmesh_database_creates_shard_log_response_from_raw_mutations(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.log_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.log_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="mesh-log-source")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:1", {"body": "first"}, shard_id="notes", shard_epoch=3)
|
|
source.database.put_raw_message(document, "note:1", {"body": "second"}, shard_id="notes", shard_epoch=3)
|
|
source.database.put_raw_message(document, "note:2", {"body": "other"}, shard_id="other", shard_epoch=1)
|
|
source.database.delete_raw_message(document, "note:1", shard_id="notes", shard_epoch=3)
|
|
|
|
self.assertEqual(source.database.shard_ids(), ["notes", "other"])
|
|
self.assertEqual(source.database.shard_schema_ids("notes"), ["mesh.log_note.v1"])
|
|
|
|
typed_response = source.database.build_shard_log_response(
|
|
message_id="log-1",
|
|
shard_id="notes",
|
|
shard_epoch=3,
|
|
after_sequence=1,
|
|
limit=2,
|
|
)
|
|
response = source.database.create_shard_log_response(
|
|
message_id="log-1",
|
|
shard_id="notes",
|
|
shard_epoch=3,
|
|
after_sequence=1,
|
|
limit=2,
|
|
)
|
|
|
|
self.assertIsInstance(typed_response, CultNetShardLogResponse)
|
|
self.assertEqual(typed_response.message_id, "log-1")
|
|
self.assertEqual([entry.sequence for entry in typed_response.entries], [2, 3])
|
|
self.assertEqual(typed_response.to_wire(), response)
|
|
self.assertEqual([entry.sequence for entry in source.build_shard_log_response(shard_id="notes").entries], [1, 2, 3])
|
|
self.assertEqual(response["schemaVersion"], "cultnet.shard_log_response.v0")
|
|
self.assertEqual(response["messageId"], "log-1")
|
|
self.assertEqual(response["shardId"], "notes")
|
|
self.assertEqual(response["shardEpoch"], 3)
|
|
self.assertFalse(response["resyncRequired"])
|
|
self.assertEqual([entry["sequence"] for entry in response["entries"]], [2, 3])
|
|
self.assertEqual([entry["changeKind"] for entry in response["entries"]], ["updated", "removed"])
|
|
self.assertEqual(response["entries"][0]["put"]["document"]["recordKey"], "note:1")
|
|
self.assertEqual(response["entries"][1]["delete"]["recordKey"], "note:1")
|
|
empty = source.database.create_shard_log_response(shard_id="missing")
|
|
self.assertEqual(empty["shardEpoch"], 0)
|
|
self.assertEqual(empty["entries"], [])
|
|
self.assertTrue(empty["resyncRequired"])
|
|
self.assertEqual(empty["reason"], "unknown_shard")
|
|
stale = source.database.create_shard_log_response(shard_id="notes", shard_epoch=2)
|
|
self.assertEqual(stale["shardEpoch"], 3)
|
|
self.assertEqual(stale["entries"], [])
|
|
self.assertTrue(stale["resyncRequired"])
|
|
self.assertEqual(stale["reason"], "stale_epoch")
|
|
with self.assertRaises(ValueError):
|
|
source.database.create_shard_log_response(shard_id="notes", after_sequence=-1)
|
|
with self.assertRaises(ValueError):
|
|
source.database.create_shard_log_response(shard_id="notes", limit=-1)
|
|
|
|
target = CultMesh.create_node(runtime_id="mesh-log-target")
|
|
target.database.register_document(document)
|
|
target.database.put(document, "note:1", {"body": "first"})
|
|
applied = target.database.apply_shard_log_response(response)
|
|
self.assertEqual([record.change_kind for record in applied], ["updated", "removed"])
|
|
self.assertIsNone(target.database.get(document, "note:1"))
|
|
self.assertIsNone(target.database.get(document, "note:2"))
|
|
|
|
typed_target = CultMesh.create_node(runtime_id="mesh-log-typed-target")
|
|
typed_target.database.register_document(document)
|
|
typed_target.database.put(document, "note:1", {"body": "first"})
|
|
typed_applied = typed_target.database.apply_shard_log_response(typed_response)
|
|
self.assertEqual([record.change_kind for record in typed_applied], ["updated", "removed"])
|
|
self.assertIsNone(typed_target.database.get(document, "note:1"))
|
|
|
|
def test_cultnet_file_shard_mutation_log_store_persists_authoritative_entries(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.durable_log_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.durable_log_note.v1",
|
|
)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
store: CultNetShardMutationLogStore = CultNetFileShardMutationLogStore(Path(tmp) / "logs")
|
|
source = CultMesh.create_node(runtime_id="durable-primary")
|
|
source.database.register_document(document)
|
|
source.database.use_shard_mutation_log_store(store)
|
|
source.database.put_raw_message(document, "note:1", {"body": "first"}, shard_id="durable", shard_epoch=4)
|
|
source.database.put_raw_message(document, "note:2", {"body": "second"}, shard_id="durable", shard_epoch=4)
|
|
|
|
reopened = CultMesh.create_node(runtime_id="durable-primary")
|
|
reopened.database.register_document(document)
|
|
reopened.database.use_shard_mutation_log_store(CultNetFileShardMutationLogStore(Path(tmp) / "logs"))
|
|
response = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4)
|
|
self.assertEqual(reopened.database.shard_ids(), ["durable"])
|
|
self.assertEqual(reopened.database.shard_schema_ids("durable"), ["mesh.durable_log_note.v1"])
|
|
self.assertEqual([entry.sequence for entry in response.entries], [1, 2])
|
|
self.assertEqual(response.entries[0].raw_document.record_key, "note:1")
|
|
|
|
store.compact_through("durable", 1)
|
|
compacted = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4, after_sequence=0)
|
|
retained = reopened.database.build_shard_log_response(shard_id="durable", shard_epoch=4, after_sequence=1)
|
|
self.assertTrue(compacted.resync_required)
|
|
self.assertEqual(compacted.reason, "compacted_history")
|
|
self.assertEqual(compacted.compacted_through, 1)
|
|
self.assertEqual([entry.sequence for entry in retained.entries], [2])
|
|
|
|
def test_cultnet_shard_replicator_pulls_logs_and_resyncs_from_snapshot(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.replica_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.replica_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="primary")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:1", {"body": "one"}, shard_id="notes", shard_epoch=3)
|
|
source.database.put_raw_message(document, "note:2", {"body": "two"}, shard_id="notes", shard_epoch=3)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
cursor_path = Path(tmp) / "replica-cursors.msgpack"
|
|
server = CultMesh.serve_node(source)
|
|
try:
|
|
shard = CultNetShardDescriptor(
|
|
shard_id="notes",
|
|
owner_runtime_id="primary",
|
|
epoch=3,
|
|
is_primary=False,
|
|
schema_ids=("mesh.replica_note.v1",),
|
|
primary_endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
)
|
|
target = CultMesh.create_node(runtime_id="replica")
|
|
target.database.register_document(document)
|
|
cursor_store = CultNetFileShardReplicaCursorStore(cursor_path)
|
|
replicator = CultNetShardReplicator(
|
|
target.database,
|
|
CultNetShardReplicatorOptions(
|
|
fetcher=CultNetSchemaShardLogFetcher(timeout_seconds=2.0),
|
|
snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(timeout_seconds=2.0),
|
|
cursor_store=cursor_store,
|
|
batch_size=16,
|
|
),
|
|
)
|
|
|
|
first_sequence = replicator.pull_once(shard)
|
|
restarted_replicator = CultNetShardReplicator(
|
|
target.database,
|
|
CultNetShardReplicatorOptions(
|
|
fetcher=CultNetSchemaShardLogFetcher(timeout_seconds=2.0),
|
|
snapshot_fetcher=CultNetSchemaShardSnapshotFetcher(timeout_seconds=2.0),
|
|
cursor_store=CultNetFileShardReplicaCursorStore(cursor_path),
|
|
batch_size=16,
|
|
),
|
|
)
|
|
second_sequence = restarted_replicator.pull_once(shard)
|
|
|
|
self.assertEqual(first_sequence, 2)
|
|
self.assertEqual(second_sequence, 2)
|
|
self.assertEqual(target.get_required(document, "note:1")["body"], "one")
|
|
self.assertEqual(target.get_required(document, "note:2")["body"], "two")
|
|
cursor = CultNetFileShardReplicaCursorStore(cursor_path).read("notes")
|
|
self.assertIsNotNone(cursor)
|
|
assert cursor is not None
|
|
self.assertEqual(cursor.last_applied_sequence, 2)
|
|
self.assertEqual(cursor.shard_epoch, 3)
|
|
finally:
|
|
server.stop()
|
|
|
|
resync_target = CultMesh.create_node(runtime_id="resync-replica")
|
|
resync_target.database.register_document(document)
|
|
snapshot = source.database.build_snapshot_response(shard_id="notes", shard_epoch=3, shard_log_sequence=2)
|
|
|
|
class CompactingFetcher:
|
|
def fetch(self, shard: CultNetShardDescriptor, *, after_sequence: int, limit: int | None = None) -> CultNetShardLogResponse:
|
|
return CultNetShardLogResponse(
|
|
message_id="compact",
|
|
shard_id=shard.shard_id,
|
|
shard_epoch=shard.epoch,
|
|
entries=(),
|
|
resync_required=True,
|
|
reason="compacted_history",
|
|
compacted_through=after_sequence,
|
|
)
|
|
|
|
class SnapshotFetcher:
|
|
def fetch(self, shard: CultNetShardDescriptor) -> CultNetRawSnapshotResponse:
|
|
return snapshot
|
|
|
|
resync_replicator = CultNetShardReplicator(
|
|
resync_target.database,
|
|
CultNetShardReplicatorOptions(
|
|
fetcher=CompactingFetcher(),
|
|
snapshot_fetcher=SnapshotFetcher(),
|
|
cursor_store=CultNetInMemoryShardReplicaCursorStore(),
|
|
),
|
|
)
|
|
resync_sequence = resync_replicator.pull_once(CultNetShardDescriptor(
|
|
shard_id="notes",
|
|
owner_runtime_id="primary",
|
|
epoch=3,
|
|
is_primary=False,
|
|
schema_ids=("mesh.replica_note.v1",),
|
|
primary_endpoints=("cultnet://primary:3075",),
|
|
))
|
|
|
|
self.assertEqual(resync_sequence, 2)
|
|
self.assertEqual(resync_target.get_required(document, "note:1")["body"], "one")
|
|
|
|
def test_cultnet_shard_replicator_background_loop_polls_non_primary_shards(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.background_replica_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.background_replica_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="background-primary")
|
|
source.database.register_document(document)
|
|
target = CultMesh.create_node(runtime_id="background-replica")
|
|
target.database.register_document(document)
|
|
cursor_store = CultNetInMemoryShardReplicaCursorStore()
|
|
errors: list[Exception] = []
|
|
|
|
class FlakySourceFetcher:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def fetch(self, shard: CultNetShardDescriptor, *, after_sequence: int, limit: int | None = None) -> CultNetShardLogResponse:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
raise RuntimeError("transient fetch failure")
|
|
return source.database.build_shard_log_response(
|
|
shard_id=shard.shard_id,
|
|
shard_epoch=shard.epoch,
|
|
after_sequence=after_sequence,
|
|
limit=limit,
|
|
)
|
|
|
|
fetcher = FlakySourceFetcher()
|
|
shard = CultNetShardDescriptor(
|
|
shard_id="background",
|
|
owner_runtime_id="background-primary",
|
|
epoch=7,
|
|
is_primary=False,
|
|
schema_ids=("mesh.background_replica_note.v1",),
|
|
primary_endpoints=("cultnet://127.0.0.1:3075",),
|
|
)
|
|
replicator = CultNetShardReplicator(
|
|
target.database,
|
|
CultNetShardReplicatorOptions(
|
|
fetcher=fetcher,
|
|
cursor_store=cursor_store,
|
|
batch_size=1,
|
|
poll_interval_seconds=0.01,
|
|
on_error=errors.append,
|
|
),
|
|
)
|
|
try:
|
|
source.database.put_raw_message(document, "note:1", {"body": "one"}, shard_id="background", shard_epoch=7)
|
|
replicator.start([shard])
|
|
self._wait_until(lambda: target.database.get(document, "note:1") is not None)
|
|
source.database.put_raw_message(document, "note:2", {"body": "two"}, shard_id="background", shard_epoch=7)
|
|
self._wait_until(lambda: target.database.get(document, "note:2") is not None)
|
|
finally:
|
|
replicator.stop()
|
|
|
|
cursor = cursor_store.read("background")
|
|
self.assertEqual(target.get_required(document, "note:1")["body"], "one")
|
|
self.assertEqual(target.get_required(document, "note:2")["body"], "two")
|
|
self.assertIsNotNone(cursor)
|
|
assert cursor is not None
|
|
self.assertEqual(cursor.last_applied_sequence, 2)
|
|
self.assertEqual(cursor.shard_epoch, 7)
|
|
self.assertEqual(len(errors), 1)
|
|
self.assertEqual(str(errors[0]), "transient fetch failure")
|
|
|
|
def test_cultnet_schema_write_forwarder_sends_raw_writes_to_primary(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.forward_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.forward_note.v1",
|
|
)
|
|
primary = CultMesh.create_node(runtime_id="forward-primary")
|
|
primary.database.register_document(document)
|
|
server = CultMesh.serve_node(primary)
|
|
try:
|
|
shard = CultNetShardDescriptor(
|
|
shard_id="forward",
|
|
owner_runtime_id="forward-primary",
|
|
epoch=9,
|
|
is_primary=False,
|
|
schema_ids=("mesh.forward_note.v1",),
|
|
primary_endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
)
|
|
forwarder: CultNetShardWriteForwarder = CultNetSchemaWriteForwarder(timeout_seconds=2.0)
|
|
put = document_put_raw(
|
|
message_id="forward-put",
|
|
key="note:forward",
|
|
schema_id=document.catalog_entry().schema_id,
|
|
stored_at="2026-06-14T00:00:00Z",
|
|
payload=document.encode_payload({"body": "forwarded"}),
|
|
).to_wire()
|
|
delete = document_delete(
|
|
message_id="forward-delete",
|
|
schema_id=document.catalog_entry().schema_id,
|
|
record_key="note:forward",
|
|
).to_wire()
|
|
|
|
forwarder.forward_put(shard, put)
|
|
self._eventually(lambda: primary.database.get(document, "note:forward") is not None)
|
|
self.assertEqual(primary.database.get_required(document, "note:forward")["body"], "forwarded")
|
|
forwarder.forward_delete(shard, delete)
|
|
self._eventually(lambda: primary.database.get(document, "note:forward") is None)
|
|
|
|
log = primary.database.build_shard_log_response(shard_id="forward", shard_epoch=9)
|
|
self.assertIsNone(primary.database.get(document, "note:forward"))
|
|
self.assertEqual([entry.change_kind for entry in log.entries], ["added", "removed"])
|
|
self.assertEqual(log.entries[0].put["shardId"], "forward")
|
|
self.assertEqual(log.entries[1].delete["shardEpoch"], 9)
|
|
finally:
|
|
server.stop()
|
|
|
|
def _eventually(self, predicate: Callable[[], bool], *, timeout_seconds: float = 2.0) -> None:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.01)
|
|
self.assertTrue(predicate())
|
|
|
|
def test_cultmesh_database_watchers_observe_name_and_index_changes(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.named_watch",
|
|
[("name", 0), ("kind", 1)],
|
|
name="name",
|
|
indexes={"kind": "kind"},
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-named-watch")
|
|
node.database.register_document(document)
|
|
name_changes: list[CultMeshDatabaseChange] = []
|
|
index_changes: list[CultMeshDatabaseChange] = []
|
|
|
|
node.database.watch_by_name(document, "Potion", name_changes.append)
|
|
node.database.watch_by_index(document, "kind", "consumable", index_changes.append)
|
|
|
|
node.database.put(document, "item:1", {"name": "Potion", "kind": "consumable"})
|
|
node.database.put(document, "item:2", {"name": "Sword", "kind": "weapon"})
|
|
node.database.put(document, "item:1", {"name": "Elixir", "kind": "rare"})
|
|
node.database.delete(document, "item:1")
|
|
|
|
self.assertEqual([change.change_kind for change in name_changes], ["added", "updated"])
|
|
self.assertEqual([change.record_key for change in name_changes], ["item:1", "item:1"])
|
|
self.assertEqual(name_changes[1].previous_value, {"name": "Potion", "kind": "consumable"})
|
|
self.assertEqual([change.change_kind for change in index_changes], ["added", "updated"])
|
|
self.assertEqual([change.record_key for change in index_changes], ["item:1", "item:1"])
|
|
self.assertEqual(index_changes[1].value, {"name": "Elixir", "kind": "rare"})
|
|
|
|
def test_cultmesh_database_collection_reads_resolve_same_schema_aliases(self) -> None:
|
|
@dataclass
|
|
class CanonicalItem:
|
|
name: str
|
|
kind: str
|
|
|
|
@dataclass
|
|
class UiItem:
|
|
name: str
|
|
kind: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.alias_collection_item",
|
|
[("name", 0), ("kind", 1)],
|
|
cls=CanonicalItem,
|
|
name="name",
|
|
indexes={"kind": "kind"},
|
|
schema_id="mesh.alias_collection_item.v1",
|
|
schema_name="mesh.alias_collection_item",
|
|
schema_version="mesh.alias_collection_item.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.alias_collection_item.ui",
|
|
[("name", 0), ("kind", 1)],
|
|
cls=UiItem,
|
|
name="name",
|
|
indexes={"kind": "kind"},
|
|
schema_id="mesh.alias_collection_item.v1",
|
|
schema_name="mesh.alias_collection_item",
|
|
schema_version="mesh.alias_collection_item.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-alias-collection")
|
|
node.database.register_document(document)
|
|
|
|
node.database.put(alias, "item:potion", UiItem("Potion", "consumable"))
|
|
node.database.put(document, "item:sword", CanonicalItem("Sword", "weapon"))
|
|
|
|
all_items = node.database.get_all(alias)
|
|
potion = node.database.get_by_name(alias, "Potion")
|
|
sword = node.get_by_index(alias, "kind", "weapon")
|
|
|
|
self.assertEqual([item.name for item in all_items], ["Potion", "Sword"])
|
|
self.assertTrue(all(isinstance(item, UiItem) for item in all_items))
|
|
self.assertIsInstance(potion, UiItem)
|
|
self.assertEqual(potion.kind, "consumable")
|
|
self.assertIsInstance(sword, UiItem)
|
|
self.assertEqual(sword.name, "Sword")
|
|
self.assertIsNone(node.database.get_by_name(alias, "Missing"))
|
|
|
|
def test_cultmesh_database_alias_collection_watches_emit_alias_values(self) -> None:
|
|
@dataclass
|
|
class CanonicalItem:
|
|
name: str
|
|
kind: str
|
|
|
|
@dataclass
|
|
class UiItem:
|
|
name: str
|
|
kind: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.alias_collection_watch_item",
|
|
[("name", 0), ("kind", 1)],
|
|
cls=CanonicalItem,
|
|
name="name",
|
|
indexes={"kind": "kind"},
|
|
schema_id="mesh.alias_collection_watch_item.v1",
|
|
schema_name="mesh.alias_collection_watch_item",
|
|
schema_version="mesh.alias_collection_watch_item.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.alias_collection_watch_item.ui",
|
|
[("name", 0), ("kind", 1)],
|
|
cls=UiItem,
|
|
name="name",
|
|
indexes={"kind": "kind"},
|
|
schema_id="mesh.alias_collection_watch_item.v1",
|
|
schema_name="mesh.alias_collection_watch_item",
|
|
schema_version="mesh.alias_collection_watch_item.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-alias-collection-watch")
|
|
node.database.register_document(document)
|
|
name_changes: list[CultMeshDatabaseChange] = []
|
|
index_changes: list[CultMeshDatabaseChange] = []
|
|
|
|
node.database.watch_by_name(alias, "Potion", name_changes.append)
|
|
node.database.watch_by_index(alias, "kind", "consumable", index_changes.append)
|
|
|
|
node.database.put(document, "item:potion", CanonicalItem("Potion", "consumable"))
|
|
node.database.put(document, "item:potion", CanonicalItem("Elixir", "rare"))
|
|
|
|
self.assertEqual([change.change_kind for change in name_changes], ["added", "updated"])
|
|
self.assertTrue(all(change.document.type == alias.type for change in name_changes))
|
|
self.assertIsInstance(name_changes[0].value, UiItem)
|
|
self.assertIsInstance(name_changes[1].value, UiItem)
|
|
self.assertIsInstance(name_changes[1].previous_value, UiItem)
|
|
self.assertEqual(name_changes[1].previous_value.name, "Potion")
|
|
self.assertEqual(name_changes[1].value.name, "Elixir")
|
|
self.assertEqual([change.change_kind for change in index_changes], ["added", "updated"])
|
|
self.assertTrue(all(change.document.type == alias.type for change in index_changes))
|
|
self.assertIsInstance(index_changes[0].value, UiItem)
|
|
self.assertIsInstance(index_changes[1].previous_value, UiItem)
|
|
|
|
def test_cultmesh_database_watch_by_name_and_index_validate_lookup_shape(self) -> None:
|
|
document = define_database_entry_type("mesh.unnamed_watch", [("body", 0)])
|
|
node = CultMesh.create_node(runtime_id="mesh-watch-validation")
|
|
node.database.register_document(document)
|
|
|
|
with self.assertRaises(ValueError):
|
|
node.database.watch_by_name(document, "missing", lambda _: None)
|
|
with self.assertRaises(ValueError):
|
|
node.database.watch_by_index(document, "missing", "value", lambda _: None)
|
|
with self.assertRaises(ValueError):
|
|
node.database.watch_by_index(document, "", "value", lambda _: None)
|
|
with self.assertRaises(ValueError):
|
|
node.database.watch_by_index(document, "missing", "", lambda _: None)
|
|
|
|
def test_cultmesh_facade_matches_peer_runtime_entrypoints(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
store_path = Path(tmp) / "mesh.cc"
|
|
node = CultMesh.start_node(store_path, runtime_id="mesh-facade")
|
|
|
|
self.assertEqual(node.runtime_id, "mesh-facade")
|
|
self.assertIsInstance(CultMesh.create_node(), type(node))
|
|
self.assertIsInstance(CultMesh.create_verse_catalog(), CultMeshVerseCatalog)
|
|
self.assertIsInstance(CultMesh.create_peer_catalog(), CultMeshPeerCatalog)
|
|
self.assertIsInstance(CultMesh.create_authority_lease_catalog(), CultMeshAuthorityLeaseCatalog)
|
|
self.assertIsInstance(CultMesh.create_stream_catalog(), CultMeshStreamCatalog)
|
|
self.assertIsInstance(CultMesh.create_schema_catalog(), CultNetSchemaCatalog)
|
|
self.assertIsInstance(CultMesh.create_builtin_schema_catalog(), CultNetSchemaCatalog)
|
|
self.assertIsInstance(CultMesh.create_shard_catalog(), CultNetShardCatalog)
|
|
builtins = CultMesh.create_builtin_schema_catalog(
|
|
include_schema_json=False,
|
|
kinds=["shared_contract"],
|
|
)
|
|
self.assertEqual(
|
|
builtins.list()[0].schema_id,
|
|
"https://github.com/GameCult/CultLib/contracts/cultnet/cultnet.transport-profile.schema.json",
|
|
)
|
|
self.assertIsNone(builtins.list()[0].schema_json)
|
|
peer_exchange_schema_id = (
|
|
"https://github.com/GameCult/CultLib/contracts/cultnet/cultmesh.peer-exchange-request.schema.json"
|
|
)
|
|
filtered_builtin = CultMesh.create_builtin_schema_catalog(
|
|
include_schema_json=True,
|
|
schema_ids=[peer_exchange_schema_id],
|
|
)
|
|
self.assertEqual(
|
|
filtered_builtin.list()[0].schema_version,
|
|
"cultmesh.peer_exchange_request.v0",
|
|
)
|
|
self.assertIn(
|
|
"cultmesh.peer_exchange_request.v0",
|
|
filtered_builtin.list()[0].schema_json or "",
|
|
)
|
|
shard_catalog = CultMesh.create_shard_catalog()
|
|
shard_catalog.upsert(
|
|
CultNetShardDescriptor(
|
|
shard_id="python-notes-a",
|
|
owner_runtime_id="mesh-facade",
|
|
epoch=7,
|
|
is_primary=True,
|
|
schema_ids=("python.note.v1",),
|
|
key_prefix="note:",
|
|
)
|
|
)
|
|
self.assertEqual(
|
|
[
|
|
shard.shard_id
|
|
for shard in shard_catalog.list(schema_ids=["python.note.v1"], record_keys=["note:1"])
|
|
],
|
|
["python-notes-a"],
|
|
)
|
|
verse_client = CultMesh.create_verse_discovery_client("127.0.0.1", 4010, timeout_seconds=1.5)
|
|
peer_client = CultMesh.create_peer_exchange_client("127.0.0.1", 4011, timeout_seconds=1.5)
|
|
raw_client = CultMesh.create_client("127.0.0.1", 4012, timeout_seconds=1.5)
|
|
connected_client = CultMesh.connect_client("127.0.0.1", 4013, timeout_seconds=1.5)
|
|
self.assertIsInstance(verse_client, CultMeshVerseDiscoveryClient)
|
|
self.assertIsInstance(peer_client, CultMeshPeerExchangeClient)
|
|
self.assertIsInstance(verse_client, CultMeshDiscoveryClient)
|
|
self.assertIsInstance(peer_client, CultMeshDiscoveryClient)
|
|
self.assertIsInstance(raw_client, CultNetRawClient)
|
|
self.assertIsInstance(connected_client, CultNetRawClient)
|
|
self.assertEqual((verse_client.host, verse_client.port, verse_client.timeout_seconds), ("127.0.0.1", 4010, 1.5))
|
|
self.assertEqual((peer_client.host, peer_client.port, peer_client.timeout_seconds), ("127.0.0.1", 4011, 1.5))
|
|
self.assertEqual((raw_client.host, raw_client.port, raw_client.timeout_seconds), ("127.0.0.1", 4012, 1.5))
|
|
self.assertEqual((connected_client.host, connected_client.port, connected_client.timeout_seconds), ("127.0.0.1", 4013, 1.5))
|
|
|
|
def test_cultmesh_simulation_fact_uses_csharp_slot_contract(self) -> None:
|
|
import msgpack # type: ignore
|
|
|
|
candidate = {
|
|
"shardId": "arena",
|
|
"shardEpoch": 4,
|
|
"frame": 100,
|
|
"subjectId": "bob",
|
|
"claimKind": "hit",
|
|
"claimHash": compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"),
|
|
"claimSummary": "alice shot bob first",
|
|
"witnessCount": 2,
|
|
"supportWeight": 2.0,
|
|
"totalWeight": 2.0,
|
|
"confidence": 1.0,
|
|
"hasQuorum": True,
|
|
}
|
|
|
|
fact = CultMeshSimulationFact.from_candidate(candidate, committed_at="2026-06-13T00:00:00Z")
|
|
payload = simulation_fact_document.encode_payload(fact)
|
|
slots = msgpack.unpackb(payload, raw=False)
|
|
|
|
self.assertEqual(CultMeshSimulationFact.create_record_key(candidate), f"simulation:{fact.fact_id}")
|
|
self.assertEqual(slots[0], fact.fact_id)
|
|
self.assertEqual(slots[1], "arena")
|
|
self.assertEqual(slots[2], 4)
|
|
self.assertEqual(slots[3], 100)
|
|
self.assertEqual(slots[4], "bob")
|
|
self.assertEqual(slots[5], "hit")
|
|
self.assertEqual(slots[6], candidate["claimHash"])
|
|
self.assertEqual(slots[7], "alice shot bob first")
|
|
self.assertEqual(slots[8], 2)
|
|
self.assertEqual(slots[9], 2.0)
|
|
self.assertEqual(slots[10], 2.0)
|
|
self.assertEqual(slots[11], 1.0)
|
|
self.assertEqual(slots[12], "2026-06-13T00:00:00Z")
|
|
|
|
def test_cultmesh_simulation_fact_committer_rejects_without_quorum_and_stores_fact(self) -> None:
|
|
candidate = {
|
|
"shardId": "arena",
|
|
"shardEpoch": 4,
|
|
"frame": 100,
|
|
"subjectId": "bob",
|
|
"claimKind": "hit",
|
|
"claimHash": compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"),
|
|
"claimSummary": "alice shot bob first",
|
|
"witnessCount": 2,
|
|
"supportWeight": 2.0,
|
|
"totalWeight": 2.0,
|
|
"confidence": 1.0,
|
|
"hasQuorum": True,
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
node = CultMesh.create_node(Path(tmp) / "facts.cc", runtime_id="mesh-facts")
|
|
committer = CultMesh.create_simulation_fact_committer(node)
|
|
|
|
rejected = dict(candidate)
|
|
rejected["hasQuorum"] = False
|
|
with self.assertRaisesRegex(ValueError, "before quorum"):
|
|
committer.commit(rejected)
|
|
|
|
committed = committer.commit(candidate, committed_at="2026-06-13T00:00:00Z")
|
|
stored = node.get_required(simulation_fact_document, committed.key)
|
|
|
|
self.assertEqual(stored.claim_hash, candidate["claimHash"])
|
|
self.assertEqual(stored.committed_at, "2026-06-13T00:00:00Z")
|
|
self.assertEqual(committed.fact.fact_id, stored.fact_id)
|
|
|
|
def test_cultmesh_game_session_submits_observations_and_commits_quorum_once(self) -> None:
|
|
claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100")
|
|
first = simulation_observation(
|
|
message_id="obs-1",
|
|
witness_runtime_id="watcher-1",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
).to_wire()
|
|
second = simulation_observation(
|
|
message_id="obs-2",
|
|
witness_runtime_id="watcher-2",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
).to_wire()
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
node = CultMesh.create_node(Path(tmp) / "session.cc", runtime_id="mesh-session")
|
|
session = CultMesh.create_game_session(
|
|
node,
|
|
CultMeshGameSessionOptions(
|
|
consensus_options=CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0)
|
|
),
|
|
)
|
|
seen_candidates: list[CultNetSimulationConsensusCandidate] = []
|
|
seen_facts: list[CultMeshDatabaseChange] = []
|
|
unsubscribe_candidates = session.watch_candidates(seen_candidates.append)
|
|
unsubscribe_facts = session.watch_simulation_facts(seen_facts.append)
|
|
|
|
self.assertEqual(session.submit_and_commit(CultNetSimulationObservation.from_wire(first)), [])
|
|
typed_candidates = session.submit_observation_candidates(CultNetSimulationObservation.from_wire(second))
|
|
commits = session.commit_quorum_candidate_objects(typed_candidates)
|
|
replay = session.commit_quorum_candidates(session.submit_observation(second))
|
|
unsubscribe_candidates()
|
|
unsubscribe_facts()
|
|
|
|
self.assertEqual(len(typed_candidates), 1)
|
|
self.assertTrue(typed_candidates[0].has_quorum)
|
|
self.assertEqual(len(commits), 1)
|
|
self.assertEqual(replay, [])
|
|
self.assertEqual([candidate.claim_hash for candidate in seen_candidates], [claim_hash, claim_hash, claim_hash])
|
|
self.assertEqual(len(seen_facts), 1)
|
|
self.assertEqual(seen_facts[0].change_kind, "added")
|
|
self.assertEqual(seen_facts[0].value.claim_hash, claim_hash)
|
|
stored = node.get_required(simulation_fact_document, commits[0].key)
|
|
self.assertEqual(stored.claim_hash, claim_hash)
|
|
self.assertEqual(stored.witness_count, 2)
|
|
|
|
def test_cultmesh_game_session_serves_catalogs_and_observations_over_wire(self) -> None:
|
|
verse_catalog = CultMeshVerseCatalog()
|
|
peer_catalog = CultMeshPeerCatalog()
|
|
verse_catalog.upsert(CultMeshVerseDescriptor(
|
|
verse_id="session-verse",
|
|
display_name="Session Verse",
|
|
authority_model="local",
|
|
compatibility=CultMeshVerseCompatibility("cultmesh.v0", "session-rules"),
|
|
))
|
|
peer_catalog.upsert(CultMeshPeerCard(
|
|
peer_id="session-peer",
|
|
verse_id="session-verse",
|
|
endpoints=("cultnet://127.0.0.1:0",),
|
|
roles=("simulation-observer", "read-replica"),
|
|
shard_ids=("arena",),
|
|
))
|
|
session = CultMesh.create_game_session(
|
|
CultMesh.create_node(runtime_id="session-peer"),
|
|
CultMeshGameSessionOptions(
|
|
verse_catalog=verse_catalog,
|
|
peer_catalog=peer_catalog,
|
|
consensus_options=CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0),
|
|
),
|
|
)
|
|
server = session.serve(display_name="Session Peer")
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
hello_response = raw_client.request(
|
|
hello(runtime_id="session-prober"),
|
|
expected_schema_version="cultnet.hello.v0",
|
|
)
|
|
discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
verses = discovery_client.fetch_verses(transport_version="cultmesh.v0")
|
|
peers = discovery_client.fetch_peers(verse_id="session-verse", roles=["simulation-observer"])
|
|
claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100")
|
|
first = raw_client.request(
|
|
simulation_observation(
|
|
message_id="session-wire-1",
|
|
witness_runtime_id="watcher-1",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
),
|
|
expected_schema_version="cultnet.simulation_consensus_candidate.v0",
|
|
)
|
|
second = raw_client.request(
|
|
simulation_observation(
|
|
message_id="session-wire-2",
|
|
witness_runtime_id="watcher-2",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
),
|
|
expected_schema_version="cultnet.simulation_consensus_candidate.v0",
|
|
)
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"])
|
|
self.assertEqual(verses[0].verse_id, "session-verse")
|
|
self.assertEqual(peers[0].peer_id, "session-peer")
|
|
self.assertIn("simulation-observer", peers[0].roles)
|
|
self.assertFalse(first["hasQuorum"])
|
|
self.assertEqual(first["witnessCount"], 1)
|
|
self.assertTrue(second["hasQuorum"])
|
|
self.assertEqual(second["witnessCount"], 2)
|
|
|
|
def test_cultmesh_game_session_can_disable_served_simulation_observations(self) -> None:
|
|
verse_catalog = CultMeshVerseCatalog()
|
|
verse_catalog.upsert(CultMeshVerseDescriptor(
|
|
verse_id="session-quiet",
|
|
display_name="Quiet Session",
|
|
authority_model="local",
|
|
compatibility=CultMeshVerseCompatibility("cultmesh.v0", "quiet-rules"),
|
|
))
|
|
session = CultMesh.create_game_session(
|
|
CultMesh.create_node(runtime_id="quiet-session-peer"),
|
|
CultMeshGameSessionOptions(
|
|
verse_catalog=verse_catalog,
|
|
serve_simulation_observations=False,
|
|
),
|
|
)
|
|
server = session.serve(display_name="Quiet Session Peer")
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
hello_response = raw_client.request(
|
|
hello(runtime_id="quiet-session-prober"),
|
|
expected_schema_version="cultnet.hello.v0",
|
|
)
|
|
discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
verses = discovery_client.fetch_verses(transport_version="cultmesh.v0")
|
|
with self.assertRaisesRegex(CultNetPeerError, "Simulation observations are not enabled") as raised:
|
|
raw_client.request(
|
|
simulation_observation(
|
|
message_id="quiet-session-sim",
|
|
witness_runtime_id="watcher-1",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=compute_simulation_claim_hash("hit", "alice", "bob", "frame:100"),
|
|
),
|
|
expected_schema_version="cultnet.simulation_consensus_candidate.v0",
|
|
)
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertNotIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"])
|
|
self.assertEqual(verses[0].verse_id, "session-quiet")
|
|
self.assertEqual(raised.exception.response["code"], "simulation_observations_disabled")
|
|
|
|
def test_cultmesh_game_session_prediction_requires_scope_and_reconciles_shard_log(self) -> None:
|
|
note_doc = define_database_entry_type(
|
|
"mesh.input",
|
|
[("body", 0)],
|
|
schema_id="mesh.input.v1",
|
|
)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
node = CultMesh.create_node(Path(tmp) / "prediction.cc", runtime_id="client-a")
|
|
node.register_document(note_doc)
|
|
session = CultMesh.create_game_session(
|
|
node,
|
|
CultMeshGameSessionOptions(
|
|
client_authority_scopes=(
|
|
CultNetClientAuthorityScope(
|
|
"client-a",
|
|
schema_ids=("mesh.input.v1",),
|
|
key_prefix="input:client-a",
|
|
),
|
|
)
|
|
),
|
|
)
|
|
|
|
with self.assertRaisesRegex(ValueError, "does not have client prediction authority"):
|
|
session.predict(note_doc, "input:other", {"body": "nope"})
|
|
|
|
prediction = session.predict(note_doc, "input:client-a:move", {"body": "predicted"})
|
|
put = document_put_raw(
|
|
message_id="authoritative",
|
|
key=prediction.key,
|
|
schema_id=prediction.schema_id,
|
|
stored_at="2026-06-13T00:00:00Z",
|
|
payload=note_doc.encode_payload({"body": "authoritative"}),
|
|
shard_id="inputs",
|
|
shard_epoch=1,
|
|
)
|
|
authoritative_response = {
|
|
"schemaVersion": "cultnet.shard_log_response.v0",
|
|
"messageId": "inputs-log",
|
|
"shardId": "inputs",
|
|
"shardEpoch": 1,
|
|
"entries": [
|
|
{
|
|
"sequence": 1,
|
|
"committedAt": "2026-06-13T00:00:00Z",
|
|
"changeKind": "updated",
|
|
"put": put.to_wire(),
|
|
}
|
|
],
|
|
"resyncRequired": False,
|
|
}
|
|
changes = session.apply_shard_log_response(CultNetShardLogResponse.from_wire(authoritative_response))
|
|
|
|
self.assertEqual(changes[0].change_kind, "reconciled")
|
|
self.assertEqual(node.get_required(note_doc, prediction.key)["body"], "authoritative")
|
|
self.assertEqual(session.pending_predictions(), ())
|
|
self.assertEqual(session.resimulation_inputs(), ())
|
|
|
|
def test_cultmesh_game_session_partial_reconciliation_keeps_unmatched_predictions(self) -> None:
|
|
note_doc = define_database_entry_type(
|
|
"mesh.partial_input",
|
|
[("body", 0)],
|
|
schema_id="mesh.partial_input.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="client-partial")
|
|
node.register_document(note_doc)
|
|
session = CultMesh.create_game_session(
|
|
node,
|
|
CultMeshGameSessionOptions(
|
|
client_authority_scopes=(
|
|
CultNetClientAuthorityScope(
|
|
"client-partial",
|
|
schema_ids=("mesh.partial_input.v1",),
|
|
key_prefix="input:client-partial",
|
|
),
|
|
)
|
|
),
|
|
)
|
|
pending = session.predict(note_doc, "input:client-partial:pending", {"body": "pending-local"})
|
|
accepted = session.predict(note_doc, "input:client-partial:accepted", {"body": "accepted-local"})
|
|
put = document_put_raw(
|
|
message_id="authoritative-partial",
|
|
key=accepted.key,
|
|
schema_id=accepted.schema_id,
|
|
stored_at="2026-06-13T00:00:00Z",
|
|
payload=note_doc.encode_payload({"body": "accepted-authority"}),
|
|
shard_id="inputs",
|
|
shard_epoch=1,
|
|
)
|
|
|
|
changes = session.apply_shard_log_response({
|
|
"schemaVersion": "cultnet.shard_log_response.v0",
|
|
"messageId": "inputs-partial-log",
|
|
"shardId": "inputs",
|
|
"shardEpoch": 1,
|
|
"entries": [
|
|
{
|
|
"sequence": 1,
|
|
"committedAt": "2026-06-13T00:00:00Z",
|
|
"changeKind": "updated",
|
|
"put": put.to_wire(),
|
|
}
|
|
],
|
|
"resyncRequired": False,
|
|
})
|
|
|
|
self.assertEqual([(change.change_kind, change.record_key) for change in changes], [
|
|
("reconciled", accepted.key),
|
|
])
|
|
self.assertEqual(node.get_required(note_doc, accepted.key)["body"], "accepted-authority")
|
|
self.assertEqual(node.get_required(note_doc, pending.key)["body"], "pending-local")
|
|
self.assertEqual([prediction.key for prediction in session.pending_predictions()], [pending.key])
|
|
self.assertEqual([prediction.key for prediction in session.resimulation_inputs()], [pending.key])
|
|
|
|
def test_cultmesh_game_session_rolls_back_pending_predictions(self) -> None:
|
|
note_doc = define_database_entry_type(
|
|
"mesh.rollback_input",
|
|
[("body", 0)],
|
|
schema_id="mesh.rollback_input.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="client-rollback")
|
|
node.register_document(note_doc)
|
|
node.put(note_doc, "input:client-rollback:existing", {"body": "previous"})
|
|
session = CultMesh.create_game_session(
|
|
node,
|
|
CultMeshGameSessionOptions(
|
|
client_authority_scopes=(
|
|
CultNetClientAuthorityScope(
|
|
"client-rollback",
|
|
schema_ids=("mesh.rollback_input.v1",),
|
|
key_prefix="input:client-rollback",
|
|
),
|
|
)
|
|
),
|
|
)
|
|
|
|
existing = session.predict(note_doc, "input:client-rollback:existing", {"body": "predicted-existing"})
|
|
created = session.predict(note_doc, "input:client-rollback:new", {"body": "predicted-new"})
|
|
self.assertEqual(
|
|
[prediction.key for prediction in session.pending_predictions()],
|
|
["input:client-rollback:existing", "input:client-rollback:new"],
|
|
)
|
|
self.assertEqual([prediction.key for prediction in session.resimulation_inputs()], [
|
|
"input:client-rollback:existing",
|
|
"input:client-rollback:new",
|
|
])
|
|
|
|
restored = session.rollback_prediction(existing)
|
|
self.assertIsNotNone(restored)
|
|
assert restored is not None
|
|
self.assertEqual(restored.change_kind, "rolled_back")
|
|
self.assertEqual(restored.value, {"body": "previous"})
|
|
self.assertEqual(node.get_required(note_doc, "input:client-rollback:existing")["body"], "previous")
|
|
|
|
rolled_back = session.rollback_predictions()
|
|
self.assertEqual([(change.change_kind, change.record_key, change.value) for change in rolled_back], [
|
|
("rolled_back", created.key, None),
|
|
])
|
|
self.assertIsNone(node.get(note_doc, created.key))
|
|
self.assertEqual(session.pending_predictions(), ())
|
|
|
|
def test_cultmesh_game_session_authoritative_delete_rolls_back_prediction(self) -> None:
|
|
note_doc = define_database_entry_type(
|
|
"mesh.deleted_input",
|
|
[("body", 0)],
|
|
schema_id="mesh.deleted_input.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="client-delete")
|
|
node.register_document(note_doc)
|
|
session = CultMesh.create_game_session(
|
|
node,
|
|
CultMeshGameSessionOptions(
|
|
client_authority_scopes=(
|
|
CultNetClientAuthorityScope(
|
|
"client-delete",
|
|
schema_ids=("mesh.deleted_input.v1",),
|
|
key_prefix="input:client-delete",
|
|
),
|
|
)
|
|
),
|
|
)
|
|
|
|
prediction = session.predict(note_doc, "input:client-delete:move", {"body": "predicted"})
|
|
delete = document_delete(
|
|
message_id="authoritative-delete",
|
|
schema_id=prediction.schema_id,
|
|
record_key=prediction.key,
|
|
shard_id="inputs",
|
|
shard_epoch=1,
|
|
)
|
|
changes = session.apply_shard_log_response({
|
|
"schemaVersion": "cultnet.shard_log_response.v0",
|
|
"messageId": "inputs-delete-log",
|
|
"shardId": "inputs",
|
|
"shardEpoch": 1,
|
|
"entries": [
|
|
{
|
|
"sequence": 1,
|
|
"committedAt": "2026-06-13T00:00:00Z",
|
|
"changeKind": "removed",
|
|
"delete": delete.to_wire(),
|
|
}
|
|
],
|
|
"resyncRequired": False,
|
|
})
|
|
|
|
self.assertEqual(changes[0].change_kind, "rolled_back")
|
|
self.assertIsNone(node.get(note_doc, prediction.key))
|
|
self.assertEqual(session.pending_predictions(), ())
|
|
|
|
def test_cultmesh_database_watchers_observe_authoritative_shard_log_reconciliation(self) -> None:
|
|
note_doc = define_database_entry_type(
|
|
"mesh.watched_input",
|
|
[("body", 0)],
|
|
schema_id="mesh.watched_input.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="client-watch")
|
|
node.database.register_document(note_doc)
|
|
seen: list[CultMeshDatabaseChange] = []
|
|
node.database.watch_record(note_doc, "input:client-watch:move", seen.append)
|
|
session = CultMesh.create_game_session(
|
|
node,
|
|
CultMeshGameSessionOptions(
|
|
client_authority_scopes=(
|
|
CultNetClientAuthorityScope(
|
|
"client-watch",
|
|
schema_ids=("mesh.watched_input.v1",),
|
|
key_prefix="input:client-watch",
|
|
),
|
|
)
|
|
),
|
|
)
|
|
|
|
prediction = session.predict(note_doc, "input:client-watch:move", {"body": "predicted"})
|
|
put = document_put_raw(
|
|
message_id="authoritative-watch",
|
|
key=prediction.key,
|
|
schema_id=prediction.schema_id,
|
|
stored_at="2026-06-13T00:00:00Z",
|
|
payload=note_doc.encode_payload({"body": "authoritative"}),
|
|
shard_id="inputs",
|
|
shard_epoch=1,
|
|
)
|
|
session_changes = session.apply_shard_log_response({
|
|
"schemaVersion": "cultnet.shard_log_response.v0",
|
|
"messageId": "inputs-watch-log",
|
|
"shardId": "inputs",
|
|
"shardEpoch": 1,
|
|
"entries": [
|
|
{
|
|
"sequence": 1,
|
|
"committedAt": "2026-06-13T00:00:00Z",
|
|
"changeKind": "updated",
|
|
"put": put.to_wire(),
|
|
}
|
|
],
|
|
"resyncRequired": False,
|
|
})
|
|
|
|
self.assertEqual([change.change_kind for change in seen], ["added", "updated"])
|
|
self.assertEqual(seen[1].value, {"body": "authoritative"})
|
|
self.assertEqual(seen[1].previous_value, {"body": "predicted"})
|
|
self.assertEqual(session_changes[0].change_kind, "reconciled")
|
|
|
|
def test_cultmesh_verse_catalog_response_matches_schema_v0_wire_shape(self) -> None:
|
|
import msgpack # type: ignore
|
|
|
|
catalog = CultMeshVerseCatalog()
|
|
catalog.upsert(
|
|
CultMeshVerseDescriptor(
|
|
verse_id="aetheria-main",
|
|
display_name="Aetheria",
|
|
authority_model="OperatorCluster",
|
|
compatibility=CultMeshVerseCompatibility(
|
|
transport_version="cultmesh.v0",
|
|
rules_hash="rules",
|
|
compatible_verse_ids=("aetheria-modded",),
|
|
required_plugin_ids=("core",),
|
|
optional_plugin_ids=("skylands",),
|
|
),
|
|
discovery_endpoints=("cultmesh://aetheria.example.test:3075",),
|
|
authority_runtime_ids=("runtime-a",),
|
|
description="main branch",
|
|
)
|
|
)
|
|
|
|
response = catalog.create_response(verse_catalog_request("verses-1"))
|
|
decoded = msgpack.unpackb(msgpack.packb(response, use_bin_type=True), raw=False)
|
|
self.assertEqual(decoded["schemaVersion"], "cultmesh.verse_catalog_response.v0")
|
|
self.assertEqual(decoded["messageId"], "verses-1")
|
|
self.assertEqual(decoded["verses"][0]["verseId"], "aetheria-main")
|
|
self.assertEqual(decoded["verses"][0]["compatibility"]["requiredPluginIds"], ["core"])
|
|
|
|
def test_cultmesh_verse_catalog_watches_updates_and_finds_transfer_targets(self) -> None:
|
|
source = CultMeshVerseDescriptor(
|
|
verse_id="aetheria-main",
|
|
display_name="Aetheria",
|
|
authority_model="federated",
|
|
compatibility=CultMeshVerseCompatibility(
|
|
transport_version="cultmesh.v0",
|
|
rules_hash="rules-main",
|
|
),
|
|
)
|
|
compatible = CultMeshVerseDescriptor(
|
|
verse_id="aetheria-modded",
|
|
display_name="Aetheria Modded",
|
|
authority_model="federated",
|
|
compatibility=CultMeshVerseCompatibility(
|
|
transport_version="cultmesh.v0",
|
|
rules_hash="rules-modded",
|
|
compatible_verse_ids=("aetheria-main",),
|
|
),
|
|
)
|
|
incompatible = CultMeshVerseDescriptor(
|
|
verse_id="old-world",
|
|
display_name="Old World",
|
|
authority_model="solo",
|
|
compatibility=CultMeshVerseCompatibility(
|
|
transport_version="cultmesh.v0",
|
|
rules_hash="rules-old",
|
|
),
|
|
)
|
|
catalog = CultMeshVerseCatalog()
|
|
seen: list[str] = []
|
|
unsubscribe = catalog.watch(lambda verse: seen.append(verse.verse_id))
|
|
|
|
catalog.upsert(source)
|
|
catalog.apply_response({
|
|
"schemaVersion": "cultmesh.verse_catalog_response.v0",
|
|
"messageId": "verses-watch",
|
|
"verses": [compatible.to_wire(), incompatible.to_wire()],
|
|
})
|
|
unsubscribe()
|
|
catalog.upsert(CultMeshVerseDescriptor(
|
|
verse_id="after-unsubscribe",
|
|
display_name="After",
|
|
authority_model="none",
|
|
compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules-after"),
|
|
))
|
|
|
|
self.assertEqual(seen, ["aetheria-main", "aetheria-modded", "old-world"])
|
|
self.assertEqual([verse.verse_id for verse in catalog.verses], ["aetheria-main", "aetheria-modded", "after-unsubscribe", "old-world"])
|
|
self.assertEqual(catalog.get("aetheria-modded"), compatible)
|
|
self.assertEqual([verse.verse_id for verse in catalog.find_transfer_targets(source)], ["aetheria-modded"])
|
|
|
|
def test_cultmesh_peer_exchange_response_matches_schema_v0_wire_shape(self) -> None:
|
|
import msgpack # type: ignore
|
|
|
|
catalog = CultMeshPeerCatalog()
|
|
catalog.upsert(
|
|
CultMeshPeerCard(
|
|
peer_id="peer-a",
|
|
verse_id="aetheria-main",
|
|
endpoints=("cultnet://peer-a.example.test:3075",),
|
|
roles=("discovery", "read-replica"),
|
|
shard_ids=("players",),
|
|
region="eu-west",
|
|
authority_lease_id="lease-1",
|
|
expires_at="2026-05-20T12:00:00.0000000Z",
|
|
signature="sig",
|
|
)
|
|
)
|
|
|
|
response = catalog.create_response(
|
|
peer_exchange_request("pex-1", verse_id="aetheria-main", roles=["read-replica"])
|
|
)
|
|
decoded = msgpack.unpackb(msgpack.packb(response, use_bin_type=True), raw=False)
|
|
self.assertEqual(decoded["schemaVersion"], "cultmesh.peer_exchange_response.v0")
|
|
self.assertEqual(decoded["messageId"], "pex-1")
|
|
self.assertEqual(decoded["peers"][0]["peerId"], "peer-a")
|
|
self.assertIn("read-replica", decoded["peers"][0]["roles"])
|
|
self.assertEqual(decoded["peers"][0]["authorityLeaseId"], "lease-1")
|
|
|
|
def test_cultmesh_peer_catalog_watches_updates_and_gets_peers(self) -> None:
|
|
first = CultMeshPeerCard(
|
|
peer_id="peer-a",
|
|
verse_id="aetheria-main",
|
|
endpoints=("cultnet://peer-a.example.test:3075",),
|
|
roles=("discovery", "read-replica"),
|
|
)
|
|
second = CultMeshPeerCard(
|
|
peer_id="peer-b",
|
|
verse_id="aetheria-main",
|
|
endpoints=("cultnet://peer-b.example.test:3075",),
|
|
roles=("shard-primary",),
|
|
)
|
|
catalog = CultMeshPeerCatalog()
|
|
seen: list[str] = []
|
|
unsubscribe = catalog.watch(lambda peer: seen.append(peer.peer_id))
|
|
|
|
catalog.upsert(first)
|
|
catalog.apply_response({
|
|
"schemaVersion": "cultmesh.peer_exchange_response.v0",
|
|
"messageId": "peers-watch",
|
|
"peers": [second.to_wire()],
|
|
})
|
|
unsubscribe()
|
|
catalog.upsert(CultMeshPeerCard(
|
|
peer_id="peer-c",
|
|
verse_id="aetheria-main",
|
|
endpoints=("cultnet://peer-c.example.test:3075",),
|
|
roles=("read-replica",),
|
|
))
|
|
|
|
self.assertEqual(seen, ["peer-a", "peer-b"])
|
|
self.assertTrue(first.has_role("read-replica"))
|
|
self.assertEqual([peer.peer_id for peer in catalog.peers], ["peer-a", "peer-b", "peer-c"])
|
|
self.assertEqual(catalog.get("peer-b"), second)
|
|
self.assertEqual([peer.peer_id for peer in catalog.find("aetheria-main", role="read-replica")], ["peer-a", "peer-c"])
|
|
|
|
def test_cultmesh_discovery_client_fetches_typed_catalogs_over_cultnet_frames(self) -> None:
|
|
import msgpack # type: ignore
|
|
from cultnet_py import read_frame, write_frame
|
|
|
|
verses = CultMeshVerseCatalog()
|
|
verses.upsert(
|
|
CultMeshVerseDescriptor(
|
|
verse_id="aetheria-main",
|
|
display_name="Aetheria",
|
|
authority_model="federated",
|
|
compatibility=CultMeshVerseCompatibility(
|
|
transport_version="cultmesh.v0",
|
|
rules_hash="rules-1",
|
|
required_plugin_ids=("core",),
|
|
),
|
|
discovery_endpoints=("cultmesh://aetheria.example.test:3075",),
|
|
authority_runtime_ids=("runtime-a",),
|
|
)
|
|
)
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(
|
|
CultMeshPeerCard(
|
|
peer_id="peer-a",
|
|
verse_id="aetheria-main",
|
|
endpoints=("cultnet://peer-a.example.test:3075",),
|
|
roles=("read-replica",),
|
|
shard_ids=("players",),
|
|
)
|
|
)
|
|
|
|
ready = threading.Event()
|
|
server_error: list[BaseException] = []
|
|
|
|
def serve_requests() -> None:
|
|
try:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server:
|
|
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
server.bind(("127.0.0.1", 0))
|
|
port_holder.append(server.getsockname()[1])
|
|
server.listen(8)
|
|
ready.set()
|
|
for _ in range(8):
|
|
connection, _ = server.accept()
|
|
with connection:
|
|
stream = connection.makefile("rwb")
|
|
request = msgpack.unpackb(read_frame(stream), raw=False)
|
|
if request["schemaVersion"] == "cultmesh.verse_catalog_request.v0":
|
|
response = verses.create_response(request)
|
|
elif request["schemaVersion"] == "cultmesh.peer_exchange_request.v0":
|
|
response = peers.create_response(request)
|
|
else:
|
|
raise AssertionError(f"unexpected request {request['schemaVersion']}")
|
|
write_frame(stream, msgpack.packb(response, use_bin_type=True))
|
|
stream.flush()
|
|
except BaseException as error:
|
|
server_error.append(error)
|
|
ready.set()
|
|
|
|
port_holder: list[int] = []
|
|
thread = threading.Thread(target=serve_requests, daemon=True)
|
|
thread.start()
|
|
self.assertTrue(ready.wait(2.0))
|
|
self.assertFalse(server_error)
|
|
|
|
client = CultMeshDiscoveryClient("127.0.0.1", port_holder[0], timeout_seconds=2.0)
|
|
fetched_verses = client.fetch_verses(transport_version="cultmesh.v0")
|
|
fetched_peers = client.fetch_peers(verse_id="aetheria-main", roles=["read-replica"])
|
|
local_verses = CultMeshVerseCatalog()
|
|
local_peers = CultMeshPeerCatalog()
|
|
synced_verses = client.sync_verse_catalog(local_verses, transport_version="cultmesh.v0")
|
|
synced_peers = client.sync_peer_catalog(local_peers, verse_id="aetheria-main", roles=["read-replica"])
|
|
discovered_verses = CultMeshVerseCatalog()
|
|
discovered_peers = CultMeshPeerCatalog()
|
|
endpoint = f"cultnet://127.0.0.1:{port_holder[0]}"
|
|
verse_discovery = CultMeshVerseDiscoveryClient.from_endpoint(endpoint, timeout_seconds=2.0)
|
|
peer_discovery = CultMeshPeerExchangeClient.from_endpoint(endpoint, timeout_seconds=2.0)
|
|
discovered_verse_count = verse_discovery.discover(
|
|
discovered_verses,
|
|
endpoints=[endpoint, endpoint, ""],
|
|
transport_version="cultmesh.v0",
|
|
)
|
|
discovered_peer_count = peer_discovery.discover(
|
|
discovered_peers,
|
|
verse_id="aetheria-main",
|
|
endpoints=[endpoint, endpoint, ""],
|
|
roles=["read-replica"],
|
|
)
|
|
fanout_verses = CultMeshVerseCatalog()
|
|
fanout_verses.upsert(
|
|
CultMeshVerseDescriptor(
|
|
verse_id="aetheria-main",
|
|
display_name="Known Aetheria",
|
|
authority_model="federated",
|
|
compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules-1"),
|
|
discovery_endpoints=(endpoint, endpoint, ""),
|
|
)
|
|
)
|
|
fanout_peers = CultMeshPeerCatalog()
|
|
fanout_peers.upsert(
|
|
CultMeshPeerCard(
|
|
peer_id="known-peer",
|
|
verse_id="aetheria-main",
|
|
endpoints=(endpoint, endpoint, ""),
|
|
roles=("read-replica",),
|
|
)
|
|
)
|
|
fanout_verse_count = verse_discovery.fanout(fanout_verses, transport_version="cultmesh.v0")
|
|
fanout_peer_count = peer_discovery.fanout(
|
|
fanout_peers,
|
|
verse_id="aetheria-main",
|
|
roles=["read-replica"],
|
|
)
|
|
|
|
thread.join(2.0)
|
|
self.assertFalse(server_error)
|
|
self.assertEqual(fetched_verses[0].verse_id, "aetheria-main")
|
|
self.assertEqual(fetched_verses[0].compatibility.required_plugin_ids, ("core",))
|
|
self.assertEqual(fetched_peers[0].peer_id, "peer-a")
|
|
self.assertEqual(fetched_peers[0].shard_ids, ("players",))
|
|
self.assertEqual(synced_verses[0].verse_id, "aetheria-main")
|
|
self.assertEqual(local_peers.find("aetheria-main", role="read-replica")[0].peer_id, "peer-a")
|
|
self.assertEqual(synced_peers[0].roles, ("read-replica",))
|
|
self.assertEqual(discovered_verse_count, 1)
|
|
self.assertEqual(discovered_peer_count, 1)
|
|
self.assertEqual(discovered_verses.get("aetheria-main").display_name, "Aetheria")
|
|
self.assertEqual(discovered_peers.find("aetheria-main", role="read-replica")[0].peer_id, "peer-a")
|
|
self.assertEqual(fanout_verse_count, 1)
|
|
self.assertEqual(fanout_peer_count, 1)
|
|
self.assertEqual(fanout_verses.get("aetheria-main").display_name, "Aetheria")
|
|
self.assertIn(
|
|
"peer-a",
|
|
{peer.peer_id for peer in fanout_peers.find("aetheria-main", role="read-replica")},
|
|
)
|
|
|
|
def test_cultmesh_discovery_client_fans_out_simulation_observations_to_peers(self) -> None:
|
|
first_hub = CultNetSimulationObservationHub(
|
|
CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0)
|
|
)
|
|
second_hub = CultNetSimulationObservationHub(
|
|
CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0)
|
|
)
|
|
first_server = CultMesh.serve_node(
|
|
CultMesh.create_node(runtime_id="sim-peer-a"),
|
|
observation_hub=first_hub,
|
|
)
|
|
second_server = CultMesh.serve_node(
|
|
CultMesh.create_node(runtime_id="sim-peer-b"),
|
|
observation_hub=second_hub,
|
|
)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="sim-peer-a",
|
|
verse_id="arena",
|
|
endpoints=(f"cultnet://127.0.0.1:{first_server.port}",),
|
|
roles=("simulation-witness",),
|
|
))
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="sim-peer-b",
|
|
verse_id="arena",
|
|
endpoints=(f"cultnet://127.0.0.1:{second_server.port}",),
|
|
roles=("simulation-witness",),
|
|
))
|
|
claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:200")
|
|
message = simulation_observation(
|
|
message_id="fanout-observation",
|
|
witness_runtime_id="python-fanout",
|
|
shard_id="arena",
|
|
shard_epoch=1,
|
|
frame=200,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice hit bob",
|
|
).to_wire()
|
|
|
|
client = CultMeshDiscoveryClient("127.0.0.1", first_server.port, timeout_seconds=2.0)
|
|
candidates = client.fanout_simulation_observation(
|
|
peers,
|
|
message,
|
|
verse_id="arena",
|
|
roles=["simulation-witness"],
|
|
)
|
|
finally:
|
|
first_server.stop()
|
|
second_server.stop()
|
|
|
|
self.assertEqual([candidate["schemaVersion"] for candidate in candidates], [
|
|
"cultnet.simulation_consensus_candidate.v0",
|
|
"cultnet.simulation_consensus_candidate.v0",
|
|
])
|
|
self.assertEqual({candidate["claimHash"] for candidate in candidates}, {claim_hash})
|
|
self.assertEqual({candidate["messageId"] for candidate in candidates}, {"fanout-observation"})
|
|
self.assertEqual({candidate["hasQuorum"] for candidate in candidates}, {True})
|
|
|
|
def test_cultmesh_simulation_observation_fanout_flushes_queued_messages(self) -> None:
|
|
hub = CultNetSimulationObservationHub(
|
|
CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0)
|
|
)
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="sim-flush-peer"), observation_hub=hub)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="sim-flush-peer",
|
|
verse_id="arena",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("simulation-witness",),
|
|
))
|
|
claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:210")
|
|
seen_candidates: list[dict[str, object]] = []
|
|
fanout = CultMeshSimulationObservationFanout(
|
|
CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0),
|
|
peers,
|
|
verse_id="arena",
|
|
roles=["simulation-witness"],
|
|
on_candidate=seen_candidates.append,
|
|
)
|
|
fanout.enqueue(simulation_observation(
|
|
message_id="queued-observation",
|
|
witness_runtime_id="python-fanout",
|
|
shard_id="arena",
|
|
shard_epoch=1,
|
|
frame=210,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice hit bob",
|
|
).to_wire())
|
|
|
|
candidates = fanout.flush()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(fanout.pending_count(), 0)
|
|
self.assertEqual([candidate["claimHash"] for candidate in candidates], [claim_hash])
|
|
self.assertEqual(seen_candidates[0]["messageId"], "queued-observation")
|
|
|
|
def test_cultmesh_simulation_observation_fanout_keeps_failed_messages_pending(self) -> None:
|
|
closed_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
closed_socket.bind(("127.0.0.1", 0))
|
|
closed_port = closed_socket.getsockname()[1]
|
|
closed_socket.close()
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="closed-simulation-peer",
|
|
verse_id="arena",
|
|
endpoints=(f"cultnet://127.0.0.1:{closed_port}",),
|
|
roles=("simulation-witness",),
|
|
))
|
|
errors: list[tuple[str, str]] = []
|
|
fanout = CultMeshSimulationObservationFanout(
|
|
CultMeshDiscoveryClient("127.0.0.1", closed_port, timeout_seconds=0.2),
|
|
peers,
|
|
verse_id="arena",
|
|
roles=["simulation-witness"],
|
|
on_error=lambda endpoint, error: errors.append((endpoint, str(error))),
|
|
)
|
|
fanout.enqueue(simulation_observation(
|
|
message_id="retry-observation",
|
|
witness_runtime_id="python-fanout",
|
|
shard_id="arena",
|
|
shard_epoch=1,
|
|
frame=212,
|
|
subject_id="bob",
|
|
claim_kind="miss",
|
|
claim_hash=compute_simulation_claim_hash("miss", "alice", "bob", "frame:212"),
|
|
claim_summary="alice missed bob",
|
|
).to_wire())
|
|
|
|
candidates = fanout.flush()
|
|
|
|
self.assertEqual(candidates, [])
|
|
self.assertEqual(fanout.pending_count(), 1)
|
|
self.assertEqual(len(errors), 1)
|
|
self.assertIn(f"127.0.0.1:{closed_port}", errors[0][0])
|
|
|
|
def test_cultmesh_simulation_observation_fanout_runs_background_loop(self) -> None:
|
|
hub = CultNetSimulationObservationHub(
|
|
CultNetSimulationConsensusOptions(minimum_witnesses=1, quorum_ratio=1.0)
|
|
)
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="sim-loop-peer"), observation_hub=hub)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="sim-loop-peer",
|
|
verse_id="arena",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("simulation-witness",),
|
|
))
|
|
claim_hash = compute_simulation_claim_hash("block", "dana", "eve", "frame:211")
|
|
seen_candidates: list[dict[str, object]] = []
|
|
fanout = CultMeshSimulationObservationFanout(
|
|
CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0),
|
|
peers,
|
|
verse_id="arena",
|
|
roles=["simulation-witness"],
|
|
interval_seconds=0.05,
|
|
on_candidate=seen_candidates.append,
|
|
)
|
|
fanout.start()
|
|
fanout.enqueue(simulation_observation(
|
|
message_id="loop-observation",
|
|
witness_runtime_id="python-fanout",
|
|
shard_id="arena",
|
|
shard_epoch=1,
|
|
frame=211,
|
|
subject_id="eve",
|
|
claim_kind="block",
|
|
claim_hash=claim_hash,
|
|
claim_summary="dana blocked eve",
|
|
).to_wire())
|
|
deadline = time.monotonic() + 2.0
|
|
while not seen_candidates and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
fanout.stop()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(fanout.pending_count(), 0)
|
|
self.assertTrue(seen_candidates)
|
|
self.assertEqual(seen_candidates[0]["claimHash"], claim_hash)
|
|
self.assertEqual(seen_candidates[0]["messageId"], "loop-observation")
|
|
|
|
def test_cultmesh_discovery_client_fans_out_snapshots_to_peers(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.snapshot_fanout_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.snapshot_fanout_note.v1",
|
|
)
|
|
first = CultMesh.create_node(runtime_id="snapshot-peer-a")
|
|
first.database.register_document(document)
|
|
first.database.put_raw_message(document, "note:a", {"body": "from-a"}, shard_id="notes", shard_epoch=1)
|
|
second = CultMesh.create_node(runtime_id="snapshot-peer-b")
|
|
second.database.register_document(document)
|
|
second.database.put_raw_message(document, "note:b", {"body": "from-b"}, shard_id="notes", shard_epoch=1)
|
|
target = CultMesh.create_node(runtime_id="snapshot-target")
|
|
target.database.register_document(document)
|
|
first_server = CultMesh.serve_node(first)
|
|
second_server = CultMesh.serve_node(second)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="snapshot-peer-a",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{first_server.port}", f"cultnet://127.0.0.1:{first_server.port}"),
|
|
roles=("read-replica",),
|
|
))
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="snapshot-peer-b",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{second_server.port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
|
|
client = CultMeshDiscoveryClient("127.0.0.1", first_server.port, timeout_seconds=2.0)
|
|
responses = client.fanout_snapshot_responses(
|
|
peers,
|
|
verse_id="mesh",
|
|
roles=["read-replica"],
|
|
schema_ids=["mesh.snapshot_fanout_note.v1"],
|
|
)
|
|
applied = client.sync_snapshots(
|
|
target.database,
|
|
peers,
|
|
verse_id="mesh",
|
|
roles=["read-replica"],
|
|
schema_ids=["mesh.snapshot_fanout_note.v1"],
|
|
)
|
|
finally:
|
|
first_server.stop()
|
|
second_server.stop()
|
|
|
|
self.assertEqual(len(responses), 2)
|
|
self.assertTrue(all(isinstance(response, CultNetRawSnapshotResponse) for response in responses))
|
|
self.assertEqual(
|
|
sorted(record.record_key for response in responses for record in response.documents),
|
|
["note:a", "note:b"],
|
|
)
|
|
self.assertEqual(sorted(record.record_key for record in applied), ["note:a", "note:b"])
|
|
self.assertEqual(target.database.get_required(document, "note:a")["body"], "from-a")
|
|
self.assertEqual(target.database.get_required(document, "note:b")["body"], "from-b")
|
|
|
|
def test_cultmesh_discovery_client_sync_documents_returns_requested_aliases(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.snapshot_fanout_alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.snapshot_fanout_alias_note.v1",
|
|
schema_name="mesh.snapshot_fanout_alias_note",
|
|
schema_version="mesh.snapshot_fanout_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.snapshot_fanout_alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.snapshot_fanout_alias_note.v1",
|
|
schema_name="mesh.snapshot_fanout_alias_note",
|
|
schema_version="mesh.snapshot_fanout_alias_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="snapshot-alias-source")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:alias", CanonicalNote("from-source"))
|
|
target = CultMesh.create_node(runtime_id="snapshot-alias-target")
|
|
target.database.register_document(document)
|
|
server = CultMesh.serve_node(source)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="snapshot-alias-source",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
|
|
values = CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0).sync_documents(
|
|
target.database,
|
|
peers,
|
|
verse_id="mesh",
|
|
roles=["read-replica"],
|
|
documents=[(alias, "note:alias")],
|
|
)
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(len(values), 1)
|
|
self.assertIsInstance(values[0], UiNote)
|
|
self.assertEqual(values[0].body, "from-source")
|
|
self.assertEqual(target.database.get_required(document, "note:alias").body, "from-source")
|
|
self.assertIsInstance(target.database.get_required(alias, "note:alias"), UiNote)
|
|
|
|
def test_cultmesh_document_subscription_syncs_alias_snapshot_and_changes(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.subscription_alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.subscription_alias_note.v1",
|
|
schema_name="mesh.subscription_alias_note",
|
|
schema_version="mesh.subscription_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.subscription_alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.subscription_alias_note.v1",
|
|
schema_name="mesh.subscription_alias_note",
|
|
schema_version="mesh.subscription_alias_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="subscription-alias-source")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:live", CanonicalNote("initial"))
|
|
target = CultMesh.create_node(runtime_id="subscription-alias-target")
|
|
target.database.register_document(document)
|
|
server = CultMesh.serve_node(source)
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
with CultMesh.subscribe_document(target, raw_client, alias, "note:live") as subscription:
|
|
initial = subscription.sync_initial()
|
|
subscription.send(document_put_raw(
|
|
message_id="subscription-alias-put",
|
|
key="note:live",
|
|
schema_id="mesh.subscription_alias_note.v1",
|
|
stored_at="2026-06-14T00:00:00Z",
|
|
payload=document.encode_payload(CanonicalNote("updated")),
|
|
))
|
|
updated = subscription.read_next_change()
|
|
subscription.send(document_delete(
|
|
message_id="subscription-alias-delete",
|
|
schema_id="mesh.subscription_alias_note.v1",
|
|
record_key="note:live",
|
|
))
|
|
removed = subscription.read_next_change()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertIsInstance(initial, UiNote)
|
|
self.assertEqual(initial.body, "initial")
|
|
self.assertEqual(updated.change_kind, "updated")
|
|
self.assertIsInstance(updated.value, UiNote)
|
|
self.assertEqual(updated.value.body, "updated")
|
|
self.assertIsInstance(updated.previous_value, UiNote)
|
|
self.assertEqual(updated.previous_value.body, "initial")
|
|
self.assertEqual(removed.change_kind, "removed")
|
|
self.assertIsNone(removed.value)
|
|
self.assertIsInstance(removed.previous_value, UiNote)
|
|
self.assertEqual(removed.previous_value.body, "updated")
|
|
self.assertIsNone(target.database.get(alias, "note:live"))
|
|
|
|
def test_cultmesh_snapshot_fanout_syncs_once_and_reports_applied_records(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.snapshot_loop_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.snapshot_loop_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="snapshot-loop-source")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:sync-once", {"body": "from-source"})
|
|
target = CultMesh.create_node(runtime_id="snapshot-loop-target")
|
|
target.database.register_document(document)
|
|
server = CultMesh.serve_node(source)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="snapshot-loop-source",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
seen: list[str] = []
|
|
fanout = CultMeshSnapshotFanout(
|
|
CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0),
|
|
target.database,
|
|
peers,
|
|
verse_id="mesh",
|
|
roles=["read-replica"],
|
|
schema_ids=["mesh.snapshot_loop_note.v1"],
|
|
on_applied=lambda record: seen.append(record.record_key),
|
|
)
|
|
|
|
applied = fanout.sync_once()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual([record.record_key for record in applied], ["note:sync-once"])
|
|
self.assertEqual(seen, ["note:sync-once"])
|
|
self.assertEqual(target.database.get_required(document, "note:sync-once")["body"], "from-source")
|
|
|
|
def test_cultmesh_snapshot_fanout_syncs_typed_documents_and_reports_alias_values(self) -> None:
|
|
@dataclass
|
|
class CanonicalNote:
|
|
body: str
|
|
|
|
@dataclass
|
|
class UiNote:
|
|
body: str
|
|
|
|
document = define_database_entry_type(
|
|
"mesh.snapshot_loop_alias_note",
|
|
[("body", 0)],
|
|
cls=CanonicalNote,
|
|
schema_id="mesh.snapshot_loop_alias_note.v1",
|
|
schema_name="mesh.snapshot_loop_alias_note",
|
|
schema_version="mesh.snapshot_loop_alias_note.v1",
|
|
)
|
|
alias = define_database_entry_type(
|
|
"mesh.snapshot_loop_alias_note.ui",
|
|
[("body", 0)],
|
|
cls=UiNote,
|
|
schema_id="mesh.snapshot_loop_alias_note.v1",
|
|
schema_name="mesh.snapshot_loop_alias_note",
|
|
schema_version="mesh.snapshot_loop_alias_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="snapshot-loop-alias-source")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:typed-loop", CanonicalNote("from-source"))
|
|
target = CultMesh.create_node(runtime_id="snapshot-loop-alias-target")
|
|
target.database.register_document(document)
|
|
server = CultMesh.serve_node(source)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="snapshot-loop-alias-source",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
seen: list[UiNote] = []
|
|
fanout = CultMeshSnapshotFanout(
|
|
CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0),
|
|
target.database,
|
|
peers,
|
|
verse_id="mesh",
|
|
roles=["read-replica"],
|
|
documents=[(alias, "note:typed-loop")],
|
|
on_document=seen.append,
|
|
)
|
|
|
|
values = fanout.sync_once()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(len(values), 1)
|
|
self.assertIsInstance(values[0], UiNote)
|
|
self.assertEqual(values[0].body, "from-source")
|
|
self.assertEqual(seen, [values[0]])
|
|
self.assertEqual(target.database.get_required(document, "note:typed-loop").body, "from-source")
|
|
self.assertIsInstance(target.database.get_required(alias, "note:typed-loop"), UiNote)
|
|
|
|
def test_cultmesh_snapshot_fanout_runs_background_loop(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.snapshot_background_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.snapshot_background_note.v1",
|
|
)
|
|
source = CultMesh.create_node(runtime_id="snapshot-background-source")
|
|
source.database.register_document(document)
|
|
source.database.put_raw_message(document, "note:background", {"body": "from-background"})
|
|
target = CultMesh.create_node(runtime_id="snapshot-background-target")
|
|
target.database.register_document(document)
|
|
server = CultMesh.serve_node(source)
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="snapshot-background-source",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
seen: list[str] = []
|
|
fanout = CultMeshSnapshotFanout(
|
|
CultMeshDiscoveryClient("127.0.0.1", server.port, timeout_seconds=2.0),
|
|
target.database,
|
|
peers,
|
|
verse_id="mesh",
|
|
roles=["read-replica"],
|
|
schema_ids=["mesh.snapshot_background_note.v1"],
|
|
interval_seconds=0.05,
|
|
on_applied=lambda record: seen.append(record.record_key),
|
|
)
|
|
fanout.start()
|
|
deadline = time.monotonic() + 2.0
|
|
while not seen and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
fanout.stop()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertTrue(seen)
|
|
self.assertEqual(target.database.get_required(document, "note:background")["body"], "from-background")
|
|
|
|
def test_cultmesh_peer_health_monitor_probes_reachable_and_failed_endpoints(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.health_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.health_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="health-peer")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node)
|
|
closed_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
closed_socket.bind(("127.0.0.1", 0))
|
|
closed_port = closed_socket.getsockname()[1]
|
|
closed_socket.close()
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="health-peer",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="closed-peer",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{closed_port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
seen: list[tuple[str, bool]] = []
|
|
monitor = CultMeshPeerHealthMonitor(
|
|
runtime_id="health-prober",
|
|
timeout_seconds=0.2,
|
|
on_update=lambda health: seen.append((health.peer_id, health.is_reachable)),
|
|
)
|
|
|
|
results = monitor.probe_catalog(peers, verse_id="mesh", roles=["read-replica"])
|
|
finally:
|
|
server.stop()
|
|
|
|
by_peer = {result.peer_id: result for result in results}
|
|
self.assertTrue(by_peer["health-peer"].is_reachable)
|
|
self.assertEqual(by_peer["health-peer"].runtime_id, "health-peer")
|
|
self.assertEqual(by_peer["health-peer"].runtime_kind, "python")
|
|
self.assertEqual(by_peer["health-peer"].display_name, "health-peer")
|
|
self.assertEqual(by_peer["health-peer"].supported_document_types, ("mesh.health_note",))
|
|
self.assertIn("cultnet.hello.v0", by_peer["health-peer"].supported_message_versions)
|
|
self.assertIn("cultnet.error.v0", by_peer["health-peer"].supported_message_versions)
|
|
self.assertEqual(by_peer["health-peer"].supported_mutation_contracts[0]["documentType"], "mesh.health_note")
|
|
self.assertIn("shardLog", by_peer["health-peer"].supported_mutation_contracts[0]["operations"])
|
|
self.assertFalse(by_peer["closed-peer"].is_reachable)
|
|
self.assertIsNone(by_peer["closed-peer"].runtime_kind)
|
|
self.assertEqual(by_peer["closed-peer"].supported_document_types, ())
|
|
self.assertEqual(by_peer["closed-peer"].supported_message_versions, ())
|
|
self.assertEqual(by_peer["closed-peer"].supported_mutation_contracts, ())
|
|
self.assertTrue(by_peer["closed-peer"].error)
|
|
self.assertEqual({peer_id for peer_id, _ in seen}, {"health-peer", "closed-peer"})
|
|
self.assertEqual(len(monitor.latest()), 2)
|
|
|
|
def test_cultmesh_peer_health_monitor_runs_background_probe_loop(self) -> None:
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="health-loop-peer"))
|
|
try:
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(CultMeshPeerCard(
|
|
peer_id="health-loop-peer",
|
|
verse_id="mesh",
|
|
endpoints=(f"cultnet://127.0.0.1:{server.port}",),
|
|
roles=("read-replica",),
|
|
))
|
|
seen: list[str] = []
|
|
monitor = CultMeshPeerHealthMonitor(
|
|
runtime_id="health-loop-prober",
|
|
timeout_seconds=0.2,
|
|
interval_seconds=0.05,
|
|
on_update=lambda health: seen.append(health.peer_id),
|
|
)
|
|
monitor.start(peers, verse_id="mesh", roles=["read-replica"])
|
|
deadline = time.monotonic() + 2.0
|
|
while time.monotonic() < deadline:
|
|
latest = monitor.latest()
|
|
if latest and latest[0].is_reachable:
|
|
break
|
|
time.sleep(0.01)
|
|
monitor.stop()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertTrue(seen)
|
|
latest = monitor.latest()
|
|
self.assertEqual(latest[0].peer_id, "health-loop-peer")
|
|
self.assertTrue(latest[0].is_reachable)
|
|
|
|
def test_cultmesh_daemon_entrypoint_serves_framed_cultnet_hello(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
ready_path = Path(temp) / "ready.json"
|
|
package_src = Path(__file__).resolve().parents[1] / "src"
|
|
env = dict(os.environ)
|
|
existing_pythonpath = env.get("PYTHONPATH")
|
|
env["PYTHONPATH"] = (
|
|
str(package_src)
|
|
if not existing_pythonpath
|
|
else f"{package_src}{os.pathsep}{existing_pythonpath}"
|
|
)
|
|
process = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"daemon-test-peer",
|
|
"--display-name",
|
|
"Daemon Test Peer",
|
|
"--port",
|
|
"0",
|
|
"--ready-file",
|
|
str(ready_path),
|
|
"--seed-interop-note",
|
|
"--seed-shard-id",
|
|
"daemon-interop",
|
|
"--verse-id",
|
|
"daemon-verse",
|
|
"--verse-display-name",
|
|
"Daemon Verse",
|
|
"--role",
|
|
"shard-primary",
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
deadline = time.monotonic() + 5.0
|
|
while not ready_path.exists() and time.monotonic() < deadline:
|
|
if process.poll() is not None:
|
|
break
|
|
time.sleep(0.05)
|
|
if not ready_path.exists():
|
|
_, stderr = process.communicate(timeout=2.0)
|
|
self.fail(f"cultmesh daemon did not publish readiness: {stderr}")
|
|
ready = json.loads(ready_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(ready["schemaVersion"], READY_SCHEMA_VERSION)
|
|
self.assertEqual(ready["runtimeId"], "daemon-test-peer")
|
|
self.assertEqual(ready["runtimeKind"], "python")
|
|
self.assertEqual(ready["displayName"], "Daemon Test Peer")
|
|
self.assertEqual(ready["endpoint"], f"cultnet://127.0.0.1:{ready['port']}")
|
|
self.assertEqual(ready["shardIds"], ["daemon-interop"])
|
|
self.assertEqual(ready["supportedDocumentTypes"], ["cultcache.interop-note"])
|
|
self.assertIn("cultnet.hello.v0", ready["supportedMessageVersions"])
|
|
self.assertIn("cultnet.document_put_raw.v0", ready["supportedMessageVersions"])
|
|
self.assertNotIn("cultnet.simulation_observation.v0", ready["supportedMessageVersions"])
|
|
self.assertEqual(ready["supportedMutationContracts"][0]["documentType"], "cultcache.interop-note")
|
|
self.assertIn("shardLog", ready["supportedMutationContracts"][0]["operations"])
|
|
self.assertEqual(ready["snapshotLimits"], {"maxSnapshotBytes": None, "maxSnapshotDocuments": None})
|
|
self.assertEqual(ready["verses"][0]["verseId"], "daemon-verse")
|
|
self.assertEqual(ready["verses"][0]["displayName"], "Daemon Verse")
|
|
self.assertEqual(ready["verses"][0]["discoveryEndpoints"], [ready["endpoint"]])
|
|
self.assertEqual(ready["verses"][0]["authorityRuntimeIds"], ["daemon-test-peer"])
|
|
self.assertEqual(ready["peers"][0]["peerId"], "daemon-test-peer")
|
|
self.assertEqual(ready["peers"][0]["endpoints"], [ready["endpoint"]])
|
|
self.assertEqual(ready["peers"][0]["roles"], ["shard-primary"])
|
|
self.assertEqual(ready["peers"][0]["shardIds"], ["daemon-interop"])
|
|
|
|
client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
hello_response = client.request(
|
|
hello(runtime_id="daemon-test-prober"),
|
|
expected_schema_version="cultnet.hello.v0",
|
|
)
|
|
|
|
self.assertEqual(hello_response["runtimeId"], "daemon-test-peer")
|
|
self.assertEqual(hello_response["displayName"], "Daemon Test Peer")
|
|
self.assertIn("cultnet.hello.v0", hello_response["supportedMessageVersions"])
|
|
self.assertIn("cultnet.schema_catalog_request.v0", hello_response["supportedMessageVersions"])
|
|
self.assertNotIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"])
|
|
self.assertEqual(hello_response["supportedDocumentTypes"], ["cultcache.interop-note"])
|
|
self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed")
|
|
self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["port"], int(ready["port"]))
|
|
self.assertEqual(ready["supportedMessageVersions"], hello_response["supportedMessageVersions"])
|
|
self.assertEqual(ready["supportedMutationContracts"], hello_response["supportedMutationContracts"])
|
|
self.assertEqual(ready["transportProfiles"], hello_response["transportProfiles"])
|
|
|
|
schema_response = client.fetch_schema_catalog(schema_ids=["cultcache.interop-note"], include_schema_json=True)
|
|
self.assertEqual(schema_response["schemas"][0]["schemaVersion"], INTEROP_SCHEMA_VERSION)
|
|
self.assertEqual(schema_response["schemas"][0]["documentType"], "cultcache.interop-note")
|
|
|
|
snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"])
|
|
self.assertEqual(snapshot_response.documents[0].record_key, "note:daemon-test-peer")
|
|
note = interop_note_document.decode_payload(snapshot_response.documents[0].payload)
|
|
self.assertEqual(note["authorRuntimeId"], "daemon-test-peer")
|
|
self.assertIn("daemon", note["tags"])
|
|
|
|
shard_catalog = client.fetch_shard_catalog(schema_ids=["cultcache.interop-note"])
|
|
self.assertEqual(shard_catalog["shards"][0]["shardId"], "daemon-interop")
|
|
self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["cultcache.interop-note"])
|
|
|
|
shard_log = client.fetch_shard_log_response(shard_id="daemon-interop", shard_epoch=1)
|
|
self.assertFalse(shard_log.resync_required)
|
|
self.assertEqual(shard_log.entries[0].sequence, 1)
|
|
self.assertEqual(shard_log.entries[0].change_kind, "added")
|
|
self.assertIsNotNone(shard_log.entries[0].raw_document)
|
|
assert shard_log.entries[0].raw_document is not None
|
|
self.assertEqual(shard_log.entries[0].raw_document.record_key, "note:daemon-test-peer")
|
|
|
|
discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
verses = discovery_client.fetch_verses(transport_version="cultmesh.v0")
|
|
self.assertEqual(verses[0].verse_id, "daemon-verse")
|
|
self.assertEqual(verses[0].display_name, "Daemon Verse")
|
|
self.assertEqual(verses[0].discovery_endpoints, (ready["endpoint"],))
|
|
self.assertEqual(verses[0].authority_runtime_ids, ("daemon-test-peer",))
|
|
self.assertEqual([verse.to_wire() for verse in verses], ready["verses"])
|
|
|
|
peers = discovery_client.fetch_peers(verse_id="daemon-verse", roles=["shard-primary"])
|
|
self.assertEqual(peers[0].peer_id, "daemon-test-peer")
|
|
self.assertEqual(peers[0].endpoints, (ready["endpoint"],))
|
|
self.assertEqual(peers[0].roles, ("shard-primary",))
|
|
self.assertEqual(peers[0].shard_ids, ("daemon-interop",))
|
|
self.assertEqual([peer.to_wire() for peer in peers], ready["peers"])
|
|
|
|
monitor = CultMeshPeerHealthMonitor(runtime_id="daemon-health-prober", timeout_seconds=2.0)
|
|
health = monitor.probe_peer(peers[0])[0]
|
|
self.assertTrue(health.is_reachable)
|
|
self.assertEqual(health.runtime_id, "daemon-test-peer")
|
|
self.assertEqual(health.supported_document_types, ("cultcache.interop-note",))
|
|
self.assertEqual(health.supported_mutation_contracts[0]["documentType"], "cultcache.interop-note")
|
|
finally:
|
|
process.terminate()
|
|
try:
|
|
process.communicate(timeout=5.0)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.communicate(timeout=5.0)
|
|
|
|
def test_cultmesh_daemon_restarts_with_persisted_cache_and_shard_log(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
cache_path = root / "mesh.cc"
|
|
shard_log_path = root / "shard-log"
|
|
env = dict(os.environ)
|
|
package_src = Path(__file__).resolve().parents[1] / "src"
|
|
existing_pythonpath = env.get("PYTHONPATH")
|
|
env["PYTHONPATH"] = (
|
|
str(package_src)
|
|
if not existing_pythonpath
|
|
else f"{package_src}{os.pathsep}{existing_pythonpath}"
|
|
)
|
|
|
|
first_ready = root / "first-ready.json"
|
|
first = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"durable-daemon-peer",
|
|
"--port",
|
|
"0",
|
|
"--cache-file",
|
|
str(cache_path),
|
|
"--enable-durable-shard-logs",
|
|
"--shard-log-file",
|
|
str(shard_log_path),
|
|
"--seed-interop-note",
|
|
"--seed-shard-id",
|
|
"durable-interop",
|
|
"--ready-file",
|
|
str(first_ready),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
self._wait_for_ready_file(first, first_ready)
|
|
finally:
|
|
self._terminate_process(first)
|
|
|
|
second_ready = root / "second-ready.json"
|
|
second = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"durable-daemon-peer",
|
|
"--port",
|
|
"0",
|
|
"--cache-file",
|
|
str(cache_path),
|
|
"--enable-durable-shard-logs",
|
|
"--shard-log-file",
|
|
str(shard_log_path),
|
|
"--register-interop-note",
|
|
"--verse-id",
|
|
"durable-verse",
|
|
"--role",
|
|
"shard-primary",
|
|
"--ready-file",
|
|
str(second_ready),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
ready = self._wait_for_ready_file(second, second_ready)
|
|
client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
|
|
snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"])
|
|
self.assertEqual(snapshot_response.documents[0].record_key, "note:durable-daemon-peer")
|
|
note = interop_note_document.decode_payload(snapshot_response.documents[0].payload)
|
|
self.assertEqual(note["authorRuntimeId"], "durable-daemon-peer")
|
|
|
|
shard_log = client.fetch_shard_log_response(shard_id="durable-interop", shard_epoch=1)
|
|
self.assertFalse(shard_log.resync_required)
|
|
self.assertEqual(shard_log.entries[0].sequence, 1)
|
|
self.assertIsNotNone(shard_log.entries[0].raw_document)
|
|
assert shard_log.entries[0].raw_document is not None
|
|
self.assertEqual(shard_log.entries[0].raw_document.record_key, "note:durable-daemon-peer")
|
|
|
|
discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
peers = discovery_client.fetch_peers(verse_id="durable-verse", roles=["shard-primary"])
|
|
self.assertEqual(peers[0].shard_ids, ("durable-interop",))
|
|
finally:
|
|
self._terminate_process(second)
|
|
|
|
def test_cultmesh_daemon_serves_live_database_subscription_mutations(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
ready_path = Path(temp) / "ready.json"
|
|
package_src = Path(__file__).resolve().parents[1] / "src"
|
|
env = dict(os.environ)
|
|
existing_pythonpath = env.get("PYTHONPATH")
|
|
env["PYTHONPATH"] = (
|
|
str(package_src)
|
|
if not existing_pythonpath
|
|
else f"{package_src}{os.pathsep}{existing_pythonpath}"
|
|
)
|
|
process = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"live-daemon-peer",
|
|
"--port",
|
|
"0",
|
|
"--seed-interop-note",
|
|
"--seed-shard-id",
|
|
"live-interop",
|
|
"--ready-file",
|
|
str(ready_path),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
ready = self._wait_for_ready_file(process, ready_path)
|
|
client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
live_note = {
|
|
"schemaVersion": INTEROP_SCHEMA_VERSION,
|
|
"documentId": "note:daemon-live",
|
|
"authorRuntimeId": "daemon-live-client",
|
|
"title": "daemon live subscription note",
|
|
"body": "live subscription put from Python daemon smoke",
|
|
"tags": ["daemon", "subscription", "interop"],
|
|
}
|
|
|
|
with client.subscribe_database(
|
|
subscription_id="daemon-live-sub",
|
|
schema_ids=["cultcache.interop-note"],
|
|
) as subscription:
|
|
snapshot = subscription.read_next_snapshot_response()
|
|
subscription.send(document_put_raw(
|
|
message_id="daemon-live-put",
|
|
key="note:daemon-live",
|
|
schema_id="cultcache.interop-note",
|
|
stored_at="2026-06-14T00:00:00Z",
|
|
payload=interop_note_document.encode_payload(live_note),
|
|
source_runtime_id="daemon-live-client",
|
|
shard_id="live-interop",
|
|
shard_epoch=1,
|
|
))
|
|
change = subscription.read_next_change()
|
|
|
|
self.assertEqual(snapshot.documents[0].record_key, "note:live-daemon-peer")
|
|
self.assertEqual(change.change_kind, "added")
|
|
self.assertEqual(change.record_key, "note:daemon-live")
|
|
self.assertIsNotNone(change.raw_document)
|
|
assert change.raw_document is not None
|
|
self.assertEqual(
|
|
interop_note_document.decode_payload(change.raw_document.payload),
|
|
live_note,
|
|
)
|
|
|
|
shard_log = client.fetch_shard_log_response(shard_id="live-interop", shard_epoch=1)
|
|
self.assertFalse(shard_log.resync_required)
|
|
self.assertEqual([entry.sequence for entry in shard_log.entries], [1, 2])
|
|
self.assertEqual(shard_log.entries[1].change_kind, "added")
|
|
self.assertIsNotNone(shard_log.entries[1].raw_document)
|
|
assert shard_log.entries[1].raw_document is not None
|
|
self.assertEqual(shard_log.entries[1].raw_document.record_key, "note:daemon-live")
|
|
finally:
|
|
self._terminate_process(process)
|
|
|
|
def test_cultmesh_daemon_serves_compacted_shard_log_resync_boundary(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
cache_path = root / "mesh.cc"
|
|
shard_log_path = root / "shard-log"
|
|
store = CultNetFileShardMutationLogStore(shard_log_path)
|
|
node = CultMesh.create_node(
|
|
cache_path,
|
|
runtime_id="compacted-daemon-peer",
|
|
enable_durable_shard_logs=True,
|
|
shard_log_path=shard_log_path,
|
|
)
|
|
node.database.register_document(interop_note_document)
|
|
node.database.put_raw_message(
|
|
interop_note_document,
|
|
"note:compacted-one",
|
|
{
|
|
"schemaVersion": INTEROP_SCHEMA_VERSION,
|
|
"documentId": "note:compacted-one",
|
|
"authorRuntimeId": "compacted-daemon-peer",
|
|
"title": "compacted daemon note one",
|
|
"body": "first retained cache value",
|
|
"tags": ["daemon", "compaction"],
|
|
},
|
|
message_id="compacted-put-1",
|
|
shard_id="compacted-interop",
|
|
shard_epoch=7,
|
|
)
|
|
node.database.put_raw_message(
|
|
interop_note_document,
|
|
"note:compacted-two",
|
|
{
|
|
"schemaVersion": INTEROP_SCHEMA_VERSION,
|
|
"documentId": "note:compacted-two",
|
|
"authorRuntimeId": "compacted-daemon-peer",
|
|
"title": "compacted daemon note two",
|
|
"body": "second retained cache value",
|
|
"tags": ["daemon", "compaction"],
|
|
},
|
|
message_id="compacted-put-2",
|
|
shard_id="compacted-interop",
|
|
shard_epoch=7,
|
|
)
|
|
store.compact_through("compacted-interop", 1)
|
|
|
|
ready_path = root / "ready.json"
|
|
package_src = Path(__file__).resolve().parents[1] / "src"
|
|
env = dict(os.environ)
|
|
existing_pythonpath = env.get("PYTHONPATH")
|
|
env["PYTHONPATH"] = (
|
|
str(package_src)
|
|
if not existing_pythonpath
|
|
else f"{package_src}{os.pathsep}{existing_pythonpath}"
|
|
)
|
|
process = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"compacted-daemon-peer",
|
|
"--port",
|
|
"0",
|
|
"--cache-file",
|
|
str(cache_path),
|
|
"--enable-durable-shard-logs",
|
|
"--shard-log-file",
|
|
str(shard_log_path),
|
|
"--register-interop-note",
|
|
"--ready-file",
|
|
str(ready_path),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
ready = self._wait_for_ready_file(process, ready_path)
|
|
client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
|
|
snapshot_response = client.fetch_snapshot_response(schema_ids=["cultcache.interop-note"])
|
|
self.assertEqual(
|
|
{document.record_key for document in snapshot_response.documents},
|
|
{"note:compacted-one", "note:compacted-two"},
|
|
)
|
|
|
|
compacted = client.fetch_shard_log_response(
|
|
shard_id="compacted-interop",
|
|
shard_epoch=7,
|
|
after_sequence=0,
|
|
)
|
|
retained = client.fetch_shard_log_response(
|
|
shard_id="compacted-interop",
|
|
shard_epoch=7,
|
|
after_sequence=1,
|
|
)
|
|
|
|
self.assertTrue(compacted.resync_required)
|
|
self.assertEqual(compacted.reason, "compacted_history")
|
|
self.assertEqual(compacted.compacted_through, 1)
|
|
self.assertEqual(compacted.entries, ())
|
|
self.assertFalse(retained.resync_required)
|
|
self.assertEqual([entry.sequence for entry in retained.entries], [2])
|
|
self.assertIsNotNone(retained.entries[0].raw_document)
|
|
assert retained.entries[0].raw_document is not None
|
|
self.assertEqual(retained.entries[0].raw_document.record_key, "note:compacted-two")
|
|
finally:
|
|
self._terminate_process(process)
|
|
|
|
def test_cultmesh_daemon_accepts_schema_write_forwarder_mutations(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
ready_path = Path(temp) / "ready.json"
|
|
package_src = Path(__file__).resolve().parents[1] / "src"
|
|
env = dict(os.environ)
|
|
existing_pythonpath = env.get("PYTHONPATH")
|
|
env["PYTHONPATH"] = (
|
|
str(package_src)
|
|
if not existing_pythonpath
|
|
else f"{package_src}{os.pathsep}{existing_pythonpath}"
|
|
)
|
|
process = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"forwarder-daemon-peer",
|
|
"--port",
|
|
"0",
|
|
"--register-interop-note",
|
|
"--ready-file",
|
|
str(ready_path),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
ready = self._wait_for_ready_file(process, ready_path)
|
|
endpoint = f"cultnet://127.0.0.1:{ready['port']}"
|
|
shard = CultNetShardDescriptor(
|
|
shard_id="forwarded-interop",
|
|
owner_runtime_id="forwarder-daemon-peer",
|
|
epoch=3,
|
|
is_primary=False,
|
|
schema_ids=("cultcache.interop-note",),
|
|
primary_endpoints=(endpoint,),
|
|
)
|
|
forwarder: CultNetShardWriteForwarder = CultNetSchemaWriteForwarder(timeout_seconds=2.0)
|
|
note = {
|
|
"schemaVersion": INTEROP_SCHEMA_VERSION,
|
|
"documentId": "note:forwarded-daemon",
|
|
"authorRuntimeId": "forwarding-client",
|
|
"title": "forwarded daemon note",
|
|
"body": "forwarded through the schema write-forwarder",
|
|
"tags": ["daemon", "forwarded"],
|
|
}
|
|
|
|
forwarder.forward_put(shard, document_put_raw(
|
|
message_id="forwarded-daemon-put",
|
|
key="note:forwarded-daemon",
|
|
schema_id="cultcache.interop-note",
|
|
stored_at="2026-06-14T00:00:00Z",
|
|
payload=interop_note_document.encode_payload(note),
|
|
source_runtime_id="forwarding-client",
|
|
).to_wire())
|
|
|
|
client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
self._eventually(lambda: bool(client.fetch_snapshot_response(
|
|
schema_ids=["cultcache.interop-note"],
|
|
record_keys=["note:forwarded-daemon"],
|
|
).documents))
|
|
snapshot = client.fetch_snapshot_response(
|
|
schema_ids=["cultcache.interop-note"],
|
|
record_keys=["note:forwarded-daemon"],
|
|
)
|
|
self.assertEqual(
|
|
interop_note_document.decode_payload(snapshot.documents[0].payload),
|
|
note,
|
|
)
|
|
|
|
forwarder.forward_delete(shard, document_delete(
|
|
message_id="forwarded-daemon-delete",
|
|
schema_id="cultcache.interop-note",
|
|
record_key="note:forwarded-daemon",
|
|
).to_wire())
|
|
self._eventually(lambda: not client.fetch_snapshot_response(
|
|
schema_ids=["cultcache.interop-note"],
|
|
record_keys=["note:forwarded-daemon"],
|
|
).documents)
|
|
|
|
shard_log = client.fetch_shard_log_response(shard_id="forwarded-interop", shard_epoch=3)
|
|
self.assertEqual([entry.change_kind for entry in shard_log.entries], ["added", "removed"])
|
|
self.assertEqual(shard_log.entries[0].put["shardId"], "forwarded-interop")
|
|
self.assertEqual(shard_log.entries[0].put["shardEpoch"], 3)
|
|
self.assertEqual(shard_log.entries[1].delete["shardId"], "forwarded-interop")
|
|
self.assertEqual(shard_log.entries[1].delete["shardEpoch"], 3)
|
|
finally:
|
|
self._terminate_process(process)
|
|
|
|
def test_cultmesh_daemon_refuses_a_snapshot_document_limit_of_zero_at_startup(self) -> None:
|
|
# A limit of zero is a server that could answer no snapshot, which the Rust document
|
|
# server's options also refuse; the daemon passes the flag to the server, and exits with
|
|
# a usage error naming the rule instead of serving. The limit's enforcement as a peer error is pinned
|
|
# in process by test_cultmesh_local_server_rejects_oversized_snapshot_responses.
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
ready_path = Path(temp) / "ready.json"
|
|
package_src = Path(__file__).resolve().parents[1] / "src"
|
|
env = dict(os.environ)
|
|
existing_pythonpath = env.get("PYTHONPATH")
|
|
env["PYTHONPATH"] = (
|
|
str(package_src)
|
|
if not existing_pythonpath
|
|
else f"{package_src}{os.pathsep}{existing_pythonpath}"
|
|
)
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"limited-daemon-peer",
|
|
"--port",
|
|
"0",
|
|
"--seed-interop-note",
|
|
"--max-snapshot-documents",
|
|
"0",
|
|
"--ready-file",
|
|
str(ready_path),
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
timeout=30,
|
|
)
|
|
# A usage error from argparse: exit status 2, the rule named, no traceback.
|
|
self.assertEqual(completed.returncode, 2, completed.stderr)
|
|
self.assertIn("error: max_snapshot_documents must be greater than zero", completed.stderr)
|
|
self.assertNotIn("Traceback", completed.stderr)
|
|
self.assertFalse(ready_path.exists())
|
|
# A port no socket can bind is a usage error too.
|
|
completed = subprocess.run(
|
|
[sys.executable, "-m", "cultmesh_py.daemon", "--port", "99999"],
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
timeout=30,
|
|
)
|
|
self.assertEqual(completed.returncode, 2, completed.stderr)
|
|
self.assertIn("port must be between 0 and 65535", completed.stderr)
|
|
self.assertNotIn("Traceback", completed.stderr)
|
|
|
|
def test_cultmesh_daemon_serves_opt_in_simulation_observations(self) -> None:
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
ready_path = Path(temp) / "ready.json"
|
|
package_src = Path(__file__).resolve().parents[1] / "src"
|
|
env = dict(os.environ)
|
|
existing_pythonpath = env.get("PYTHONPATH")
|
|
env["PYTHONPATH"] = (
|
|
str(package_src)
|
|
if not existing_pythonpath
|
|
else f"{package_src}{os.pathsep}{existing_pythonpath}"
|
|
)
|
|
process = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"cultmesh_py.daemon",
|
|
"--runtime-id",
|
|
"simulation-daemon-peer",
|
|
"--port",
|
|
"0",
|
|
"--enable-simulation-observations",
|
|
"--simulation-minimum-witnesses",
|
|
"2",
|
|
"--simulation-quorum-ratio",
|
|
"1.0",
|
|
"--verse-id",
|
|
"simulation-verse",
|
|
"--ready-file",
|
|
str(ready_path),
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
try:
|
|
ready = self._wait_for_ready_file(process, ready_path)
|
|
self.assertIn("cultnet.simulation_observation.v0", ready["supportedMessageVersions"])
|
|
self.assertIn("cultnet.simulation_consensus_candidate.v0", ready["supportedMessageVersions"])
|
|
client = CultMesh.create_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
hello_response = client.request(
|
|
hello(runtime_id="simulation-daemon-prober"),
|
|
expected_schema_version="cultnet.hello.v0",
|
|
)
|
|
self.assertEqual(ready["supportedMessageVersions"], hello_response["supportedMessageVersions"])
|
|
discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", int(ready["port"]), timeout_seconds=2.0)
|
|
peers = discovery_client.fetch_peers(verse_id="simulation-verse", roles=["simulation-observer"])
|
|
self.assertEqual(peers[0].peer_id, "simulation-daemon-peer")
|
|
self.assertIn("simulation-observer", peers[0].roles)
|
|
self.assertIn("read-replica", peers[0].roles)
|
|
|
|
claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100")
|
|
first = client.request(
|
|
simulation_observation(
|
|
message_id="daemon-sim-1",
|
|
witness_runtime_id="watcher-1",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
),
|
|
expected_schema_version="cultnet.simulation_consensus_candidate.v0",
|
|
)
|
|
second = client.request(
|
|
simulation_observation(
|
|
message_id="daemon-sim-2",
|
|
witness_runtime_id="watcher-2",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
),
|
|
expected_schema_version="cultnet.simulation_consensus_candidate.v0",
|
|
)
|
|
|
|
self.assertEqual(first["messageId"], "daemon-sim-1")
|
|
self.assertFalse(first["hasQuorum"])
|
|
self.assertEqual(first["witnessCount"], 1)
|
|
self.assertEqual(second["messageId"], "daemon-sim-2")
|
|
self.assertTrue(second["hasQuorum"])
|
|
self.assertEqual(second["witnessCount"], 2)
|
|
self.assertEqual(second["claimHash"], claim_hash)
|
|
finally:
|
|
self._terminate_process(process)
|
|
|
|
def _wait_for_ready_file(self, process: subprocess.Popen[str], ready_path: Path) -> dict[str, object]:
|
|
deadline = time.monotonic() + 5.0
|
|
while not ready_path.exists() and time.monotonic() < deadline:
|
|
if process.poll() is not None:
|
|
break
|
|
time.sleep(0.05)
|
|
if not ready_path.exists():
|
|
_, stderr = process.communicate(timeout=2.0)
|
|
self.fail(f"cultmesh daemon did not publish readiness: {stderr}")
|
|
return json.loads(ready_path.read_text(encoding="utf-8"))
|
|
|
|
def _terminate_process(self, process: subprocess.Popen[str]) -> None:
|
|
process.terminate()
|
|
try:
|
|
process.communicate(timeout=5.0)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.communicate(timeout=5.0)
|
|
|
|
def test_cultmesh_local_server_serves_node_and_catalogs_over_clients(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.server_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.server_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-server")
|
|
node.database.register_document(document)
|
|
node.database.put_raw_message(document, "note:1", {"body": "served"}, shard_id="notes", shard_epoch=1)
|
|
observation_hub = CultNetSimulationObservationHub(
|
|
CultNetSimulationConsensusOptions(minimum_witnesses=2, quorum_ratio=1.0)
|
|
)
|
|
seen_candidates: list[CultNetSimulationConsensusCandidate] = []
|
|
unsubscribe_candidates = observation_hub.watch_candidates(seen_candidates.append)
|
|
claim_hash = compute_simulation_claim_hash("hit", "alice", "bob", "frame:100")
|
|
verses = CultMeshVerseCatalog()
|
|
verses.upsert(
|
|
CultMeshVerseDescriptor(
|
|
verse_id="server-verse",
|
|
display_name="Server Verse",
|
|
authority_model="local",
|
|
compatibility=CultMeshVerseCompatibility("cultmesh.v0", "rules"),
|
|
)
|
|
)
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(
|
|
CultMeshPeerCard(
|
|
peer_id="mesh-server",
|
|
verse_id="server-verse",
|
|
endpoints=("cultnet://127.0.0.1:0",),
|
|
roles=("read-replica",),
|
|
)
|
|
)
|
|
|
|
server = CultMesh.serve_node(
|
|
node,
|
|
verse_catalog=verses,
|
|
peer_catalog=peers,
|
|
observation_hub=observation_hub,
|
|
display_name="Mesh Server",
|
|
)
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
hello_response = raw_client.request(hello(runtime_id="probe"), expected_schema_version="cultnet.hello.v0")
|
|
schema_response = raw_client.fetch_schema_catalog(schema_ids=["mesh.server_note.v1"], include_schema_json=True)
|
|
rudp_client = CultMesh.create_client(
|
|
endpoint=f"rudp://127.0.0.1:{server.port}",
|
|
timeout_seconds=2.0,
|
|
connection_id=server.rudp_connection_id,
|
|
runtime_id="rudp-probe",
|
|
)
|
|
rudp_hello_response = rudp_client.request(
|
|
hello(runtime_id="rudp-probe"),
|
|
expected_schema_version="cultnet.hello.v0",
|
|
)
|
|
rudp_schema_response = rudp_client.fetch_schema_catalog(
|
|
schema_ids=["mesh.server_note.v1"],
|
|
include_schema_json=True,
|
|
)
|
|
wire_schema_response = raw_client.fetch_schema_catalog(kinds=["wire_message"], include_schema_json=True)
|
|
synced_schema_catalog = CultNetSchemaCatalog()
|
|
synced_wire_descriptors = raw_client.sync_schema_catalog(
|
|
synced_schema_catalog,
|
|
kinds=["wire_message"],
|
|
include_schema_json=True,
|
|
)
|
|
snapshot_response = raw_client.fetch_snapshot(schema_ids=["mesh.server_note.v1"])
|
|
typed_snapshot_response = raw_client.fetch_snapshot_response(schema_ids=["mesh.server_note.v1"])
|
|
shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.server_note.v1"])
|
|
synced_shard_catalog = CultNetShardCatalog()
|
|
synced_shards = raw_client.sync_shard_catalog(synced_shard_catalog, schema_ids=["mesh.server_note.v1"])
|
|
shard_log = raw_client.fetch_shard_log(shard_id="notes", shard_epoch=1)
|
|
typed_shard_log = raw_client.fetch_shard_log_response(shard_id="notes", shard_epoch=1)
|
|
self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["protocol"], "tcp_framed")
|
|
self.assertEqual(hello_response["transportProfiles"][0]["transports"][0]["port"], server.port)
|
|
stale_shard_log = raw_client.fetch_shard_log(shard_id="notes", shard_epoch=0)
|
|
discovery_client = CultMesh.create_verse_discovery_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
fetched_verses = discovery_client.fetch_verses(transport_version="cultmesh.v0")
|
|
fetched_peers = discovery_client.fetch_peers(verse_id="server-verse", roles=["read-replica"])
|
|
first_candidate = raw_client.request(
|
|
simulation_observation(
|
|
message_id="obs-1",
|
|
witness_runtime_id="watcher-1",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
),
|
|
expected_schema_version="cultnet.simulation_consensus_candidate.v0",
|
|
)
|
|
quorum_candidate = raw_client.request(
|
|
simulation_observation(
|
|
message_id="obs-2",
|
|
witness_runtime_id="watcher-2",
|
|
shard_id="arena",
|
|
shard_epoch=4,
|
|
frame=100,
|
|
subject_id="bob",
|
|
claim_kind="hit",
|
|
claim_hash=claim_hash,
|
|
claim_summary="alice shot bob first",
|
|
),
|
|
expected_schema_version="cultnet.simulation_consensus_candidate.v0",
|
|
)
|
|
with raw_client.subscribe_database(subscription_id="sub-server", schema_ids=["mesh.server_note.v1"]) as subscription:
|
|
subscription_snapshot = subscription.read_next_snapshot_response()
|
|
subscription.send(document_put_raw(
|
|
message_id="server-put",
|
|
key="note:2",
|
|
schema_id="mesh.server_note.v1",
|
|
stored_at="2026-06-14T00:00:00Z",
|
|
payload=document.encode_payload({"body": "subscribed"}),
|
|
shard_id="primary",
|
|
shard_epoch=1,
|
|
))
|
|
subscription_change = subscription.read_next_change()
|
|
subscription.send(document_delete(
|
|
message_id="server-delete",
|
|
schema_id="mesh.server_note.v1",
|
|
record_key="note:2",
|
|
shard_id="primary",
|
|
shard_epoch=1,
|
|
))
|
|
subscription_delete = subscription.read_next_change()
|
|
finally:
|
|
unsubscribe_candidates()
|
|
server.stop()
|
|
|
|
self.assertEqual(hello_response["runtimeId"], "mesh-server")
|
|
self.assertEqual(hello_response["displayName"], "Mesh Server")
|
|
self.assertIn("cultnet.database_subscribe.v0", hello_response["supportedMessageVersions"])
|
|
self.assertIn("cultnet.error.v0", hello_response["supportedMessageVersions"])
|
|
self.assertIn("cultnet.document_put_raw.v0", hello_response["supportedMessageVersions"])
|
|
self.assertIn("cultnet.simulation_observation.v0", hello_response["supportedMessageVersions"])
|
|
self.assertEqual(hello_response["supportedMutationContracts"][0]["documentType"], "mesh.server_note")
|
|
self.assertIn("documentDelete", hello_response["supportedMutationContracts"][0]["operations"])
|
|
self.assertIn("shardLog", hello_response["supportedMutationContracts"][0]["operations"])
|
|
advertised_transports = {
|
|
transport["protocol"]: transport
|
|
for profile in hello_response["transportProfiles"]
|
|
for transport in profile["transports"]
|
|
}
|
|
self.assertEqual(advertised_transports["tcp_framed"]["port"], server.port)
|
|
self.assertEqual(advertised_transports["rudp"]["port"], server.port)
|
|
self.assertEqual(rudp_hello_response["runtimeId"], "mesh-server")
|
|
self.assertEqual(rudp_schema_response["schemas"][0]["schemaId"], "mesh.server_note.v1")
|
|
self.assertEqual(schema_response["schemas"][0]["schemaId"], "mesh.server_note.v1")
|
|
self.assertIn("schemaJson", schema_response["schemas"][0])
|
|
self.assertIn("cultnet.document_delete.v0", schema_response["schemas"][0]["wireContracts"])
|
|
self.assertIn("cultnet.shard_log_response.v0", schema_response["schemas"][0]["wireContracts"])
|
|
wire_descriptors = {schema["schemaVersion"]: schema for schema in wire_schema_response["schemas"]}
|
|
self.assertEqual(wire_descriptors["cultnet.error.v0"]["kind"], "wire_message")
|
|
self.assertEqual(wire_descriptors["cultnet.document_put_raw.v0"]["kind"], "wire_message")
|
|
self.assertIn("schemaJson", wire_descriptors["cultnet.document_put_raw.v0"])
|
|
self.assertIn("cultmesh.peer_exchange_response.v0", wire_descriptors)
|
|
self.assertIn("cultnet.error.v0", {descriptor.schema_version for descriptor in synced_wire_descriptors})
|
|
self.assertIn("cultnet.document_put_raw.v0", {descriptor.schema_version for descriptor in synced_wire_descriptors})
|
|
self.assertIsNotNone(synced_schema_catalog.get("https://github.com/GameCult/CultLib/contracts/cultnet/cultnet.document-put-raw.schema.json"))
|
|
self.assertEqual(snapshot_response["documents"][0]["recordKey"], "note:1")
|
|
self.assertEqual(typed_snapshot_response.documents[0].record_key, "note:1")
|
|
self.assertEqual(typed_snapshot_response.documents[0].schema_id, "mesh.server_note.v1")
|
|
self.assertEqual(shard_catalog["shards"][0]["schemaIds"], ["mesh.server_note.v1"])
|
|
self.assertEqual(shard_catalog["shards"][0]["epoch"], 1)
|
|
self.assertEqual(shard_catalog["shards"][0]["shardId"], "notes")
|
|
self.assertEqual(synced_shards[0].shard_id, "notes")
|
|
self.assertEqual(synced_shards[0].epoch, 1)
|
|
self.assertEqual(synced_shard_catalog.get("notes"), synced_shards[0])
|
|
self.assertEqual(shard_log["entries"][0]["changeKind"], "added")
|
|
self.assertTrue(stale_shard_log["resyncRequired"])
|
|
self.assertEqual(stale_shard_log["reason"], "stale_epoch")
|
|
self.assertEqual(stale_shard_log["shardEpoch"], 1)
|
|
self.assertEqual(stale_shard_log["entries"], [])
|
|
self.assertEqual(typed_shard_log.last_sequence, 1)
|
|
self.assertEqual(typed_shard_log.entries[0].put["document"]["recordKey"], "note:1")
|
|
self.assertIsInstance(typed_shard_log.entries[0].raw_document, CultNetRawDocumentRecord)
|
|
self.assertEqual(typed_shard_log.entries[0].raw_document.record_key, "note:1")
|
|
self.assertEqual(fetched_verses[0].verse_id, "server-verse")
|
|
self.assertEqual(fetched_peers[0].peer_id, "mesh-server")
|
|
self.assertEqual(first_candidate["messageId"], "obs-1")
|
|
self.assertFalse(first_candidate["hasQuorum"])
|
|
self.assertEqual(quorum_candidate["messageId"], "obs-2")
|
|
self.assertEqual(quorum_candidate["claimHash"], claim_hash)
|
|
self.assertTrue(quorum_candidate["hasQuorum"])
|
|
self.assertEqual(quorum_candidate["witnessCount"], 2)
|
|
self.assertEqual([candidate.claim_hash for candidate in seen_candidates], [claim_hash, claim_hash])
|
|
self.assertIsInstance(subscription_snapshot, CultNetRawSnapshotResponse)
|
|
self.assertEqual(subscription_snapshot.to_wire()["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
|
self.assertEqual(subscription_change.change_kind, "added")
|
|
self.assertEqual(subscription_change.record_key, "note:2")
|
|
self.assertEqual(subscription_change.document["recordKey"], "note:2")
|
|
self.assertIsInstance(subscription_change.raw_document, CultNetRawDocumentRecord)
|
|
self.assertEqual(subscription_change.raw_document.schema_id, "mesh.server_note.v1")
|
|
self.assertEqual(subscription_delete.change_kind, "removed")
|
|
self.assertEqual(subscription_delete.record_key, "note:2")
|
|
self.assertIsNone(subscription_delete.raw_document)
|
|
self.assertIsNone(node.database.get(document, "note:2"))
|
|
|
|
def test_cultmesh_local_server_resolves_snapshot_and_shard_catalog_schema_aliases(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.alias_server_note",
|
|
[("body", 0)],
|
|
schema_id="sha256:mesh-alias-server-note",
|
|
schema_name="mesh.alias_server_note",
|
|
schema_version="mesh.alias_server_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-alias-server")
|
|
node.database.register_document(document)
|
|
node.database.put_raw_message(
|
|
document,
|
|
"note:alias",
|
|
{"body": "served through alias"},
|
|
shard_id="notes",
|
|
shard_epoch=1,
|
|
)
|
|
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
snapshot = raw_client.fetch_snapshot(schema_ids=["mesh.alias_server_note.v1"])
|
|
typed_snapshot = raw_client.fetch_snapshot_response(schema_ids=["mesh.alias_server_note.v1"])
|
|
shard_catalog = raw_client.fetch_shard_catalog(schema_ids=["mesh.alias_server_note.v1"])
|
|
finally:
|
|
server.stop()
|
|
|
|
local_schema_id = document.catalog_entry().schema_id
|
|
self.assertEqual(snapshot["documents"][0]["schemaId"], local_schema_id)
|
|
self.assertEqual(snapshot["documents"][0]["recordKey"], "note:alias")
|
|
self.assertEqual(typed_snapshot.documents[0].schema_id, local_schema_id)
|
|
self.assertEqual(shard_catalog["shards"][0]["schemaIds"], [local_schema_id])
|
|
self.assertEqual(shard_catalog["shards"][0]["shardId"], "notes")
|
|
|
|
def test_cultmesh_shard_snapshot_keeps_foreign_schema_alias_membership(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.alias_shard_note",
|
|
[
|
|
("schema_version", 0),
|
|
("body", 1),
|
|
],
|
|
schema_id="sha256:mesh-alias-shard-note",
|
|
schema_name="mesh.alias_shard_note",
|
|
schema_version="mesh.alias_shard_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-alias-shard-server")
|
|
node.database.register_document(document)
|
|
|
|
node.database.apply_raw_put_message(document_put_raw(
|
|
message_id="put-foreign-shard-alias",
|
|
key="note:foreign-shard",
|
|
schema_id="runtime.generated.mesh.alias_shard_note.ui.99",
|
|
stored_at="2026-06-27T00:00:00Z",
|
|
payload=document.encode_payload({
|
|
"schema_version": "mesh.alias_shard_note.v1",
|
|
"body": "visible through shard snapshot",
|
|
}),
|
|
shard_id="notes",
|
|
shard_epoch=1,
|
|
).to_wire())
|
|
node.database.put_raw_message(
|
|
document,
|
|
"note:other-shard",
|
|
{
|
|
"schema_version": "mesh.alias_shard_note.v1",
|
|
"body": "hidden by shard filter",
|
|
},
|
|
shard_id="other",
|
|
shard_epoch=1,
|
|
)
|
|
|
|
response = node.database.build_snapshot_response(
|
|
schema_ids=["mesh.alias_shard_note.v1"],
|
|
shard_id="notes",
|
|
shard_epoch=1,
|
|
)
|
|
|
|
local_schema_id = document.catalog_entry().schema_id
|
|
self.assertEqual(response.shard_id, "notes")
|
|
self.assertEqual(response.shard_epoch, 1)
|
|
self.assertEqual([(record.schema_id, record.record_key) for record in response.documents], [
|
|
(local_schema_id, "note:foreign-shard"),
|
|
])
|
|
self.assertEqual(node.cache.get_required_envelope(document, "note:foreign-shard").schema_id, local_schema_id)
|
|
|
|
node.database.apply_raw_delete_message(document_delete(
|
|
message_id="delete-foreign-shard-alias",
|
|
schema_id="mesh.alias_shard_note.v1",
|
|
record_key="note:foreign-shard",
|
|
shard_id="notes",
|
|
shard_epoch=1,
|
|
).to_wire())
|
|
deleted_response = node.database.build_snapshot_response(
|
|
schema_ids=["mesh.alias_shard_note.v1"],
|
|
shard_id="notes",
|
|
shard_epoch=1,
|
|
)
|
|
self.assertEqual(deleted_response.documents, ())
|
|
|
|
def test_cultmesh_local_server_notifies_schema_alias_subscription_for_raw_put(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.alias_sub_note",
|
|
[
|
|
("schema_version", 0),
|
|
("body", 1),
|
|
],
|
|
schema_id="sha256:mesh-alias-sub-note",
|
|
schema_name="mesh.alias_sub_note",
|
|
schema_version="mesh.alias_sub_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-alias-sub-server")
|
|
node.database.register_document(document)
|
|
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
with raw_client.subscribe_database(
|
|
subscription_id="sub-alias",
|
|
schema_ids=["mesh.alias_sub_note.v1"],
|
|
include_snapshot=False,
|
|
) as subscription:
|
|
subscription.send(document_put_raw(
|
|
message_id="put-foreign-alias",
|
|
key="note:alias-sub",
|
|
schema_id="runtime.generated.mesh.alias_sub_note.ui.99",
|
|
stored_at="2026-06-27T00:00:00Z",
|
|
payload=document.encode_payload({
|
|
"schema_version": "mesh.alias_sub_note.v1",
|
|
"body": "notified",
|
|
}),
|
|
))
|
|
change = subscription.read_next_change()
|
|
finally:
|
|
server.stop()
|
|
|
|
local_schema_id = document.catalog_entry().schema_id
|
|
self.assertEqual(change.subscription_id, "sub-alias")
|
|
self.assertEqual(change.change_kind, "added")
|
|
self.assertEqual(change.record_key, "note:alias-sub")
|
|
self.assertIsInstance(change.raw_document, CultNetRawDocumentRecord)
|
|
self.assertEqual(change.raw_document.schema_id, "runtime.generated.mesh.alias_sub_note.ui.99")
|
|
self.assertEqual(node.database.get_required(document, "note:alias-sub")["body"], "notified")
|
|
self.assertEqual(node.cache.get_required_envelope(document, "note:alias-sub").schema_id, local_schema_id)
|
|
|
|
def test_cultmesh_local_server_notifies_schema_alias_subscription_for_delete(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.alias_delete_note",
|
|
[("body", 0)],
|
|
schema_id="sha256:mesh-alias-delete-note",
|
|
schema_name="mesh.alias_delete_note",
|
|
schema_version="mesh.alias_delete_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-alias-delete-server")
|
|
node.database.register_document(document)
|
|
node.database.put_raw_message(
|
|
document,
|
|
"note:delete-alias",
|
|
{"body": "remove me"},
|
|
shard_id="notes",
|
|
shard_epoch=1,
|
|
)
|
|
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
with raw_client.subscribe_database(
|
|
subscription_id="sub-delete-alias",
|
|
schema_ids=[document.catalog_entry().schema_id],
|
|
include_snapshot=False,
|
|
) as subscription:
|
|
subscription.send(document_delete(
|
|
message_id="delete-foreign-alias",
|
|
schema_id="mesh.alias_delete_note.v1",
|
|
record_key="note:delete-alias",
|
|
shard_id="notes",
|
|
shard_epoch=1,
|
|
))
|
|
change = subscription.read_next_change()
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(change.subscription_id, "sub-delete-alias")
|
|
self.assertEqual(change.change_kind, "removed")
|
|
self.assertEqual(change.schema_id, "mesh.alias_delete_note.v1")
|
|
self.assertEqual(change.record_key, "note:delete-alias")
|
|
self.assertIsNone(change.raw_document)
|
|
self.assertIsNone(node.database.get(document, "note:delete-alias"))
|
|
|
|
def test_cultmesh_local_server_rejects_oversized_snapshot_responses(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.snapshot_limit_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.snapshot_limit_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-snapshot-limits")
|
|
node.database.register_document(document)
|
|
node.database.put_raw_message(document, "note:1", {"body": "first"})
|
|
node.database.put_raw_message(document, "note:2", {"body": "second"})
|
|
|
|
server = CultMesh.serve_node(node, max_snapshot_documents=1)
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
filtered_snapshot = raw_client.fetch_snapshot(record_keys=["note:1"])
|
|
error = raw_client.request(
|
|
snapshot_request(schema_ids=["mesh.snapshot_limit_note.v1"]),
|
|
expected_schema_version="cultnet.error.v0",
|
|
)
|
|
with self.assertRaisesRegex(CultNetPeerError, "Snapshot document limit exceeded"):
|
|
raw_client.fetch_snapshot(schema_ids=["mesh.snapshot_limit_note.v1"])
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(filtered_snapshot["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
|
self.assertEqual([document["recordKey"] for document in filtered_snapshot["documents"]], ["note:1"])
|
|
self.assertEqual(error["schemaVersion"], "cultnet.error.v0")
|
|
self.assertIn("Snapshot document limit exceeded", error["error"])
|
|
self.assertEqual(error["code"], "snapshot_document_limit_exceeded")
|
|
self.assertEqual(error["details"]["documentCount"], 2)
|
|
self.assertEqual(error["details"]["maxSnapshotDocuments"], 1)
|
|
|
|
def test_cultmesh_local_server_refuses_a_put_it_could_never_serve(self) -> None:
|
|
# A put is admitted only if some snapshot request could return it. The put arrives under a
|
|
# long foreign schema alias and is stored and served under the shorter local schema id, so
|
|
# the size that decides is the served one, not the put's own.
|
|
document = define_database_entry_type(
|
|
"mesh.served_bound_note",
|
|
[("schema_version", 0), ("body", 1)],
|
|
schema_id="sha256:mesh-served-bound",
|
|
schema_name="mesh.served_bound_note",
|
|
schema_version="mesh.served_bound_note.v1",
|
|
)
|
|
|
|
def put(message_id: str, key: str, body_length: int) -> dict[str, Any]:
|
|
return document_put_raw(
|
|
message_id=message_id,
|
|
key=key,
|
|
schema_id="runtime.generated.mesh.served_bound_note.ui.99",
|
|
stored_at="2026-09-30T00:00:00Z",
|
|
payload=document.encode_payload({
|
|
"schema_version": "mesh.served_bound_note.v1",
|
|
"body": "x" * body_length,
|
|
}),
|
|
).to_wire()
|
|
|
|
def exchange(server: Any, *messages: dict[str, Any]) -> tuple[dict[str, Any], bytes]:
|
|
client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
with client.open_transport() as transport:
|
|
for message in messages:
|
|
transport.send("schema", msgpack.packb(message, use_bin_type=True))
|
|
payload = transport.receive().payload
|
|
return msgpack.unpackb(payload, raw=False), payload
|
|
|
|
empty_id_snapshot = snapshot_request(message_id="").to_wire()
|
|
|
|
# The bound is what the server actually serves for the at-bound document alone, under the
|
|
# shortest message id it answers (an empty one), measured on an unbounded server.
|
|
def served_alone_bytes(key: str, body_length: int) -> int:
|
|
node = CultMesh.create_node(runtime_id="mesh-served-bound-probe")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node, enable_rudp=False)
|
|
try:
|
|
response, payload = exchange(server, put("probe", key, body_length), empty_id_snapshot)
|
|
finally:
|
|
server.stop()
|
|
self.assertEqual(response["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
|
self.assertEqual(len(response["documents"]), 1)
|
|
return len(payload)
|
|
|
|
limit = served_alone_bytes("note:fits", 100)
|
|
self.assertEqual(served_alone_bytes("note:over", 101), limit + 1, "fixture: the documents must straddle the bound by one byte")
|
|
|
|
node = CultMesh.create_node(runtime_id="mesh-served-bound")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node, max_snapshot_bytes=limit, enable_rudp=False)
|
|
try:
|
|
refused, _ = exchange(server, put("put-over", "note:over", 101))
|
|
stored_after_refusal = node.database.get(document, "note:over")
|
|
served, served_payload = exchange(server, put("put-at", "note:fits", 100), empty_id_snapshot)
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(refused["schemaVersion"], "cultnet.error.v0")
|
|
self.assertEqual(refused["messageId"], "put-over")
|
|
self.assertEqual(refused["code"], "document_unservable")
|
|
self.assertEqual(refused["details"], {"responseBytes": limit + 1, "maxSnapshotBytes": limit})
|
|
self.assertIn(str(limit + 1), refused["error"])
|
|
self.assertIn(str(limit), refused["error"])
|
|
self.assertIsNone(stored_after_refusal)
|
|
|
|
self.assertEqual(served["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
|
self.assertEqual([record["recordKey"] for record in served["documents"]], ["note:fits"])
|
|
self.assertEqual(served["documents"][0]["schemaId"], document.catalog_entry().schema_id)
|
|
self.assertEqual(len(served_payload), limit)
|
|
|
|
def _rudp_bound_note(self) -> Any:
|
|
return define_database_entry_type(
|
|
"mesh.rudp_bound_note",
|
|
[("schema_version", 0), ("body", 1)],
|
|
schema_id="sha256:mesh-rudp-bound",
|
|
schema_name="mesh.rudp_bound_note",
|
|
schema_version="mesh.rudp_bound_note.v1",
|
|
)
|
|
|
|
@staticmethod
|
|
def _rudp_note_put(document: Any, message_id: str, key: str, body_length: int) -> dict[str, Any]:
|
|
return document_put_raw(
|
|
message_id=message_id,
|
|
key=key,
|
|
schema_id=document.catalog_entry().schema_id,
|
|
stored_at="2026-09-30T00:00:00Z",
|
|
payload=document.encode_payload({"schema_version": "mesh.rudp_bound_note.v1", "body": "x" * body_length}),
|
|
).to_wire()
|
|
|
|
@staticmethod
|
|
def _rudp_exchange(server: Any, *messages: dict[str, Any]) -> tuple[dict[str, Any], bytes]:
|
|
"""Sends each message over RUDP in 1200-byte fragments; returns the first reply."""
|
|
transport = create_rudp_schema_transport(
|
|
host="127.0.0.1",
|
|
port=server.port,
|
|
connection_id=server.rudp_connection_id,
|
|
timeout_seconds=4.0,
|
|
max_fragment_bytes=1200,
|
|
)
|
|
try:
|
|
for message in messages:
|
|
transport.send("schema", msgpack.packb(message, use_bin_type=True))
|
|
payload = transport.receive(timeout_seconds=4.0).payload
|
|
finally:
|
|
transport.close()
|
|
return msgpack.unpackb(payload, raw=False), payload
|
|
|
|
def test_cultmesh_local_server_serves_a_100_kb_put_over_rudp_in_fragments(self) -> None:
|
|
# The reply is larger than any one UDP datagram: it arrives only because it is sent
|
|
# in fragments, and the RUDP thread serves on afterwards.
|
|
document = self._rudp_bound_note()
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-large")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
served, payload = self._rudp_exchange(
|
|
server,
|
|
self._rudp_note_put(document, "put-large", "note:large", 100_000),
|
|
snapshot_request(message_id="large").to_wire(),
|
|
)
|
|
again, _ = self._rudp_exchange(server, snapshot_request(message_id="again").to_wire())
|
|
alive = server._rudp_thread is not None and server._rudp_thread.is_alive()
|
|
finally:
|
|
server.stop()
|
|
self.assertEqual(served["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
|
self.assertEqual([record["recordKey"] for record in served["documents"]], ["note:large"])
|
|
self.assertGreater(len(served["documents"][0]["payload"]), 100_000)
|
|
self.assertGreater(len(payload), 65_507, "fixture: the reply cannot be one UDP datagram")
|
|
self.assertEqual(again["messageId"], "again")
|
|
self.assertTrue(alive)
|
|
self.assertEqual(server.rudp_send_failures, 0)
|
|
|
|
def _rudp_body_for(self, document: Any, key: str) -> Callable[[int], int]:
|
|
"""The body length whose document, keyed `key`, is served alone under the empty message
|
|
id in exactly the given bytes, measured on an unbounded server: what a snapshot sends."""
|
|
|
|
def served_alone_bytes(body_length: int) -> int:
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-bound-probe")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
_, payload = self._rudp_exchange(
|
|
server, self._rudp_note_put(document, "probe", key, body_length),
|
|
snapshot_request(message_id="").to_wire(),
|
|
)
|
|
finally:
|
|
server.stop()
|
|
return len(payload)
|
|
|
|
probe = served_alone_bytes(500)
|
|
|
|
def body_for(served_bytes: int) -> int:
|
|
body_length = 500 + served_bytes - probe
|
|
self.assertEqual(served_alone_bytes(body_length), served_bytes, "fixture: served size is linear in the body")
|
|
return body_length
|
|
|
|
return body_for
|
|
|
|
def test_cultmesh_local_server_refuses_a_rudp_put_whose_reply_would_overflow_the_reliable_queue(self) -> None:
|
|
# Replies go in fragments of 100 bytes and a peer's reliable queue holds 8: a put over RUDP
|
|
# whose served reply takes 8 fragments is admitted and served, 9 or 20 are refused. A
|
|
# snapshot of two such documents needs 16 and cannot be queued: the peer is answered with a
|
|
# small cultnet.error.v0 rather than left to time out, its session ends, and the thread
|
|
# serves on.
|
|
document = self._rudp_bound_note()
|
|
body_for = self._rudp_body_for(document, "note:q")
|
|
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-bound")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node, rudp_max_fragment_bytes=100, rudp_max_pending_reliable_packets=8)
|
|
try:
|
|
refusals = [
|
|
self._rudp_exchange(server, self._rudp_note_put(document, f"put-{served}", "note:q", body_for(served)))[0]
|
|
for served in (2000, 801)
|
|
]
|
|
stored_after_refusals = node.database.get(document, "note:q")
|
|
served, served_payload = self._rudp_exchange(
|
|
server,
|
|
self._rudp_note_put(document, "put-at", "note:q", body_for(800)),
|
|
snapshot_request(message_id="").to_wire(),
|
|
)
|
|
unqueueable, _ = self._rudp_exchange(
|
|
server,
|
|
self._rudp_note_put(document, "put-r", "note:r", body_for(800)),
|
|
snapshot_request(message_id="both").to_wire(),
|
|
)
|
|
failures_after = server.rudp_send_failures
|
|
# An empty snapshot fits the small queue; a hello reply would not.
|
|
after_reply, _ = self._rudp_exchange(
|
|
server, snapshot_request(message_id="after-queue", record_keys=["note:none"]).to_wire()
|
|
)
|
|
alive = server._rudp_thread is not None and server._rudp_thread.is_alive()
|
|
finally:
|
|
server.stop()
|
|
|
|
for refusal, (served_bytes, fragments) in zip(refusals, ((2000, 20), (801, 9))):
|
|
self.assertEqual(refusal["schemaVersion"], "cultnet.error.v0")
|
|
self.assertEqual(refusal["messageId"], f"put-{served_bytes}")
|
|
self.assertEqual(refusal["code"], "document_unservable")
|
|
self.assertEqual(refusal["details"], {
|
|
"responseBytes": served_bytes,
|
|
"maxSnapshotBytes": None,
|
|
"fragmentCount": fragments,
|
|
"maxFragmentCount": 8,
|
|
})
|
|
self.assertIn(f"{served_bytes} bytes; it takes {fragments} fragments", refusal["error"])
|
|
self.assertNotIn("note:q", refusal["error"])
|
|
self.assertIsNone(stored_after_refusals)
|
|
self.assertEqual(served["schemaVersion"], "cultnet.snapshot_response_raw.v0")
|
|
self.assertEqual([record["recordKey"] for record in served["documents"]], ["note:q"])
|
|
self.assertEqual(len(served_payload), 800)
|
|
self.assertEqual(unqueueable["schemaVersion"], "cultnet.error.v0")
|
|
self.assertEqual(unqueueable["messageId"], "both")
|
|
self.assertEqual(unqueueable["code"], "response_unqueueable")
|
|
self.assertEqual(failures_after, 0)
|
|
self.assertEqual(after_reply["messageId"], "after-queue")
|
|
self.assertTrue(alive)
|
|
|
|
class _RawPeer:
|
|
"""A peer driven packet by packet, so a test decides what it acknowledges: nothing."""
|
|
|
|
def __init__(self, server: Any) -> None:
|
|
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
self.sock.bind(("127.0.0.1", 0))
|
|
self.sock.settimeout(0.05)
|
|
self.address = self.sock.getsockname()[:2]
|
|
self.target = ("127.0.0.1", server.port)
|
|
self.session = CultNetRudpSession(
|
|
CultNetRudpSessionOptions(
|
|
connection_id=server.rudp_connection_id, initial_sequence=100, resend_delay_ms=100_000
|
|
)
|
|
)
|
|
self.sock.sendto(encode_rudp_packet(self.session.create_connect(0)), self.target)
|
|
accept = self.receive(1.0)
|
|
assert accept is not None and accept.packet_type == CultNetRudpPacketType.ACCEPT, accept
|
|
self.session.receive(accept, 1)
|
|
|
|
def receive(self, timeout: float = 0.3) -> Any:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
wire, _ = self.sock.recvfrom(65535)
|
|
except (TimeoutError, socket.timeout):
|
|
continue
|
|
return decode_rudp_packet(wire)
|
|
return None
|
|
|
|
def drain(self, timeout: float = 0.5) -> list[Any]:
|
|
# Bounded overall: the server resends what the peer never acknowledges.
|
|
packets = []
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
packet = self.receive(0.05)
|
|
if packet is not None:
|
|
packets.append(packet)
|
|
return packets
|
|
|
|
def send(self, message: dict[str, Any]) -> list[int]:
|
|
packets = self.session.send_many(
|
|
"schema",
|
|
msgpack.packb(message, use_bin_type=True),
|
|
CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=2),
|
|
max_fragment_bytes=1200,
|
|
)
|
|
for packet in packets:
|
|
self.sock.sendto(encode_rudp_packet(packet), self.target)
|
|
return [packet.sequence for packet in packets]
|
|
|
|
def close(self) -> None:
|
|
self.sock.close()
|
|
|
|
@staticmethod
|
|
def _acknowledges(packet: Any, sequence: int) -> bool:
|
|
return packet.ack == sequence or any(
|
|
packet.ack_mask & (1 << bit) and packet.ack > bit and packet.ack - bit - 1 == sequence
|
|
for bit in range(32)
|
|
)
|
|
|
|
def _brim_server(self) -> tuple[Any, Any]:
|
|
"""A server whose peer's reliable queue (8) is filled exactly by an 8-fragment reply the
|
|
peer never acknowledges, and the peer."""
|
|
document = self._rudp_bound_note()
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-brim")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node, rudp_max_fragment_bytes=100, rudp_max_pending_reliable_packets=8)
|
|
|
|
def served(body: int) -> int:
|
|
put = self._rudp_note_put(document, "p", "note:q", body)
|
|
return len(msgpack.packb(node.database.raw_put_served_alone(put, message_id=""), use_bin_type=True))
|
|
|
|
body = 400 + 800 - served(400)
|
|
self.assertEqual(served(body), 800, "fixture: served size is linear in the body")
|
|
node.database.apply_raw_put_message(self._rudp_note_put(document, "p", "note:q", body))
|
|
peer = self._RawPeer(server)
|
|
peer.send(snapshot_request(message_id="", record_keys=["note:q"]).to_wire())
|
|
first = {p.sequence for p in peer.drain(0.3) if p.packet_type == CultNetRudpPacketType.DATA}
|
|
self.assertEqual(len(first), 8, "fixture: the first reply fills the queue exactly")
|
|
return server, peer
|
|
|
|
def test_cultmesh_local_server_answers_a_peer_whose_queue_is_full_to_the_brim(self) -> None:
|
|
# The next reply cannot be queued. The peer is answered with a cultnet.error.v0 that takes
|
|
# no room in the full queue and acknowledges nothing the peer sent, then its session ends.
|
|
server, peer = self._brim_server()
|
|
try:
|
|
refused = peer.send(snapshot_request(message_id="b", record_keys=["note:none"]).to_wire())
|
|
packets = peer.drain(0.5)
|
|
alive = server._rudp_thread is not None and server._rudp_thread.is_alive()
|
|
finally:
|
|
peer.close()
|
|
server.stop()
|
|
parts = sorted(
|
|
(
|
|
p for p in packets
|
|
if p.packet_type == CultNetRudpPacketType.DATA and p.channel_id == "schema" and not p.reliable
|
|
),
|
|
key=lambda p: p.fragment_index,
|
|
)
|
|
self.assertTrue(parts, "the refusal reaches the peer")
|
|
for part in parts:
|
|
self.assertFalse(
|
|
any(self._acknowledges(part, sequence) for sequence in refused),
|
|
f"the refusal acknowledges the refused request: ack={part.ack} mask={part.ack_mask:#x}",
|
|
)
|
|
refusal = msgpack.unpackb(b"".join(bytes(p.payload) for p in parts), raw=False)
|
|
self.assertEqual(refusal["schemaVersion"], "cultnet.error.v0")
|
|
self.assertEqual(refusal["messageId"], "b")
|
|
self.assertEqual(refusal["code"], "response_unqueueable")
|
|
goodbyes = [bytes(p.payload) for p in packets if p.packet_type == CultNetRudpPacketType.DISCONNECT]
|
|
self.assertEqual(goodbyes, [b"session refused a packet"], "then the session ends")
|
|
self.assertTrue(alive)
|
|
|
|
def test_cultmesh_local_server_names_a_refusal_that_can_never_be_sent_in_the_goodbye(self) -> None:
|
|
# The refusal's own fragment fails permanently: the failure is not dropped, the goodbye
|
|
# names it by its fixed code, never the error's text.
|
|
import errno
|
|
|
|
server, peer = self._brim_server()
|
|
original_sendto = socket.socket.sendto
|
|
|
|
def sendto(sock: socket.socket, data: bytes, *args: Any) -> int:
|
|
packet = decode_rudp_packet(data)
|
|
if (
|
|
args and tuple(args[-1]) == tuple(peer.address)
|
|
and packet.packet_type == CultNetRudpPacketType.DATA and not packet.reliable
|
|
):
|
|
raise OSError(errno.EMSGSIZE, "Message too long CANARY-7f3a")
|
|
return original_sendto(sock, data, *args)
|
|
|
|
try:
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
peer.send(snapshot_request(message_id="b", record_keys=["note:none"]).to_wire())
|
|
packets = peer.drain(0.5)
|
|
alive = server._rudp_thread is not None and server._rudp_thread.is_alive()
|
|
finally:
|
|
peer.close()
|
|
server.stop()
|
|
self.assertFalse(
|
|
[p for p in packets if p.packet_type == CultNetRudpPacketType.DATA and not p.reliable],
|
|
"the refusal could not be sent",
|
|
)
|
|
goodbyes = [bytes(p.payload) for p in packets if p.packet_type == CultNetRudpPacketType.DISCONNECT]
|
|
self.assertEqual(goodbyes, [b"packet could not be sent: EMSGSIZE"])
|
|
self.assertTrue(alive)
|
|
|
|
def test_cultmesh_local_server_refuses_a_tcp_message_it_cannot_handle_and_serves_on(self) -> None:
|
|
# Messages whose handling raises: the TCP peer is refused with fixed text, the log names
|
|
# the error type only, and the connection's thread serves its next message.
|
|
shapes = [
|
|
{"schemaVersion": "cultnet.snapshot_request.v0", "messageId": "snapshot", "recordKeys": 5},
|
|
{"schemaVersion": "cultnet.shard_log_request.v0", "messageId": "shard-log", "shardId": "a", "afterSequence": "CANARY-7f3a"},
|
|
{"schemaVersion": "cultnet.database_subscribe.v0", "messageId": "subscribe", "subscriptionId": "s", "schemaIds": 5},
|
|
{"schemaVersion": "cultnet.schema_catalog_request.v0", "messageId": "catalog", "schemaIds": 5},
|
|
{"schemaVersion": VERSE_CATALOG_REQUEST, "messageId": "verse", "verseIds": 5},
|
|
]
|
|
deaths: list[str] = []
|
|
original_hook = threading.excepthook
|
|
threading.excepthook = lambda args: deaths.append(f"{args.exc_type.__name__}")
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-tcp-shapes"))
|
|
try:
|
|
with self.assertLogs("cultmesh_py.server", level="WARNING") as logs:
|
|
with CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=4.0).open_transport() as transport:
|
|
refusals = []
|
|
for shape in shapes:
|
|
transport.send("schema", msgpack.packb(shape, use_bin_type=True))
|
|
refusals.append((shape["messageId"], msgpack.unpackb(transport.receive().payload, raw=False)))
|
|
transport.send("schema", msgpack.packb(hello(runtime_id="same-connection").to_wire(), use_bin_type=True))
|
|
after = msgpack.unpackb(transport.receive().payload, raw=False)
|
|
finally:
|
|
server.stop()
|
|
threading.excepthook = original_hook
|
|
for message_id, refusal in refusals:
|
|
self.assertEqual(refusal["schemaVersion"], "cultnet.error.v0", message_id)
|
|
self.assertEqual(refusal["messageId"], message_id)
|
|
self.assertEqual(refusal["code"], "message_unhandled", message_id)
|
|
self.assertEqual(refusal["error"], "The message could not be handled.", message_id)
|
|
self.assertEqual(after["schemaVersion"], "cultnet.hello.v0")
|
|
self.assertEqual(deaths, [])
|
|
self.assertEqual(len(logs.output), len(shapes))
|
|
self.assertFalse([line for line in logs.output if "CANARY" in line])
|
|
|
|
def test_cultmesh_local_server_refuses_a_rudp_put_whose_reply_needs_more_than_65535_fragments(self) -> None:
|
|
# One message carries at most 65535 fragments, however deep the queue: with one-byte
|
|
# fragments a put served in 65536 bytes is refused, one served in 65535 is stored.
|
|
document = self._rudp_bound_note()
|
|
body_for = self._rudp_body_for(document, "note:w")
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-wide")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node, rudp_max_fragment_bytes=1)
|
|
try:
|
|
transport = create_rudp_schema_transport(
|
|
host="127.0.0.1", port=server.port, connection_id=server.rudp_connection_id,
|
|
timeout_seconds=4.0, max_fragment_bytes=1200,
|
|
)
|
|
try:
|
|
transport.send("schema", msgpack.packb(self._rudp_note_put(document, "put-over", "note:w", body_for(65_536)), use_bin_type=True))
|
|
refusal = msgpack.unpackb(transport.receive(timeout_seconds=8.0).payload, raw=False)
|
|
transport.send("schema", msgpack.packb(self._rudp_note_put(document, "put-at", "note:w", body_for(65_535)), use_bin_type=True))
|
|
deadline = time.monotonic() + 4.0
|
|
while node.database.get(document, "note:w") is None and time.monotonic() < deadline:
|
|
transport.receive_once()
|
|
time.sleep(0.005)
|
|
finally:
|
|
transport.close()
|
|
stored = node.database.get(document, "note:w")
|
|
finally:
|
|
server.stop()
|
|
self.assertEqual(refusal["code"], "document_unservable")
|
|
self.assertEqual(refusal["details"]["fragmentCount"], 65_536)
|
|
self.assertEqual(refusal["details"]["maxFragmentCount"], 65_535)
|
|
self.assertIsNotNone(stored)
|
|
|
|
def test_cultmesh_local_server_rudp_thread_survives_a_datagram_that_cannot_leave(self) -> None:
|
|
# Fragments larger than any UDP datagram: the reply's one packet can never be sent as
|
|
# built. That is a permanent failure: it ends that peer's session and is not counted as
|
|
# a lost datagram; the thread serves the next peer.
|
|
document = self._rudp_bound_note()
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-emsgsize")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node, rudp_max_fragment_bytes=200_000)
|
|
try:
|
|
transport = create_rudp_schema_transport(
|
|
host="127.0.0.1", port=server.port, connection_id=server.rudp_connection_id,
|
|
timeout_seconds=4.0, max_fragment_bytes=1200,
|
|
)
|
|
try:
|
|
transport.send("schema", msgpack.packb(self._rudp_note_put(document, "put-large", "note:big", 80_000), use_bin_type=True))
|
|
transport.send("schema", msgpack.packb(snapshot_request(message_id="big").to_wire(), use_bin_type=True))
|
|
deadline = time.monotonic() + 1.0
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
transport.receive_once()
|
|
except ConnectionError:
|
|
break
|
|
time.sleep(0.005)
|
|
finally:
|
|
transport.close()
|
|
failures = server.rudp_send_failures
|
|
hello_reply, _ = self._rudp_exchange(server, hello(runtime_id="after").to_wire())
|
|
alive = server._rudp_thread is not None and server._rudp_thread.is_alive()
|
|
finally:
|
|
server.stop()
|
|
self.assertEqual(failures, 0)
|
|
self.assertEqual(hello_reply["schemaVersion"], "cultnet.hello.v0")
|
|
self.assertTrue(alive)
|
|
|
|
def test_cultmesh_local_server_refuses_a_malformed_put_over_rudp_and_serves_on(self) -> None:
|
|
# A put whose document has no string recordKey or no binary payload could never be stored
|
|
# or served: it is refused as malformed before any sizing reads it, and the RUDP thread
|
|
# serves the next peer.
|
|
document = self._rudp_bound_note()
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-malformed")
|
|
node.database.register_document(document)
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
refusals = []
|
|
for field_name, value in (
|
|
("recordKey", None),
|
|
("recordKey", 5),
|
|
("payload", None),
|
|
("payload", "not binary"),
|
|
):
|
|
put = self._rudp_note_put(document, f"bad-{field_name}-{value}", "note:bad", 10)
|
|
if value is None:
|
|
del put["document"][field_name]
|
|
else:
|
|
put["document"][field_name] = value
|
|
refusal, _ = self._rudp_exchange(server, put)
|
|
refusals.append((put["messageId"], refusal))
|
|
after, _ = self._rudp_exchange(server, snapshot_request(message_id="after").to_wire())
|
|
alive = server._rudp_thread is not None and server._rudp_thread.is_alive()
|
|
finally:
|
|
server.stop()
|
|
for message_id, refusal in refusals:
|
|
self.assertEqual(refusal["schemaVersion"], "cultnet.error.v0", message_id)
|
|
self.assertEqual(refusal["messageId"], message_id)
|
|
self.assertEqual(refusal["code"], "malformed_document_put", message_id)
|
|
self.assertEqual(after["messageId"], "after")
|
|
self.assertEqual(after["documents"], [])
|
|
self.assertTrue(alive)
|
|
|
|
def test_cultmesh_local_server_refuses_a_put_its_store_cannot_take_and_serves_on(self) -> None:
|
|
# Well-formed puts the store refuses: a blank key, a payload that is not MessagePack, a
|
|
# payload its schema cannot decode, a global document under another key. Each is refused
|
|
# to the peer that sent it, over RUDP and over TCP; the RUDP thread lives, the TCP
|
|
# connection serves its next message, and other peers are served.
|
|
document = self._rudp_bound_note()
|
|
global_document = define_database_entry_type(
|
|
"mesh.rudp_global_note",
|
|
[("schema_version", 0), ("body", 1)],
|
|
schema_id="sha256:mesh-rudp-global",
|
|
schema_name="mesh.rudp_global_note",
|
|
schema_version="mesh.rudp_global_note.v1",
|
|
global_document=True,
|
|
)
|
|
|
|
def cases() -> list[dict[str, Any]]:
|
|
puts = []
|
|
for name, field_name, value in (
|
|
("empty-key", "recordKey", ""),
|
|
("blank-key", "recordKey", " "),
|
|
("garbage-payload", "payload", bytes([0xC1])),
|
|
("int-payload", "payload", msgpack.packb(5)),
|
|
):
|
|
put = self._rudp_note_put(document, name, "note:ok", 10)
|
|
put["document"][field_name] = value
|
|
puts.append(put)
|
|
puts.append(self._rudp_note_put(global_document, "global-wrong-key", "not-global", 10))
|
|
return puts
|
|
|
|
node = CultMesh.create_node(runtime_id="mesh-unstorable-put")
|
|
node.database.register_document(document)
|
|
node.database.register_document(global_document)
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
rudp_refusals = [(put["messageId"], self._rudp_exchange(server, put)[0]) for put in cases()]
|
|
rudp_after, _ = self._rudp_exchange(server, hello(runtime_id="rudp-after").to_wire())
|
|
rudp_alive = server._rudp_thread is not None and server._rudp_thread.is_alive()
|
|
|
|
tcp_refusals = []
|
|
with CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=4.0).open_transport() as transport:
|
|
for put in cases():
|
|
transport.send("schema", msgpack.packb(put, use_bin_type=True))
|
|
tcp_refusals.append((put["messageId"], msgpack.unpackb(transport.receive().payload, raw=False)))
|
|
transport.send("schema", msgpack.packb(hello(runtime_id="same-connection").to_wire(), use_bin_type=True))
|
|
same_connection = msgpack.unpackb(transport.receive().payload, raw=False)
|
|
other_connection = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=4.0).request(
|
|
hello(runtime_id="other-connection").to_wire(), expected_schema_version="cultnet.hello.v0"
|
|
)
|
|
finally:
|
|
server.stop()
|
|
self.assertEqual(len(rudp_refusals), 5)
|
|
for message_id, refusal in rudp_refusals + tcp_refusals:
|
|
self.assertEqual(refusal["schemaVersion"], "cultnet.error.v0", message_id)
|
|
self.assertEqual(refusal["messageId"], message_id)
|
|
self.assertEqual(refusal["code"], "malformed_document_put", message_id)
|
|
self.assertNotIn("note:ok", refusal["error"], message_id)
|
|
self.assertEqual(len(tcp_refusals), 5)
|
|
self.assertEqual(rudp_after["schemaVersion"], "cultnet.hello.v0")
|
|
self.assertTrue(rudp_alive)
|
|
self.assertEqual(same_connection["schemaVersion"], "cultnet.hello.v0")
|
|
self.assertEqual(other_connection["schemaVersion"], "cultnet.hello.v0")
|
|
self.assertIsNone(node.database.get(document, "note:ok"))
|
|
|
|
def test_cultmesh_local_server_refuses_options_that_could_answer_nothing(self) -> None:
|
|
# The Rust document server's option rules: every limit is greater than zero, and the limits
|
|
# leave room for an empty snapshot, the smallest response the server sends. One-byte
|
|
# fragments and a queue of 16 could never carry one, so no reply would ever be sent.
|
|
node = CultMesh.create_node(runtime_id="mesh-rudp-options")
|
|
probe = CultMesh.serve_node(node)
|
|
try:
|
|
_, empty_payload = self._rudp_exchange(probe, snapshot_request(message_id="").to_wire())
|
|
finally:
|
|
probe.stop()
|
|
empty = len(empty_payload)
|
|
self.assertGreater(empty, 16, "fixture: an empty snapshot takes more than 16 one-byte fragments")
|
|
self.assertNotEqual(empty % (empty - 1), 0, "fixture: empty - 1 byte fragments leave a remainder")
|
|
|
|
def refused(**options: Any) -> str:
|
|
server = None
|
|
try:
|
|
with self.assertRaises(ValueError, msg=str(options)) as raised:
|
|
server = CultMesh.serve_node(node, **options)
|
|
finally:
|
|
if server is not None:
|
|
server.stop()
|
|
return str(raised.exception)
|
|
|
|
def accepted(**options: Any) -> None:
|
|
CultMesh.serve_node(node, **options).stop()
|
|
|
|
carry = "cannot carry an empty snapshot response"
|
|
self.assertIn(carry, refused(rudp_max_fragment_bytes=1, rudp_max_pending_reliable_packets=16))
|
|
self.assertIn(carry, refused(rudp_max_fragment_bytes=1, rudp_max_pending_reliable_packets=empty - 1))
|
|
accepted(rudp_max_fragment_bytes=1, rudp_max_pending_reliable_packets=empty)
|
|
# Fragments are counted rounding up: empty - 1 bytes a fragment takes two.
|
|
self.assertIn(carry, refused(rudp_max_fragment_bytes=empty - 1, rudp_max_pending_reliable_packets=1))
|
|
accepted(rudp_max_fragment_bytes=empty - 1, rudp_max_pending_reliable_packets=2)
|
|
self.assertIn("cannot hold an empty snapshot response", refused(max_snapshot_bytes=empty - 1))
|
|
accepted(max_snapshot_bytes=empty)
|
|
self.assertIn("greater than zero", refused(max_snapshot_bytes=0))
|
|
self.assertIn("greater than zero", refused(max_snapshot_documents=0))
|
|
accepted(max_snapshot_documents=1)
|
|
self.assertIn("greater than zero", refused(rudp_resend_delay_ms=0))
|
|
accepted(rudp_resend_delay_ms=1)
|
|
self.assertIn("greater than zero", refused(rudp_max_fragment_bytes=0))
|
|
self.assertIn("greater than zero", refused(rudp_max_pending_reliable_packets=0))
|
|
# The RUDP options configure only the RUDP server: without one, none of them is checked.
|
|
accepted(enable_rudp=False, rudp_max_fragment_bytes=1, rudp_max_pending_reliable_packets=16)
|
|
accepted(
|
|
enable_rudp=False,
|
|
rudp_max_fragment_bytes=0,
|
|
rudp_max_pending_reliable_packets=0,
|
|
rudp_resend_delay_ms=0,
|
|
)
|
|
self.assertIn("greater than zero", refused(enable_rudp=False, max_snapshot_documents=0))
|
|
self.assertIn("cannot hold an empty snapshot response", refused(enable_rudp=False, max_snapshot_bytes=empty - 1))
|
|
|
|
# The configuration at the bound does answer: an empty snapshot in one-byte fragments,
|
|
# exactly as many as the queue holds.
|
|
server = CultMesh.serve_node(node, rudp_max_fragment_bytes=1, rudp_max_pending_reliable_packets=empty)
|
|
try:
|
|
served, payload = self._rudp_exchange(server, snapshot_request(message_id="").to_wire())
|
|
finally:
|
|
server.stop()
|
|
self.assertEqual(served["documents"], [])
|
|
self.assertEqual(len(payload), empty)
|
|
self.assertEqual(server.rudp_send_failures, 0)
|
|
|
|
def test_cultmesh_local_server_returns_errors_for_bad_requests(self) -> None:
|
|
document = define_database_entry_type(
|
|
"mesh.error_note",
|
|
[("body", 0)],
|
|
schema_id="mesh.error_note.v1",
|
|
)
|
|
node = CultMesh.create_node(runtime_id="mesh-error-server")
|
|
node.database.register_document(document)
|
|
|
|
server = CultMesh.serve_node(node)
|
|
try:
|
|
raw_client = CultMesh.create_client("127.0.0.1", server.port, timeout_seconds=2.0)
|
|
unsupported = raw_client.request(
|
|
{"schemaVersion": "cultnet.nope.v0", "messageId": "bad-schema"},
|
|
expected_schema_version="cultnet.error.v0",
|
|
)
|
|
malformed_put = raw_client.request(
|
|
{"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "bad-put"},
|
|
expected_schema_version="cultnet.error.v0",
|
|
)
|
|
malformed_delete = raw_client.request(
|
|
{"schemaVersion": "cultnet.document_delete.v0", "messageId": "bad-delete", "schemaId": "mesh.error_note.v1"},
|
|
expected_schema_version="cultnet.error.v0",
|
|
)
|
|
# R-2, owed by Cut 1 (docs/cultnet-selection-cut.md, section 9): this v0-only Mesh server
|
|
# refuses a cultnet.selection.v1 carrier message through the same delegating path
|
|
# (_handle_connection_message -> handle_message) as any other unsupported schema, not a
|
|
# silent empty response.
|
|
unsupported_v1 = raw_client.request(
|
|
{"schemaVersion": "cultnet.snapshot_request.v1", "messageId": "bad-v1-selection"},
|
|
expected_schema_version="cultnet.error.v0",
|
|
)
|
|
finally:
|
|
server.stop()
|
|
|
|
self.assertEqual(unsupported["messageId"], "bad-schema")
|
|
self.assertEqual(unsupported["code"], "unsupported_schema_version")
|
|
self.assertEqual(unsupported["details"]["schemaVersion"], "cultnet.nope.v0")
|
|
self.assertIn("Unsupported CultNet message schema", unsupported["error"])
|
|
self.assertEqual(malformed_put["messageId"], "bad-put")
|
|
self.assertEqual(malformed_put["code"], "malformed_document_put")
|
|
self.assertEqual(malformed_delete["messageId"], "bad-delete")
|
|
self.assertEqual(malformed_delete["code"], "malformed_document_delete")
|
|
self.assertEqual(unsupported_v1["messageId"], "bad-v1-selection")
|
|
self.assertEqual(unsupported_v1["code"], "unsupported_schema_version")
|
|
self.assertEqual(unsupported_v1["details"]["schemaVersion"], "cultnet.snapshot_request.v1")
|
|
|
|
def test_cultmesh_authority_lease_requires_live_matching_lease(self) -> None:
|
|
peer = CultMeshPeerCard(
|
|
peer_id="voidbot-local",
|
|
verse_id="local",
|
|
endpoints=("cultmesh://localhost",),
|
|
roles=("shard-primary",),
|
|
authority_lease_id="lease:voidbot-local",
|
|
)
|
|
leases = CultMeshAuthorityLeaseCatalog()
|
|
peers = CultMeshPeerCatalog()
|
|
peers.upsert(peer)
|
|
now = datetime.now(UTC)
|
|
self.assertFalse(leases.is_authorized(peer, "shard-primary", at=now))
|
|
self.assertEqual(peers.find_authorized("local", "shard-primary", leases, at=now), [])
|
|
|
|
leases.upsert(
|
|
CultMeshAuthorityLease(
|
|
lease_id="lease:voidbot-local",
|
|
verse_id="local",
|
|
peer_id="voidbot-local",
|
|
roles=("shard-primary", "shard-primary", ""),
|
|
valid_from=now - timedelta(seconds=1),
|
|
expires_at=now + timedelta(seconds=1),
|
|
shard_ids=("shard-a",),
|
|
issuer_runtime_id="odin",
|
|
signature="signed",
|
|
)
|
|
)
|
|
|
|
lease = leases.get("lease:voidbot-local")
|
|
self.assertIsNotNone(lease)
|
|
assert lease is not None
|
|
self.assertEqual(leases.leases, (lease,))
|
|
self.assertEqual(lease.roles, ("shard-primary",))
|
|
self.assertEqual(lease.issuer_runtime_id, "odin")
|
|
self.assertEqual(lease.signature, "signed")
|
|
self.assertTrue(lease.is_valid_at(now))
|
|
self.assertTrue(lease.covers(peer, "shard-primary", shard_id="shard-a", at=now))
|
|
self.assertFalse(lease.covers(peer, "shard-primary", shard_id="shard-b", at=now))
|
|
self.assertTrue(leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now))
|
|
self.assertEqual(
|
|
peers.find_authorized("local", "shard-primary", leases, shard_id="shard-a", at=now),
|
|
[peer],
|
|
)
|
|
self.assertEqual(
|
|
peers.first_authorized("local", "shard-primary", leases, shard_id="shard-a", at=now),
|
|
peer,
|
|
)
|
|
self.assertIsNone(peers.first_authorized("local", "read-replica", leases, at=now))
|
|
self.assertFalse(leases.is_authorized(peer, "shard-primary", shard_id="shard-b", at=now))
|
|
self.assertFalse(leases.is_authorized(peer, "read-replica", at=now))
|
|
self.assertIsNone(leases.get("missing"))
|
|
|
|
def test_cultmesh_authority_lease_catalog_can_require_verified_signatures(self) -> None:
|
|
peer = CultMeshPeerCard(
|
|
peer_id="voidbot-local",
|
|
verse_id="local",
|
|
endpoints=("cultmesh://localhost",),
|
|
roles=("shard-primary",),
|
|
shard_ids=("shard-a",),
|
|
authority_lease_id="lease:voidbot-local",
|
|
)
|
|
now = datetime.now(UTC)
|
|
unsigned = CultMeshAuthorityLease(
|
|
lease_id="lease:voidbot-local",
|
|
verse_id="local",
|
|
peer_id="voidbot-local",
|
|
roles=("shard-primary",),
|
|
valid_from=now - timedelta(seconds=1),
|
|
expires_at=now + timedelta(seconds=30),
|
|
shard_ids=("shard-a",),
|
|
issuer_runtime_id="odin",
|
|
)
|
|
verifier = CultMeshHmacAuthorityLeaseVerifier({"odin": b"lease-secret"})
|
|
signed = verifier.issue(unsigned)
|
|
strict_leases = CultMesh.create_authority_lease_catalog(
|
|
signature_verifier=verifier.verify,
|
|
require_verified_signatures=True,
|
|
)
|
|
strict_leases.upsert(unsigned)
|
|
self.assertFalse(strict_leases.is_verified(unsigned.lease_id))
|
|
self.assertFalse(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now))
|
|
|
|
strict_leases.upsert(signed)
|
|
self.assertTrue(strict_leases.is_verified(signed.lease_id))
|
|
self.assertTrue(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now))
|
|
|
|
wrong_key = CultMesh.create_authority_lease_catalog(
|
|
signature_verifier=CultMeshHmacAuthorityLeaseVerifier({"odin": b"wrong-secret"}).verify,
|
|
require_verified_signatures=True,
|
|
)
|
|
wrong_key.upsert(signed)
|
|
self.assertFalse(wrong_key.is_verified(signed.lease_id))
|
|
self.assertFalse(wrong_key.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now))
|
|
|
|
tampered = CultMeshAuthorityLease(
|
|
lease_id=signed.lease_id,
|
|
verse_id=signed.verse_id,
|
|
peer_id=signed.peer_id,
|
|
roles=("shard-primary", "read-replica"),
|
|
valid_from=signed.valid_from,
|
|
expires_at=signed.expires_at,
|
|
shard_ids=signed.shard_ids,
|
|
issuer_runtime_id=signed.issuer_runtime_id,
|
|
signature=signed.signature,
|
|
)
|
|
strict_leases.upsert(tampered)
|
|
self.assertFalse(strict_leases.is_verified(tampered.lease_id))
|
|
self.assertFalse(strict_leases.is_authorized(peer, "shard-primary", shard_id="shard-a", at=now))
|
|
|
|
def test_cultmesh_stream_catalog_negotiates_transport_and_latest_frame(self) -> None:
|
|
streams = CultMeshStreamCatalog()
|
|
streams.declare(
|
|
CultMeshStreamDescriptor(
|
|
stream_id="mimir:kiyo-pro",
|
|
verse_id="studio",
|
|
owner_peer_id="starfire",
|
|
kind="video",
|
|
label="Kiyo Pro",
|
|
clock={"clockDomainId": "starfire-qpc", "confidence": 0.25},
|
|
video={"width": 1920, "height": 1080, "pixelFormat": "YUY2", "framesPerSecond": 30},
|
|
preferred_transports=("shared-d3d12-texture", "shared-memory", "cultcache-page"),
|
|
max_in_flight_frames=3,
|
|
)
|
|
)
|
|
|
|
negotiation = streams.negotiate(
|
|
"mimir:kiyo-pro",
|
|
CultMeshStreamConsumerProfile(
|
|
peer_id="fensalir",
|
|
verse_id="studio",
|
|
supported_transports=("shared-d3d12-texture", "cultcache-page"),
|
|
accepted_kinds=("video",),
|
|
can_import_gpu_handles=True,
|
|
max_in_flight_frames=2,
|
|
),
|
|
)
|
|
|
|
self.assertEqual(negotiation.transport, "shared-d3d12-texture")
|
|
self.assertEqual(negotiation.max_in_flight_frames, 2)
|
|
self.assertEqual(negotiation.copy_budget, "zero-copy-target")
|
|
|
|
streams.publish_frame(
|
|
CultMeshStreamFrameHandle(
|
|
stream_id="mimir:kiyo-pro",
|
|
sequence=42,
|
|
timestamp_ns=1_000_000_000,
|
|
duration_ns=33_333_334,
|
|
transport="shared-d3d12-texture",
|
|
native_handle="0xfeed",
|
|
fence_handle="0xbeef",
|
|
fence_value=7,
|
|
unavoidable_copy_count=0,
|
|
)
|
|
)
|
|
self.assertEqual(streams.latest_frame("mimir:kiyo-pro").sequence, 42)
|
|
|
|
|
|
class CultMeshRudpSendFailureTests(unittest.TestCase):
|
|
"""One peer whose datagrams cannot be sent must not stop the RUDP thread
|
|
serving the others."""
|
|
|
|
def _client(self, server: Any, name: str) -> tuple[CultNetRudpSocketTransportConnection, tuple[str, int]]:
|
|
sock = bind_udp_socket()
|
|
client = CultNetRudpSocketTransportConnection(
|
|
CultNetRudpSocketTransportOptions(
|
|
runtime_id=name,
|
|
socket=sock,
|
|
mode=CultNetRudpSocketMode.CLIENT,
|
|
connection_id=server.rudp_connection_id,
|
|
remote_addr=("127.0.0.1", server.port),
|
|
resend_delay_ms=25,
|
|
)
|
|
)
|
|
client.connect(b"join")
|
|
for _ in range(200):
|
|
client.receive_once()
|
|
if client.connected:
|
|
break
|
|
self.assertTrue(client.connected)
|
|
return client, sock.getsockname()[:2]
|
|
|
|
def test_a_failed_send_to_one_peer_does_not_kill_the_rudp_thread(self) -> None:
|
|
import msgpack # type: ignore
|
|
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_sendto = socket.socket.sendto
|
|
try:
|
|
x, x_addr = self._client(server, "peer-x")
|
|
y, _ = self._client(server, "peer-y")
|
|
|
|
def sendto(sock: socket.socket, data: bytes, *args: Any) -> int:
|
|
if args and tuple(args[-1]) == tuple(x_addr):
|
|
raise OSError("injected send failure")
|
|
return original_sendto(sock, data, *args)
|
|
|
|
request = hello(runtime_id="probe").to_bytes()
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
# The server's reply and ack to X both fail to send.
|
|
x.send("schema", request)
|
|
self._wait_for(lambda: server.rudp_send_failures > 0)
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
|
|
y.send("schema", request)
|
|
response = None
|
|
for _ in range(200):
|
|
response = y.receive_once()
|
|
if response is not None:
|
|
break
|
|
time.sleep(0.005)
|
|
|
|
self.assertIsNotNone(response, "Y is still served after X's send failed")
|
|
self.assertEqual(msgpack.unpackb(response.payload, raw=False)["runtimeId"], "mesh-server")
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
self.assertGreater(server.rudp_send_failures, 0)
|
|
finally:
|
|
server.stop()
|
|
|
|
def _served(self, y: CultNetRudpSocketTransportConnection) -> Any:
|
|
import msgpack # type: ignore
|
|
|
|
y.send("schema", hello(runtime_id="probe").to_bytes())
|
|
for _ in range(200):
|
|
response = y.receive_once()
|
|
if response is not None:
|
|
return msgpack.unpackb(response.payload, raw=False)
|
|
time.sleep(0.005)
|
|
return None
|
|
|
|
def test_a_vanished_peer_does_not_kill_the_rudp_thread(self) -> None:
|
|
# On Windows the closed port of a peer that went away is reported as
|
|
# ConnectionResetError (WinError 10054) by the next receive. It can only fail there.
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
try:
|
|
x, _ = self._client(server, "peer-x")
|
|
y, _ = self._client(server, "peer-y")
|
|
x.send("schema", hello(runtime_id="probe").to_bytes())
|
|
x.socket.close()
|
|
time.sleep(1.0) # the server's reply and resends to X draw ICMP port-unreachable
|
|
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
served = self._served(y)
|
|
self.assertIsNotNone(served, "Y is still served after X vanished")
|
|
self.assertEqual(served["runtimeId"], "mesh-server")
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_reset_reported_by_a_receive_is_idle_not_the_end_of_the_thread(self) -> None:
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_recvfrom = socket.socket.recvfrom
|
|
resets = [3]
|
|
try:
|
|
y, _ = self._client(server, "peer-y")
|
|
|
|
def recvfrom(sock: socket.socket, *args: Any) -> Any:
|
|
if sock is server._rudp_socket and resets[0] > 0:
|
|
resets[0] -= 1
|
|
raise ConnectionResetError(errno.ECONNRESET, "injected reset")
|
|
return original_recvfrom(sock, *args)
|
|
|
|
with patch.object(socket.socket, "recvfrom", recvfrom):
|
|
self._wait_for(lambda: resets[0] == 0)
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
self.assertIsNotNone(self._served(y))
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_permanent_send_failure_ends_only_that_peers_session_and_the_goodbye_names_it(self) -> None:
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_sendto = socket.socket.sendto
|
|
try:
|
|
x, x_addr = self._client(server, "peer-x")
|
|
y, _ = self._client(server, "peer-y")
|
|
failures = [1]
|
|
|
|
def sendto(sock: socket.socket, data: bytes, *args: Any) -> int:
|
|
if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0:
|
|
failures[0] -= 1
|
|
raise OSError(errno.EMSGSIZE, "Message too long")
|
|
return original_sendto(sock, data, *args)
|
|
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
# The server's response to X can never be sent as built.
|
|
x.send("schema", hello(runtime_id="probe").to_bytes())
|
|
for _ in range(200):
|
|
x.receive_once()
|
|
if x.disconnect_reason is not None:
|
|
break
|
|
time.sleep(0.005)
|
|
self.assertIsNotNone(x.disconnect_reason, "X is told its session ended")
|
|
self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: "))
|
|
self.assertGreater(len(x.disconnect_reason), len(b"packet could not be sent: "))
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
|
|
# Y is unaffected.
|
|
self.assertFalse(x.connected)
|
|
self.assertIsNotNone(self._served(y))
|
|
self.assertEqual(server.rudp_send_failures, 0)
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_healthy_sends_are_not_failures(self) -> None:
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
try:
|
|
x, _ = self._client(server, "peer-x")
|
|
self.assertIsNotNone(self._served(x))
|
|
self.assertEqual(server.rudp_send_failures, 0)
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_permanent_failure_in_a_resend_ends_only_that_peers_session(self) -> None:
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_sendto = socket.socket.sendto
|
|
try:
|
|
x, x_addr = self._client(server, "peer-x")
|
|
y, _ = self._client(server, "peer-y")
|
|
mode = ["transient"]
|
|
|
|
def sendto(sock: socket.socket, data: bytes, *args: Any) -> int:
|
|
if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "transient":
|
|
raise OSError(errno.ENOBUFS, "injected transient failure")
|
|
if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "permanent":
|
|
mode[0] = "healthy"
|
|
raise OSError(errno.EMSGSIZE, "Message too long")
|
|
return original_sendto(sock, data, *args)
|
|
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
# X's response is lost, so a resend is pending; then it can never be sent as built.
|
|
x.send("schema", hello(runtime_id="probe").to_bytes())
|
|
self._wait_for(lambda: server.rudp_send_failures > 0)
|
|
self.assertFalse(x.disconnect_reason, "a transient failure ends no session")
|
|
mode[0] = "permanent"
|
|
for _ in range(200):
|
|
x.receive_once()
|
|
if x.disconnect_reason is not None:
|
|
break
|
|
time.sleep(0.005)
|
|
self.assertIsNotNone(x.disconnect_reason)
|
|
self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: "))
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
self.assertIsNotNone(self._served(y))
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_permanent_failure_acknowledging_a_frame_ends_the_session(self) -> None:
|
|
import msgpack # type: ignore
|
|
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_sendto = socket.socket.sendto
|
|
try:
|
|
x, x_addr = self._client(server, "peer-x")
|
|
failures = [1]
|
|
|
|
def sendto(sock: socket.socket, data: bytes, *args: Any) -> int:
|
|
if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0:
|
|
failures[0] -= 1
|
|
raise OSError(errno.EMSGSIZE, "Message too long")
|
|
return original_sendto(sock, data, *args)
|
|
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
# Not a message the server answers, so its ack is the only datagram owed.
|
|
x.send("schema", msgpack.packb(1))
|
|
for _ in range(200):
|
|
x.receive_once()
|
|
if x.disconnect_reason is not None:
|
|
break
|
|
time.sleep(0.005)
|
|
self.assertIsNotNone(x.disconnect_reason)
|
|
self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: "))
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_connect_whose_accept_can_never_be_sent_starts_no_session(self) -> None:
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_sendto = socket.socket.sendto
|
|
try:
|
|
y, _ = self._client(server, "peer-y")
|
|
z_socket = bind_udp_socket()
|
|
z_addr = z_socket.getsockname()[:2]
|
|
failures = [1]
|
|
|
|
def sendto(sock: socket.socket, data: bytes, *args: Any) -> int:
|
|
if args and tuple(args[-1]) == tuple(z_addr) and failures[0] > 0:
|
|
failures[0] -= 1
|
|
raise OSError(errno.EMSGSIZE, "Message too long")
|
|
return original_sendto(sock, data, *args)
|
|
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
z = CultNetRudpSocketTransportConnection(
|
|
CultNetRudpSocketTransportOptions(
|
|
runtime_id="peer-z",
|
|
socket=z_socket,
|
|
mode=CultNetRudpSocketMode.CLIENT,
|
|
connection_id=server.rudp_connection_id,
|
|
remote_addr=("127.0.0.1", server.port),
|
|
resend_delay_ms=25,
|
|
)
|
|
)
|
|
z.connect(b"join")
|
|
# A session that started would resend its Accept, and the resend would get through.
|
|
time.sleep(0.5)
|
|
z_socket.settimeout(0.05)
|
|
received = []
|
|
while True:
|
|
try:
|
|
wire, _ = z_socket.recvfrom(65535)
|
|
except TimeoutError:
|
|
break
|
|
received.append(decode_rudp_packet(wire).packet_type)
|
|
self.assertEqual(received, [CultNetRudpPacketType.DISCONNECT])
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
self.assertIsNotNone(self._served(y))
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_packet_the_session_refuses_ends_only_that_peers_session(self) -> None:
|
|
from cultnet_py import CultNetRudpPacket, encode_rudp_packet
|
|
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
try:
|
|
x, _ = self._client(server, "peer-x")
|
|
y, _ = self._client(server, "peer-y")
|
|
# A fragment whose metadata the session refuses: fragment id 0 names no set.
|
|
refused = CultNetRudpPacket(
|
|
packet_type=CultNetRudpPacketType.DATA,
|
|
connection_id=server.rudp_connection_id,
|
|
sequence=0,
|
|
ack=0,
|
|
ack_mask=0,
|
|
channel_id="schema",
|
|
reliable=False,
|
|
fragment_id=0,
|
|
fragment_index=0,
|
|
fragment_count=2,
|
|
payload=b"x",
|
|
)
|
|
x.socket.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port))
|
|
for _ in range(200):
|
|
x.receive_once()
|
|
if x.disconnect_reason is not None:
|
|
break
|
|
time.sleep(0.005)
|
|
self.assertEqual(x.disconnect_reason, b"session refused a packet")
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
self.assertIsNotNone(self._served(y), "Y is still served")
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_message_whose_handling_fails_ends_only_that_peers_session(self) -> None:
|
|
import msgpack # type: ignore
|
|
|
|
canary = "canary-7f3e1c"
|
|
# Well-formed frames whose fields have the wrong type: handling raises TypeError.
|
|
frames = [
|
|
{"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "schemaIds": 5},
|
|
{"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "recordKeys": True},
|
|
]
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
try:
|
|
y, _ = self._client(server, "peer-y")
|
|
for frame in frames:
|
|
with self.subTest(frame=frame):
|
|
x, _ = self._client(server, "peer-x")
|
|
with self.assertLogs("cultmesh_py.server", level="WARNING") as logs:
|
|
x.send("schema", msgpack.packb(frame, use_bin_type=True))
|
|
for _ in range(200):
|
|
x.receive_once()
|
|
if x.disconnect_reason is not None:
|
|
break
|
|
time.sleep(0.005)
|
|
self.assertEqual(x.disconnect_reason, b"session refused a packet")
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
self.assertIsNotNone(self._served(y), "Y is still served")
|
|
self.assertEqual(len(logs.output), 1)
|
|
self.assertIn("TypeError", logs.output[0])
|
|
self.assertNotIn(canary, logs.output[0])
|
|
finally:
|
|
server.stop()
|
|
|
|
def _raw_peer(self, server: Any) -> tuple[socket.socket, Any]:
|
|
"""A peer driven packet by packet, to deliver frames in an order a client would not."""
|
|
from cultnet_py import CultNetRudpSession, CultNetRudpSessionOptions, encode_rudp_packet
|
|
|
|
sock = bind_udp_socket()
|
|
sock.settimeout(0.05)
|
|
session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=server.rudp_connection_id))
|
|
sock.sendto(encode_rudp_packet(session.create_connect(0, b"join")), ("127.0.0.1", server.port))
|
|
wire, _ = sock.recvfrom(65535)
|
|
accept = decode_rudp_packet(wire)
|
|
self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT)
|
|
session.receive(accept, 0)
|
|
return sock, session
|
|
|
|
def test_the_goodbye_for_a_refused_reliable_frame_does_not_acknowledge_it(self) -> None:
|
|
import dataclasses
|
|
|
|
from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet
|
|
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
try:
|
|
sock, session = self._raw_peer(server)
|
|
# A reliable fragment whose metadata the session refuses: fragment id 0 names no set.
|
|
# The session records its sequence before it refuses it.
|
|
(fragment,) = session.send_many("schema", b"x", CultNetRudpSendOptions(reliable=True, ordered=True))
|
|
refused = dataclasses.replace(fragment, fragment_id=0, fragment_index=0, fragment_count=2)
|
|
sock.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port))
|
|
goodbye = None
|
|
for _ in range(40):
|
|
try:
|
|
wire, _ = sock.recvfrom(65535)
|
|
except TimeoutError:
|
|
continue
|
|
packet = decode_rudp_packet(wire)
|
|
if packet.packet_type == CultNetRudpPacketType.DISCONNECT:
|
|
goodbye = packet
|
|
break
|
|
self.assertIsNotNone(goodbye)
|
|
self.assertEqual(goodbye.payload, b"session refused a packet")
|
|
distance = goodbye.ack - refused.sequence
|
|
acknowledged = distance == 0 or (0 < distance <= 32 and goodbye.ack_mask & (1 << (distance - 1)) != 0)
|
|
self.assertFalse(acknowledged, "the goodbye acknowledged the refused frame")
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_responses_after_one_that_can_never_be_sent_are_not_sent(self) -> None:
|
|
from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet
|
|
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_sendto = socket.socket.sendto
|
|
try:
|
|
sock, session = self._raw_peer(server)
|
|
addr = sock.getsockname()[:2]
|
|
request = hello(runtime_id="probe").to_bytes()
|
|
options = CultNetRudpSendOptions(reliable=True, ordered=True)
|
|
first = session.send("schema", request, options)
|
|
second = session.send("schema", request, options)
|
|
attempts: list[CultNetRudpPacketType] = []
|
|
failures = [1]
|
|
|
|
def sendto(sock_: socket.socket, data: bytes, *args: Any) -> int:
|
|
if args and tuple(args[-1]) == tuple(addr):
|
|
packet_type = decode_rudp_packet(data).packet_type
|
|
attempts.append(packet_type)
|
|
if packet_type == CultNetRudpPacketType.DATA and failures[0] > 0:
|
|
failures[0] -= 1
|
|
raise OSError(errno.EMSGSIZE, "Message too long")
|
|
return original_sendto(sock_, data, *args)
|
|
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
# The second frame waits for the first, so one receive delivers both, and
|
|
# the first of the two responses can never be sent as built.
|
|
sock.sendto(encode_rudp_packet(second), ("127.0.0.1", server.port))
|
|
time.sleep(0.1)
|
|
attempts.clear()
|
|
sock.sendto(encode_rudp_packet(first), ("127.0.0.1", server.port))
|
|
self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts)
|
|
time.sleep(0.1)
|
|
|
|
# The unsendable response, then only the goodbye: no second response, no ack.
|
|
self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT])
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
finally:
|
|
server.stop()
|
|
|
|
def test_a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it(self) -> None:
|
|
server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server"))
|
|
original_sendto = socket.socket.sendto
|
|
try:
|
|
x, x_addr = self._client(server, "peer-x")
|
|
mode = ["transient"]
|
|
attempts: list[CultNetRudpPacketType] = []
|
|
|
|
def sendto(sock: socket.socket, data: bytes, *args: Any) -> int:
|
|
if args and tuple(args[-1]) == tuple(x_addr):
|
|
packet_type = decode_rudp_packet(data).packet_type
|
|
if mode[0] != "transient":
|
|
attempts.append(packet_type)
|
|
# Only responses fail; X's requests are acknowledged, so X goes quiet.
|
|
if packet_type == CultNetRudpPacketType.DATA and mode[0] == "transient":
|
|
raise OSError(errno.ENOBUFS, "injected transient failure")
|
|
if packet_type == CultNetRudpPacketType.DATA and mode[0] == "permanent":
|
|
mode[0] = "healthy"
|
|
raise OSError(errno.EMSGSIZE, "Message too long")
|
|
return original_sendto(sock, data, *args)
|
|
|
|
with patch.object(socket.socket, "sendto", sendto):
|
|
x.send("schema", hello(runtime_id="one").to_bytes())
|
|
x.send("schema", hello(runtime_id="two").to_bytes())
|
|
for _ in range(40):
|
|
x.receive_once()
|
|
time.sleep(0.005)
|
|
self._wait_for(lambda: server.rudp_send_failures >= 2)
|
|
# Both responses are owed; the next resend of the first can never be sent.
|
|
mode[0] = "permanent"
|
|
self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts)
|
|
time.sleep(0.2)
|
|
|
|
self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT])
|
|
self.assertTrue(server._rudp_thread.is_alive())
|
|
finally:
|
|
server.stop()
|
|
|
|
def _wait_for(self, predicate: Callable[[], bool]) -> None:
|
|
deadline = time.monotonic() + 2.0
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.005)
|
|
self.fail("condition not reached")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|